From 4c67d20af73c11010738c43d56dffe4762400e50 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Nov 2024 12:35:03 +0000 Subject: [PATCH 001/691] link to element-docker-demo from contrib/docker* (#17953) --- changelog.d/17953.doc | 1 + contrib/docker/README.md | 3 +++ contrib/docker_compose_workers/README.md | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 changelog.d/17953.doc diff --git a/changelog.d/17953.doc b/changelog.d/17953.doc new file mode 100644 index 0000000000..10f5a27ba9 --- /dev/null +++ b/changelog.d/17953.doc @@ -0,0 +1 @@ +Link to element-docker-demo from contrib/docker*. diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 89c1518bd0..fdfa96795a 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -30,3 +30,6 @@ docker-compose up -d ### More information For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md) + +**For a more comprehensive Docker Compose example showcasing a full Matrix 2.0 stack, please see +https://github.com/element-hq/element-docker-demo** \ No newline at end of file diff --git a/contrib/docker_compose_workers/README.md b/contrib/docker_compose_workers/README.md index 81518f6ba1..16c8c26795 100644 --- a/contrib/docker_compose_workers/README.md +++ b/contrib/docker_compose_workers/README.md @@ -8,6 +8,9 @@ All examples and snippets assume that your Synapse service is called `synapse` i An example Docker Compose file can be found [here](docker-compose.yaml). +**For a more comprehensive Docker Compose example, showcasing a full Matrix 2.0 stack (originally based on this +docker-compose.yaml), please see https://github.com/element-hq/element-docker-demo** + ## Worker Service Examples in Docker Compose In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`). From 4587decd678300217969f1d2f69b226421a33ced Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 Nov 2024 04:37:19 -0800 Subject: [PATCH 002/691] Return suspended status when querying user account (#17952) --- changelog.d/17952.misc | 1 + docs/admin_api/user_admin_api.md | 3 ++- synapse/handlers/admin.py | 1 + tests/rest/admin/test_user.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17952.misc diff --git a/changelog.d/17952.misc b/changelog.d/17952.misc new file mode 100644 index 0000000000..84fc8bfc29 --- /dev/null +++ b/changelog.d/17952.misc @@ -0,0 +1 @@ +Return whether the user is suspended when querying the user account in the Admin API. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 96a2994b7b..a6e2e0a153 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -55,7 +55,8 @@ It returns a JSON body like the following: } ], "user_type": null, - "locked": false + "locked": false, + "suspended": false } ``` diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index d1989e9d2c..d1194545ae 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -124,6 +124,7 @@ class AdminHandler: "consent_ts": user_info.consent_ts, "user_type": user_info.user_type, "is_guest": user_info.is_guest, + "suspended": user_info.suspended, } if self._msc3866_enabled: diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 6d050e7784..fdb8fafa0e 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -3222,6 +3222,7 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertIn("consent_ts", content) self.assertIn("external_ids", content) self.assertIn("last_seen_ts", content) + self.assertIn("suspended", content) # This key was removed intentionally. Ensure it is not accidentally re-included. self.assertNotIn("password_hash", content) From 93cc9550519fd71a4929e23224ab9b4d2738705c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:23:32 +0000 Subject: [PATCH 003/691] Bump tornado from 6.4.1 to 6.4.2 (#17955) --- poetry.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index eece221095..cbabacea57 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2526,22 +2526,22 @@ files = [ [[package]] name = "tornado" -version = "6.4.1" +version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -optional = true +optional = false python-versions = ">=3.8" files = [ - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, - {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, - {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, - {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] [[package]] From 3943d2fde746b3adacffa25794dbf4acea249870 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Nov 2024 18:12:33 +0000 Subject: [PATCH 004/691] Fix up logic for delaying sending read receipts over federation. (#17933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For context of why we delay read receipts, see https://github.com/matrix-org/synapse/issues/4730. Element Web often sends read receipts in quick succession, if it reloads the timeline it'll send one for the last message in the old timeline and again for the last message in the new timeline. This caused remote users to see a read receipt for older messages come through quickly, but then the second read receipt taking a while to arrive for the most recent message. There are two things going on in this PR: 1. There was a mismatch between seconds and milliseconds, and so we ended up delaying for far longer than intended. 2. Changing the logic to reuse the `DestinationWakeupQueue` (used for presence) The changes in logic are: - Treat the first receipt and subsequent receipts in a room in the same way - Whitelist certain classes of receipts to never delay being sent, i.e. receipts in small rooms, receipts for events that were sent within the last 60s, and sending receipts to the event sender's server. - The maximum delay a receipt can have before being sent to a server is 30s, and we'll send out receipts to remotes at least at 50Hz (by default) The upshot is that this should make receipts feel more snappy over federation. This new logic should send roughly between 10%–20% of transactions immediately on matrix.org. --- changelog.d/17933.bugfix | 1 + synapse/federation/sender/__init__.py | 169 ++++++++++-------- .../sender/per_destination_queue.py | 25 +-- synapse/storage/databases/main/cache.py | 2 + .../storage/databases/main/events_worker.py | 27 +++ tests/federation/test_federation_sender.py | 158 +++++++++++----- 6 files changed, 234 insertions(+), 148 deletions(-) create mode 100644 changelog.d/17933.bugfix diff --git a/changelog.d/17933.bugfix b/changelog.d/17933.bugfix new file mode 100644 index 0000000000..8d30ac587e --- /dev/null +++ b/changelog.d/17933.bugfix @@ -0,0 +1 @@ +Fix long-standing bug where read receipts could get overly delayed being sent over federation. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 1888480881..17cddf18a3 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -140,7 +140,6 @@ from typing import ( Iterable, List, Optional, - Set, Tuple, ) @@ -170,7 +169,13 @@ from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) -from synapse.types import JsonDict, ReadReceipt, RoomStreamToken, StrCollection +from synapse.types import ( + JsonDict, + ReadReceipt, + RoomStreamToken, + StrCollection, + get_domain_from_id, +) from synapse.util import Clock from synapse.util.metrics import Measure from synapse.util.retryutils import filter_destinations_by_retry_limiter @@ -297,12 +302,10 @@ class _DestinationWakeupQueue: # being woken up. _MAX_TIME_IN_QUEUE = 30.0 - # The maximum duration in seconds between waking up consecutive destination - # queues. - _MAX_DELAY = 0.1 - sender: "FederationSender" = attr.ib() clock: Clock = attr.ib() + max_delay_s: int = attr.ib() + queue: "OrderedDict[str, Literal[None]]" = attr.ib(factory=OrderedDict) processing: bool = attr.ib(default=False) @@ -332,7 +335,7 @@ class _DestinationWakeupQueue: # We also add an upper bound to the delay, to gracefully handle the # case where the queue only has a few entries in it. current_sleep_seconds = min( - self._MAX_DELAY, self._MAX_TIME_IN_QUEUE / len(self.queue) + self.max_delay_s, self._MAX_TIME_IN_QUEUE / len(self.queue) ) while self.queue: @@ -416,19 +419,14 @@ class FederationSender(AbstractFederationSender): self._is_processing = False self._last_poked_id = -1 - # map from room_id to a set of PerDestinationQueues which we believe are - # awaiting a call to flush_read_receipts_for_room. The presence of an entry - # here for a given room means that we are rate-limiting RR flushes to that room, - # and that there is a pending call to _flush_rrs_for_room in the system. - self._queues_awaiting_rr_flush_by_room: Dict[str, Set[PerDestinationQueue]] = {} - - self._rr_txn_interval_per_room_ms = ( - 1000.0 - / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second - ) - self._external_cache = hs.get_external_cache() - self._destination_wakeup_queue = _DestinationWakeupQueue(self, self.clock) + + rr_txn_interval_per_room_s = ( + 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + self._destination_wakeup_queue = _DestinationWakeupQueue( + self, self.clock, max_delay_s=rr_txn_interval_per_room_s + ) # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( @@ -745,37 +743,48 @@ class FederationSender(AbstractFederationSender): # Some background on the rate-limiting going on here. # - # It turns out that if we attempt to send out RRs as soon as we get them from - # a client, then we end up trying to do several hundred Hz of federation - # transactions. (The number of transactions scales as O(N^2) on the size of a - # room, since in a large room we have both more RRs coming in, and more servers - # to send them to.) + # It turns out that if we attempt to send out RRs as soon as we get them + # from a client, then we end up trying to do several hundred Hz of + # federation transactions. (The number of transactions scales as O(N^2) + # on the size of a room, since in a large room we have both more RRs + # coming in, and more servers to send them to.) # - # This leads to a lot of CPU load, and we end up getting behind. The solution - # currently adopted is as follows: + # This leads to a lot of CPU load, and we end up getting behind. The + # solution currently adopted is to differentiate between receipts and + # destinations we should immediately send to, and those we can trickle + # the receipts to. # - # The first receipt in a given room is sent out immediately, at time T0. Any - # further receipts are, in theory, batched up for N seconds, where N is calculated - # based on the number of servers in the room to achieve a transaction frequency - # of around 50Hz. So, for example, if there were 100 servers in the room, then - # N would be 100 / 50Hz = 2 seconds. + # The current logic is to send receipts out immediately if: + # - the room is "small", i.e. there's only N servers to send receipts + # to, and so sending out the receipts immediately doesn't cause too + # much load; or + # - the receipt is for an event that happened recently, as users + # notice if receipts are delayed when they know other users are + # currently reading the room; or + # - the receipt is being sent to the server that sent the event, so + # that users see receipts for their own receipts quickly. # - # Then, after T+N, we flush out any receipts that have accumulated, and restart - # the timer to flush out more receipts at T+2N, etc. If no receipts accumulate, - # we stop the cycle and go back to the start. + # For destinations that we should delay sending the receipt to, we queue + # the receipts up to be sent in the next transaction, but don't trigger + # a new transaction to be sent. We then add the destination to the + # `DestinationWakeupQueue`, which will slowly iterate over each + # destination and trigger a new transaction to be sent. # - # However, in practice, it is often possible to flush out receipts earlier: in - # particular, if we are sending a transaction to a given server anyway (for - # example, because we have a PDU or a RR in another room to send), then we may - # as well send out all of the pending RRs for that server. So it may be that - # by the time we get to T+N, we don't actually have any RRs left to send out. - # Nevertheless we continue to buffer up RRs for the room in question until we - # reach the point that no RRs arrive between timer ticks. + # However, in practice, it is often possible to send out delayed + # receipts earlier: in particular, if we are sending a transaction to a + # given server anyway (for example, because we have a PDU or a RR in + # another room to send), then we may as well send out all of the pending + # RRs for that server. So it may be that by the time we get to waking up + # the destination, we don't actually have any RRs left to send out. # - # For even more background, see https://github.com/matrix-org/synapse/issues/4730. + # For even more background, see + # https://github.com/matrix-org/synapse/issues/4730. room_id = receipt.room_id + # Local read receipts always have 1 event ID. + event_id = receipt.event_ids[0] + # Work out which remote servers should be poked and poke them. domains_set = await self._storage_controllers.state.get_current_hosts_in_room_or_partial_state_approximation( room_id @@ -797,49 +806,51 @@ class FederationSender(AbstractFederationSender): if not domains: return - queues_pending_flush = self._queues_awaiting_rr_flush_by_room.get(room_id) + # We now split which domains we want to wake up immediately vs which we + # want to delay waking up. + immediate_domains: StrCollection + delay_domains: StrCollection - # if there is no flush yet scheduled, we will send out these receipts with - # immediate flushes, and schedule the next flush for this room. - if queues_pending_flush is not None: - logger.debug("Queuing receipt for: %r", domains) + if len(domains) < 10: + # For "small" rooms send to all domains immediately + immediate_domains = domains + delay_domains = () else: - logger.debug("Sending receipt to: %r", domains) - self._schedule_rr_flush_for_room(room_id, len(domains)) + metadata = await self.store.get_metadata_for_event( + receipt.room_id, event_id + ) + assert metadata is not None - for domain in domains: + sender_domain = get_domain_from_id(metadata.sender) + + if self.clock.time_msec() - metadata.received_ts < 60_000: + # We always send receipts for recent messages immediately + immediate_domains = domains + delay_domains = () + else: + # Otherwise, we delay waking up all destinations except for the + # sender's domain. + immediate_domains = [] + delay_domains = [] + for domain in domains: + if domain == sender_domain: + immediate_domains.append(domain) + else: + delay_domains.append(domain) + + for domain in immediate_domains: + # Add to destination queue and wake the destination up + queue = self._get_per_destination_queue(domain) + queue.queue_read_receipt(receipt) + queue.attempt_new_transaction() + + for domain in delay_domains: + # Add to destination queue... queue = self._get_per_destination_queue(domain) queue.queue_read_receipt(receipt) - # if there is already a RR flush pending for this room, then make sure this - # destination is registered for the flush - if queues_pending_flush is not None: - queues_pending_flush.add(queue) - else: - queue.flush_read_receipts_for_room(room_id) - - def _schedule_rr_flush_for_room(self, room_id: str, n_domains: int) -> None: - # that is going to cause approximately len(domains) transactions, so now back - # off for that multiplied by RR_TXN_INTERVAL_PER_ROOM - backoff_ms = self._rr_txn_interval_per_room_ms * n_domains - - logger.debug("Scheduling RR flush in %s in %d ms", room_id, backoff_ms) - self.clock.call_later(backoff_ms, self._flush_rrs_for_room, room_id) - self._queues_awaiting_rr_flush_by_room[room_id] = set() - - def _flush_rrs_for_room(self, room_id: str) -> None: - queues = self._queues_awaiting_rr_flush_by_room.pop(room_id) - logger.debug("Flushing RRs in %s to %s", room_id, queues) - - if not queues: - # no more RRs arrived for this room; we are done. - return - - # schedule the next flush - self._schedule_rr_flush_for_room(room_id, len(queues)) - - for queue in queues: - queue.flush_read_receipts_for_room(room_id) + # ... and schedule the destination to be woken up. + self._destination_wakeup_queue.add_to_queue(domain) async def send_presence_to_destinations( self, states: Iterable[UserPresenceState], destinations: Iterable[str] diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index d097e65ea7..b3f65e8237 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -156,7 +156,6 @@ class PerDestinationQueue: # Each receipt can only have a single receipt per # (room ID, receipt type, user ID, thread ID) tuple. self._pending_receipt_edus: List[Dict[str, Dict[str, Dict[str, dict]]]] = [] - self._rrs_pending_flush = False # stream_id of last successfully sent to-device message. # NB: may be a long or an int. @@ -258,15 +257,7 @@ class PerDestinationQueue: } ) - def flush_read_receipts_for_room(self, room_id: str) -> None: - # If there are any pending receipts for this room then force-flush them - # in a new transaction. - for edu in self._pending_receipt_edus: - if room_id in edu: - self._rrs_pending_flush = True - self.attempt_new_transaction() - # No use in checking remaining EDUs if the room was found. - break + self.mark_new_data() def send_keyed_edu(self, edu: Edu, key: Hashable) -> None: self._pending_edus_keyed[(edu.edu_type, key)] = edu @@ -603,12 +594,9 @@ class PerDestinationQueue: self._destination, last_successful_stream_ordering ) - def _get_receipt_edus(self, force_flush: bool, limit: int) -> Iterable[Edu]: + def _get_receipt_edus(self, limit: int) -> Iterable[Edu]: if not self._pending_receipt_edus: return - if not force_flush and not self._rrs_pending_flush: - # not yet time for this lot - return # Send at most limit EDUs for receipts. for content in self._pending_receipt_edus[:limit]: @@ -747,7 +735,7 @@ class _TransactionQueueManager: ) # Add read receipt EDUs. - pending_edus.extend(self.queue._get_receipt_edus(force_flush=False, limit=5)) + pending_edus.extend(self.queue._get_receipt_edus(limit=5)) edu_limit = MAX_EDUS_PER_TRANSACTION - len(pending_edus) # Next, prioritize to-device messages so that existing encryption channels @@ -795,13 +783,6 @@ class _TransactionQueueManager: if not self._pdus and not pending_edus: return [], [] - # if we've decided to send a transaction anyway, and we have room, we - # may as well send any pending RRs - if edu_limit: - pending_edus.extend( - self.queue._get_receipt_edus(force_flush=True, limit=edu_limit) - ) - if self._pdus: self._last_stream_ordering = self._pdus[ -1 diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 32c3472e58..707d18de78 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -322,6 +322,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "get_unread_event_push_actions_by_room_for_user", (room_id,) ) + self._attempt_to_invalidate_cache("get_metadata_for_event", (room_id, event_id)) self._attempt_to_invalidate_cache("_get_max_event_pos", (room_id,)) @@ -446,6 +447,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache("_get_state_group_for_event", None) self._attempt_to_invalidate_cache("get_event_ordering", None) + self._attempt_to_invalidate_cache("get_metadata_for_event", (room_id,)) self._attempt_to_invalidate_cache("is_partial_state_event", None) self._attempt_to_invalidate_cache("_get_joined_profile_from_event_id", None) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 403407068c..825fd00993 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -193,6 +193,14 @@ class _EventRow: outlier: bool +@attr.s(slots=True, frozen=True, auto_attribs=True) +class EventMetadata: + """Event metadata returned by `get_metadata_for_event(..)`""" + + sender: str + received_ts: int + + class EventRedactBehaviour(Enum): """ What to do when retrieving a redacted event from the database. @@ -2580,3 +2588,22 @@ class EventsWorkerStore(SQLBaseStore): _BackgroundUpdates.SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_BG_UPDATE, ) ) + + @cached(tree=True) + async def get_metadata_for_event( + self, room_id: str, event_id: str + ) -> Optional[EventMetadata]: + row = await self.db_pool.simple_select_one( + table="events", + keyvalues={"room_id": room_id, "event_id": event_id}, + retcols=("sender", "received_ts"), + allow_none=True, + desc="get_metadata_for_event", + ) + if row is None: + return None + + return EventMetadata( + sender=row[0], + received_ts=row[1], + ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 6a8887fe74..cd906bbbc7 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -34,6 +34,7 @@ from synapse.handlers.device import DeviceHandler from synapse.rest import admin from synapse.rest.client import login from synapse.server import HomeServer +from synapse.storage.databases.main.events_worker import EventMetadata from synapse.types import JsonDict, ReadReceipt from synapse.util import Clock @@ -55,12 +56,15 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): federation_transport_client=self.federation_transport_client, ) - hs.get_storage_controllers().state.get_current_hosts_in_room = AsyncMock( # type: ignore[method-assign] + self.main_store = hs.get_datastores().main + self.state_controller = hs.get_storage_controllers().state + + self.state_controller.get_current_hosts_in_room = AsyncMock( # type: ignore[method-assign] return_value={"test", "host2"} ) - hs.get_storage_controllers().state.get_current_hosts_in_room_or_partial_state_approximation = ( # type: ignore[method-assign] - hs.get_storage_controllers().state.get_current_hosts_in_room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = ( # type: ignore[method-assign] + self.state_controller.get_current_hosts_in_room ) return hs @@ -185,12 +189,15 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): ], ) - def test_send_receipts_with_backoff(self) -> None: - """Send two receipts in quick succession; the second should be flushed, but - only after 20ms""" + def test_send_receipts_with_backoff_small_room(self) -> None: + """Read receipt in small rooms should not be delayed""" mock_send_transaction = self.federation_transport_client.send_transaction mock_send_transaction.return_value = {} + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test", "host2"} + ) + sender = self.hs.get_federation_sender() receipt = ReadReceipt( "room_id", @@ -206,7 +213,104 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): # expect a call to send_transaction mock_send_transaction.assert_called_once() - json_cb = mock_send_transaction.call_args[0][1] + self._assert_edu_in_call(mock_send_transaction.call_args[0][1]) + + def test_send_receipts_with_backoff_recent_event(self) -> None: + """Read receipt for a recent message should not be delayed""" + mock_send_transaction = self.federation_transport_client.send_transaction + mock_send_transaction.return_value = {} + + # Pretend this is a big room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test"} | {f"host{i}" for i in range(20)} + ) + + self.main_store.get_metadata_for_event = AsyncMock( + return_value=EventMetadata( + received_ts=self.clock.time_msec(), + sender="@test:test", + ) + ) + + sender = self.hs.get_federation_sender() + receipt = ReadReceipt( + "room_id", + "m.read", + "user_id", + ["event_id"], + thread_id=None, + data={"ts": 1234}, + ) + self.get_success(sender.send_read_receipt(receipt)) + + self.pump() + + # expect a call to send_transaction for each host + self.assertEqual(mock_send_transaction.call_count, 20) + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + + mock_send_transaction.reset_mock() + + def test_send_receipts_with_backoff_sender(self) -> None: + """Read receipt for a message should not be delayed to the sender, but + is delayed to everyone else""" + mock_send_transaction = self.federation_transport_client.send_transaction + mock_send_transaction.return_value = {} + + # Pretend this is a big room + self.state_controller.get_current_hosts_in_room_or_partial_state_approximation = AsyncMock( # type: ignore[method-assign] + return_value={"test"} | {f"host{i}" for i in range(20)} + ) + + self.main_store.get_metadata_for_event = AsyncMock( + return_value=EventMetadata( + received_ts=self.clock.time_msec() - 5 * 60_000, + sender="@test:host1", + ) + ) + + sender = self.hs.get_federation_sender() + receipt = ReadReceipt( + "room_id", + "m.read", + "user_id", + ["event_id"], + thread_id=None, + data={"ts": 1234}, + ) + self.get_success(sender.send_read_receipt(receipt)) + + self.pump() + + # First, expect a call to send_transaction for the sending host + mock_send_transaction.assert_called() + + transaction = mock_send_transaction.call_args_list[0].args[0] + self.assertEqual(transaction.destination, "host1") + self._assert_edu_in_call(mock_send_transaction.call_args_list[0].args[1]) + + # We also expect a call to one of the other hosts, as the first + # destination to wake up. + self.assertEqual(mock_send_transaction.call_count, 2) + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + + mock_send_transaction.reset_mock() + + # We now expect to see 18 more transactions to the remaining hosts + # periodically. + for _ in range(18): + self.reactor.advance( + 1.0 + / self.hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + + mock_send_transaction.assert_called_once() + self._assert_edu_in_call(mock_send_transaction.call_args.args[1]) + mock_send_transaction.reset_mock() + + def _assert_edu_in_call(self, json_cb: Callable[[], JsonDict]) -> None: + """Assert that the given `json_cb` from a `send_transaction` has a + receipt in it.""" data = json_cb() self.assertEqual( data["edus"], @@ -226,46 +330,6 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): } ], ) - mock_send_transaction.reset_mock() - - # send the second RR - receipt = ReadReceipt( - "room_id", - "m.read", - "user_id", - ["other_id"], - thread_id=None, - data={"ts": 1234}, - ) - self.successResultOf(defer.ensureDeferred(sender.send_read_receipt(receipt))) - self.pump() - mock_send_transaction.assert_not_called() - - self.reactor.advance(19) - mock_send_transaction.assert_not_called() - - self.reactor.advance(10) - mock_send_transaction.assert_called_once() - json_cb = mock_send_transaction.call_args[0][1] - data = json_cb() - self.assertEqual( - data["edus"], - [ - { - "edu_type": EduTypes.RECEIPT, - "content": { - "room_id": { - "m.read": { - "user_id": { - "event_ids": ["other_id"], - "data": {"ts": 1234}, - } - } - } - }, - } - ], - ) class FederationSenderPresenceTestCases(HomeserverTestCase): From 02aa7adf4c3f3bf71ddfcd4bd80d6adcf74a444c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 26 Nov 2024 07:45:18 +0000 Subject: [PATCH 005/691] Fix `delete_old_otks` job on worker deployments (#17960) In a worker-mode deployment, the `E2eKeysHandler` is not necessarily loaded, which means the handler for the `delete_old_otks` task will not be registered. Make sure we load the handler. Introduced in https://github.com/element-hq/synapse/pull/17934 --- changelog.d/17960.bugfix | 1 + synapse/server.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/17960.bugfix diff --git a/changelog.d/17960.bugfix b/changelog.d/17960.bugfix new file mode 100644 index 0000000000..767085320d --- /dev/null +++ b/changelog.d/17960.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. diff --git a/synapse/server.py b/synapse/server.py index c7b4918813..462e15cc2f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -254,6 +254,7 @@ class HomeServer(metaclass=abc.ABCMeta): "auth", "deactivate_account", "delayed_events", + "e2e_keys", # for the `delete_old_otks` scheduled-task handler "message", "pagination", "profile", From cd7d90bd281e235d91411d84e4724d01d849b314 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:30:16 +0000 Subject: [PATCH 006/691] Bump tomli from 2.0.2 to 2.1.0 (#17959) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index cbabacea57..14bb9ad172 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2515,13 +2515,13 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.0.2" +version = "2.1.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, - {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, + {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, + {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, ] [[package]] From 8c653e1dd6c8f18f2f9e2d78d37877a70dba1b2d Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 26 Nov 2024 14:11:12 +0100 Subject: [PATCH 007/691] 1.120.0 --- CHANGES.md | 9 +++++++++ changelog.d/17960.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/17960.bugfix diff --git a/CHANGES.md b/CHANGES.md index 31d9914b39..0caac3f89e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.120.0 (2024-11-26) + +### Bugfixes + +- Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. ([\#17960](https://github.com/element-hq/synapse/issues/17960)) + + + + # Synapse 1.120.0rc1 (2024-11-20) This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the diff --git a/changelog.d/17960.bugfix b/changelog.d/17960.bugfix deleted file mode 100644 index 767085320d..0000000000 --- a/changelog.d/17960.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. diff --git a/debian/changelog b/debian/changelog index d7cec3fa8a..bd4466d7aa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.120.0) stable; urgency=medium + + * New synapse release 1.120.0. + + -- Synapse Packaging team Tue, 26 Nov 2024 13:10:23 +0000 + matrix-synapse-py3 (1.120.0~rc1) stable; urgency=medium * New Synapse release 1.120.0rc1. diff --git a/pyproject.toml b/pyproject.toml index f0a4b682de..5fd1d7c198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.120.0rc1" +version = "1.120.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From cee9da0da53ba9d04fbb989e2b35108266b2e787 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 26 Nov 2024 19:43:26 +0100 Subject: [PATCH 008/691] MSC4108: Add a Content-Type header on the PUT response (#17253) This is a workaround for some proxy setup, where the ETag header gets stripped from the response headers unless there is a Content-Type header set. In particular, we saw this bug when putting Cloudflare in front of Synapse. I'm pretty sure this is a Cloudflare bug, as this behaviour isn't documented anywhere, and doesn't make sense whatsoever. --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/17253.misc | 1 + rust/src/rendezvous/mod.rs | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/17253.misc diff --git a/changelog.d/17253.misc b/changelog.d/17253.misc new file mode 100644 index 0000000000..868691624d --- /dev/null +++ b/changelog.d/17253.misc @@ -0,0 +1 @@ +[MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index f69f45490f..55df0203f3 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -288,6 +288,13 @@ impl RendezvousHandler { let mut response = Response::new(Bytes::new()); *response.status_mut() = StatusCode::ACCEPTED; prepare_headers(response.headers_mut(), session); + + // Even though this isn't mandated by the MSC, we set a Content-Type on the response. It + // doesn't do any harm as the body is empty, but this helps escape a bug in some reverse + // proxy/cache setup which strips the ETag header if there is no Content-Type set. + // Specifically, we noticed this behaviour when placing Synapse behind Cloudflare. + response.headers_mut().typed_insert(ContentType::text()); + http_response_to_twisted(twisted_request, response)?; Ok(()) From a58f09acc78d2497fc7c3c8930c42233bcc7428c Mon Sep 17 00:00:00 2001 From: V02460 Date: Wed, 27 Nov 2024 11:46:00 +0100 Subject: [PATCH 009/691] Bump pyo3 to v0.23.2 (#17966) Keep up-to-date with pyo3 releases. This bump enables Python 3.13 support and resolves deprecations. Links for quick reference: https://github.com/PyO3/pyo3/releases https://github.com/davidhewitt/pythonize/releases https://github.com/vorner/pyo3-log --- Cargo.lock | 158 +++------------------------ changelog.d/17966.misc | 1 + rust/Cargo.toml | 6 +- rust/src/acl/mod.rs | 4 +- rust/src/events/internal_metadata.rs | 82 +++++++++----- rust/src/events/mod.rs | 4 +- rust/src/http.rs | 2 +- rust/src/lib.rs | 15 +++ rust/src/push/evaluator.rs | 2 + rust/src/push/mod.rs | 44 +++++--- rust/src/rendezvous/mod.rs | 13 ++- 11 files changed, 136 insertions(+), 195 deletions(-) create mode 100644 changelog.d/17966.misc diff --git a/Cargo.lock b/Cargo.lock index 5c8f627fd7..b7084165ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,12 +35,6 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - [[package]] name = "blake2" version = "0.10.6" @@ -162,9 +156,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hex" @@ -222,16 +216,6 @@ version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.22" @@ -265,29 +249,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "parking_lot" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - [[package]] name = "portable-atomic" version = "1.6.0" @@ -311,16 +272,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.21.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "f54b3d09cbdd1f8c20650b28e7b09e338881482f4aa908a5f61a00c98fba2690" dependencies = [ "anyhow", "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -330,9 +291,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "3015cf985888fe66cfb63ce0e321c603706cd541b7aec7ddd35c281390af45d8" dependencies = [ "once_cell", "target-lexicon", @@ -340,9 +301,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "6fca7cd8fd809b5ac4eefb89c1f98f7a7651d3739dfb341ca6980090f554c270" dependencies = [ "libc", "pyo3-build-config", @@ -350,9 +311,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af49834b8d2ecd555177e63b273b708dea75150abc6f5341d0a6e1a9623976c" +checksum = "3eb421dc86d38d08e04b927b02424db480be71b777fa3a56f32e2f2a3a1a3b08" dependencies = [ "arc-swap", "log", @@ -361,9 +322,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "34e657fa5379a79151b6ff5328d9216a84f55dc93b17b08e7c3609a969b73aa0" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -373,9 +334,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "295548d5ffd95fd1981d2d3cf4458831b21d60af046b729b6fd143b0ba7aee2f" dependencies = [ "heck", "proc-macro2", @@ -386,9 +347,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.21.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c" +checksum = "91a6ee7a084f913f98d70cdc3ebec07e852b735ae3059a1500db2661265da9ff" dependencies = [ "pyo3", "serde", @@ -433,15 +394,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.11.1" @@ -477,12 +429,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "serde" version = "1.0.215" @@ -537,12 +483,6 @@ dependencies = [ "digest", ] -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - [[package]] name = "subtle" version = "2.5.0" @@ -694,67 +634,3 @@ dependencies = [ "js-sys", "wasm-bindgen", ] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/changelog.d/17966.misc b/changelog.d/17966.misc new file mode 100644 index 0000000000..c6d6e55fbf --- /dev/null +++ b/changelog.d/17966.misc @@ -0,0 +1 @@ +Bump pyo3 and dependencies to v0.23.2. \ No newline at end of file diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 026487275c..7eebeb3b55 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,14 +30,14 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.21.0", features = [ +pyo3 = { version = "0.23.2", features = [ "macros", "anyhow", "abi3", "abi3-py38", ] } -pyo3-log = "0.10.0" -pythonize = "0.21.0" +pyo3-log = "0.12.0" +pythonize = "0.23.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs index 982720ba90..57b45475fd 100644 --- a/rust/src/acl/mod.rs +++ b/rust/src/acl/mod.rs @@ -32,14 +32,14 @@ use crate::push::utils::{glob_to_regex, GlobMatchType}; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "acl")?; + let child_module = PyModule::new(py, "acl")?; child_module.add_class::()?; m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import acl` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.acl", child_module)?; diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index ad87825f16..eeb6074c10 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -41,9 +41,11 @@ use pyo3::{ pybacked::PyBackedStr, pyclass, pymethods, types::{PyAnyMethods, PyDict, PyDictMethods, PyString}, - Bound, IntoPy, PyAny, PyObject, PyResult, Python, + Bound, IntoPyObject, PyAny, PyObject, PyResult, Python, }; +use crate::UnwrapInfallible; + /// Definitions of the various fields of the internal metadata. #[derive(Clone)] enum EventInternalMetadataData { @@ -60,31 +62,59 @@ enum EventInternalMetadataData { impl EventInternalMetadataData { /// Convert the field to its name and python object. - fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, PyObject) { + fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, Bound<'a, PyAny>) { match self { - EventInternalMetadataData::OutOfBandMembership(o) => { - (pyo3::intern!(py, "out_of_band_membership"), o.into_py(py)) - } - EventInternalMetadataData::SendOnBehalfOf(o) => { - (pyo3::intern!(py, "send_on_behalf_of"), o.into_py(py)) - } - EventInternalMetadataData::RecheckRedaction(o) => { - (pyo3::intern!(py, "recheck_redaction"), o.into_py(py)) - } - EventInternalMetadataData::SoftFailed(o) => { - (pyo3::intern!(py, "soft_failed"), o.into_py(py)) - } - EventInternalMetadataData::ProactivelySend(o) => { - (pyo3::intern!(py, "proactively_send"), o.into_py(py)) - } - EventInternalMetadataData::Redacted(o) => { - (pyo3::intern!(py, "redacted"), o.into_py(py)) - } - EventInternalMetadataData::TxnId(o) => (pyo3::intern!(py, "txn_id"), o.into_py(py)), - EventInternalMetadataData::TokenId(o) => (pyo3::intern!(py, "token_id"), o.into_py(py)), - EventInternalMetadataData::DeviceId(o) => { - (pyo3::intern!(py, "device_id"), o.into_py(py)) - } + EventInternalMetadataData::OutOfBandMembership(o) => ( + pyo3::intern!(py, "out_of_band_membership"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::SendOnBehalfOf(o) => ( + pyo3::intern!(py, "send_on_behalf_of"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::RecheckRedaction(o) => ( + pyo3::intern!(py, "recheck_redaction"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::SoftFailed(o) => ( + pyo3::intern!(py, "soft_failed"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::ProactivelySend(o) => ( + pyo3::intern!(py, "proactively_send"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::Redacted(o) => ( + pyo3::intern!(py, "redacted"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), + EventInternalMetadataData::TxnId(o) => ( + pyo3::intern!(py, "txn_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::TokenId(o) => ( + pyo3::intern!(py, "token_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), + EventInternalMetadataData::DeviceId(o) => ( + pyo3::intern!(py, "device_id"), + o.into_pyobject(py).unwrap_infallible().into_any(), + ), } } @@ -247,7 +277,7 @@ impl EventInternalMetadata { /// /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here. fn get_dict(&self, py: Python<'_>) -> PyResult { - let dict = PyDict::new_bound(py); + let dict = PyDict::new(py); for entry in &self.data { let (key, value) = entry.to_python_pair(py); diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 0bb6cdb181..209efb917b 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -30,7 +30,7 @@ mod internal_metadata; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "events")?; + let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; @@ -38,7 +38,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import events` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.events", child_module)?; diff --git a/rust/src/http.rs b/rust/src/http.rs index af052ab721..63ed05be54 100644 --- a/rust/src/http.rs +++ b/rust/src/http.rs @@ -70,7 +70,7 @@ pub fn http_request_from_twisted(request: &Bound<'_, PyAny>) -> PyResult, m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } + +pub trait UnwrapInfallible { + fn unwrap_infallible(self) -> T; +} + +impl UnwrapInfallible for Result { + fn unwrap_infallible(self) -> T { + match self { + Ok(val) => val, + Err(never) => match never {}, + } + } +} diff --git a/rust/src/push/evaluator.rs b/rust/src/push/evaluator.rs index 0d436a1d7b..db406acb88 100644 --- a/rust/src/push/evaluator.rs +++ b/rust/src/push/evaluator.rs @@ -167,6 +167,7 @@ impl PushRuleEvaluator { /// /// Returns the set of actions, if any, that match (filtering out any /// `dont_notify` and `coalesce` actions). + #[pyo3(signature = (push_rules, user_id=None, display_name=None))] pub fn run( &self, push_rules: &FilteredPushRules, @@ -236,6 +237,7 @@ impl PushRuleEvaluator { } /// Check if the given condition matches. + #[pyo3(signature = (condition, user_id=None, display_name=None))] fn matches( &self, condition: Condition, diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index ef8ed150d4..bd0e853ac3 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -65,8 +65,8 @@ use anyhow::{Context, Error}; use log::warn; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyList, PyLong, PyString}; -use pythonize::{depythonize_bound, pythonize}; +use pyo3::types::{PyBool, PyInt, PyList, PyString}; +use pythonize::{depythonize, pythonize, PythonizeError}; use serde::de::Error as _; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -79,7 +79,7 @@ pub mod utils; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "push")?; + let child_module = PyModule::new(py, "push")?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; @@ -90,7 +90,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import push` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.push", child_module)?; @@ -182,12 +182,16 @@ pub enum Action { Unknown(Value), } -impl IntoPy for Action { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Action { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'py>) -> Result { // When we pass the `Action` struct to Python we want it to be converted // to a dict. We use `pythonize`, which converts the struct using the // `serde` serialization. - pythonize(py, &self).expect("valid action") + pythonize(py, &self) } } @@ -270,13 +274,13 @@ pub enum SimpleJsonValue { } impl<'source> FromPyObject<'source> for SimpleJsonValue { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { if let Ok(s) = ob.downcast::() { Ok(SimpleJsonValue::Str(Cow::Owned(s.to_string()))) // A bool *is* an int, ensure we try bool first. } else if let Ok(b) = ob.downcast::() { Ok(SimpleJsonValue::Bool(b.extract()?)) - } else if let Ok(i) = ob.downcast::() { + } else if let Ok(i) = ob.downcast::() { Ok(SimpleJsonValue::Int(i.extract()?)) } else if ob.is_none() { Ok(SimpleJsonValue::Null) @@ -298,15 +302,19 @@ pub enum JsonValue { } impl<'source> FromPyObject<'source> for JsonValue { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { if let Ok(l) = ob.downcast::() { - match l.iter().map(SimpleJsonValue::extract).collect() { + match l + .iter() + .map(|it| SimpleJsonValue::extract_bound(&it)) + .collect() + { Ok(a) => Ok(JsonValue::Array(a)), Err(e) => Err(PyTypeError::new_err(format!( "Can't convert to JsonValue::Array: {e}" ))), } - } else if let Ok(v) = SimpleJsonValue::extract(ob) { + } else if let Ok(v) = SimpleJsonValue::extract_bound(ob) { Ok(JsonValue::Value(v)) } else { Err(PyTypeError::new_err(format!( @@ -363,15 +371,19 @@ pub enum KnownCondition { }, } -impl IntoPy for Condition { - fn into_py(self, py: Python<'_>) -> PyObject { - pythonize(py, &self).expect("valid condition") +impl<'source> IntoPyObject<'source> for Condition { + type Target = PyAny; + type Output = Bound<'source, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'source>) -> Result { + pythonize(py, &self) } } impl<'source> FromPyObject<'source> for Condition { fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { - Ok(depythonize_bound(ob.clone())?) + Ok(depythonize(ob)?) } } diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index 55df0203f3..23de668102 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -29,7 +29,7 @@ use pyo3::{ exceptions::PyValueError, pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + Bound, IntoPyObject, Py, PyAny, PyObject, PyResult, Python, }; use ulid::Ulid; @@ -37,6 +37,7 @@ use self::session::Session; use crate::{ errors::{NotFoundError, SynapseError}, http::{http_request_from_twisted, http_response_to_twisted, HeaderMapPyExt}, + UnwrapInfallible, }; mod session; @@ -125,7 +126,11 @@ impl RendezvousHandler { let base = Uri::try_from(format!("{base}_synapse/client/rendezvous")) .map_err(|_| PyValueError::new_err("Invalid base URI"))?; - let clock = homeserver.call_method0("get_clock")?.to_object(py); + let clock = homeserver + .call_method0("get_clock")? + .into_pyobject(py) + .unwrap_infallible() + .unbind(); // Construct a Python object so that we can get a reference to the // evict method and schedule it to run. @@ -318,7 +323,7 @@ impl RendezvousHandler { } pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "rendezvous")?; + let child_module = PyModule::new(py, "rendezvous")?; child_module.add_class::()?; @@ -326,7 +331,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> // We need to manually add the module to sys.modules to make `from // synapse.synapse_rust import rendezvous` work. - py.import_bound("sys")? + py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.rendezvous", child_module)?; From 59ad4b18fc81f373c57a7c7d09a2f340a129de76 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Nov 2024 13:31:43 +0000 Subject: [PATCH 010/691] Update setuptools-rust and fix building abi3 wheels (#17969) Newer versions of `setuptools-rust` ignore the `py_limited_api` flag to `RustExtension`, and instead read it from `bdist_wheel` config. c.f. https://github.com/PyO3/setuptools-rust/blob/main/CHANGELOG.md#190-2024-02-24 --- build_rust.py | 19 +++++++++++++++++++ changelog.d/17969.misc | 1 + poetry.lock | 11 +++++------ 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 changelog.d/17969.misc diff --git a/build_rust.py b/build_rust.py index 662474dcb4..d2726cee26 100644 --- a/build_rust.py +++ b/build_rust.py @@ -1,8 +1,10 @@ # A build script for poetry that adds the rust extension. +import itertools import os from typing import Any, Dict +from packaging.specifiers import SpecifierSet from setuptools_rust import Binding, RustExtension @@ -14,6 +16,8 @@ def build(setup_kwargs: Dict[str, Any]) -> None: target="synapse.synapse_rust", path=cargo_toml_path, binding=Binding.PyO3, + # This flag is a no-op in the latest versions. Instead, we need to + # specify this in the `bdist_wheel` config below. py_limited_api=True, # We force always building in release mode, as we can't tell the # difference between using `poetry` in development vs production. @@ -21,3 +25,18 @@ def build(setup_kwargs: Dict[str, Any]) -> None: ) setup_kwargs.setdefault("rust_extensions", []).append(extension) setup_kwargs["zip_safe"] = False + + # We lookup the minimum supported python version by looking at + # `python_requires` (e.g. ">=3.9.0,<4.0.0") and finding the first python + # version that matches. We then convert that into the `py_limited_api` form, + # e.g. cp39 for python 3.9. + py_limited_api: str + python_bounds = SpecifierSet(setup_kwargs["python_requires"]) + for minor_version in itertools.count(start=8): + if f"3.{minor_version}.0" in python_bounds: + py_limited_api = f"cp3{minor_version}" + break + + setup_kwargs.setdefault("options", {}).setdefault("bdist_wheel", {})[ + "py_limited_api" + ] = py_limited_api diff --git a/changelog.d/17969.misc b/changelog.d/17969.misc new file mode 100644 index 0000000000..05506daaa0 --- /dev/null +++ b/changelog.d/17969.misc @@ -0,0 +1 @@ +Update setuptools-rust and fix building abi3 wheels in latest version. diff --git a/poetry.lock b/poetry.lock index 14bb9ad172..f43fe2489a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2405,19 +2405,18 @@ test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata [[package]] name = "setuptools-rust" -version = "1.8.1" +version = "1.10.2" description = "Setuptools Rust extension plugin" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-rust-1.8.1.tar.gz", hash = "sha256:94b1dd5d5308b3138d5b933c3a2b55e6d6927d1a22632e509fcea9ddd0f7e486"}, - {file = "setuptools_rust-1.8.1-py3-none-any.whl", hash = "sha256:b5324493949ccd6aa0c03890c5f6b5f02de4512e3ac1697d02e9a6c02b18aa8e"}, + {file = "setuptools_rust-1.10.2-py3-none-any.whl", hash = "sha256:4b39c435ae9670315d522ed08fa0e8cb29f2a6048033966b6be2571a90ce4f1c"}, + {file = "setuptools_rust-1.10.2.tar.gz", hash = "sha256:5d73e7eee5f87a6417285b617c97088a7c20d1a70fcea60e3bdc94ff567c29dc"}, ] [package.dependencies] semantic-version = ">=2.8.2,<3" setuptools = ">=62.4" -tomli = {version = ">=1.2.1", markers = "python_version < \"3.11\""} [[package]] name = "signedjson" @@ -2528,7 +2527,7 @@ files = [ name = "tornado" version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, From d80cd57c54427687afcb48740d99219c88a0fff1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 28 Nov 2024 18:06:19 +0000 Subject: [PATCH 011/691] Fix new scheduled tasks jumping the queue (#17962) Currently, when a new scheduled task is added and its scheduled time has already passed, we set it to ACTIVE. This is problematic, because it means it will jump the queue ahead of all other SCHEDULED tasks; furthermore, if the Synapse process gets restarted, it will jump ahead of any ACTIVE tasks which have been started but are taking a while to run. Instead, we leave it set to SCHEDULED, but kick off a call to `_launch_scheduled_tasks`, which will decide if we actually have capacity to start a new task, and start the newly-added task if so. --- changelog.d/17962.misc | 1 + synapse/replication/tcp/commands.py | 2 +- synapse/replication/tcp/handler.py | 2 +- synapse/util/task_scheduler.py | 63 ++++++++++++++--------------- tests/util/test_task_scheduler.py | 53 ++++++++++++++---------- 5 files changed, 65 insertions(+), 56 deletions(-) create mode 100644 changelog.d/17962.misc diff --git a/changelog.d/17962.misc b/changelog.d/17962.misc new file mode 100644 index 0000000000..adf6348707 --- /dev/null +++ b/changelog.d/17962.misc @@ -0,0 +1 @@ +Fix new scheduled tasks jumping the queue. diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 7d51441e91..6ab5356660 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -495,7 +495,7 @@ class LockReleasedCommand(Command): class NewActiveTaskCommand(_SimpleCommand): - """Sent to inform instance handling background tasks that a new active task is available to run. + """Sent to inform instance handling background tasks that a new task is ready to run. Format:: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 6101226938..1fafbb48c3 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -727,7 +727,7 @@ class ReplicationCommandHandler: ) -> None: """Called when get a new NEW_ACTIVE_TASK command.""" if self._task_scheduler: - self._task_scheduler.launch_task_by_id(cmd.data) + self._task_scheduler.on_new_task(cmd.data) def new_connection(self, connection: IReplicationConnection) -> None: """Called when we have a new connection.""" diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 448960b297..3ed457bd30 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -174,9 +174,10 @@ class TaskScheduler: The id of the scheduled task """ status = TaskStatus.SCHEDULED + start_now = False if timestamp is None or timestamp < self._clock.time_msec(): timestamp = self._clock.time_msec() - status = TaskStatus.ACTIVE + start_now = True task = ScheduledTask( random_string(16), @@ -190,9 +191,11 @@ class TaskScheduler: ) await self._store.insert_scheduled_task(task) - if status == TaskStatus.ACTIVE: + # If the task is ready to run immediately, run the scheduling algorithm now + # rather than waiting + if start_now: if self._run_background_tasks: - await self._launch_task(task) + self._launch_scheduled_tasks() else: self._hs.get_replication_command_handler().send_new_active_task(task.id) @@ -300,23 +303,13 @@ class TaskScheduler: raise Exception(f"Task {id} is currently ACTIVE and can't be deleted") await self._store.delete_scheduled_task(id) - def launch_task_by_id(self, id: str) -> None: - """Try launching the task with the given ID.""" - # Don't bother trying to launch new tasks if we're already at capacity. - if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: - return + def on_new_task(self, task_id: str) -> None: + """Handle a notification that a new ready-to-run task has been added to the queue""" + # Just run the scheduler + self._launch_scheduled_tasks() - run_as_background_process("launch_task_by_id", self._launch_task_by_id, id) - - async def _launch_task_by_id(self, id: str) -> None: - """Helper async function for `launch_task_by_id`.""" - task = await self.get_task(id) - if task: - await self._launch_task(task) - - @wrap_as_background_process("launch_scheduled_tasks") - async def _launch_scheduled_tasks(self) -> None: - """Retrieve and launch scheduled tasks that should be running at that time.""" + def _launch_scheduled_tasks(self) -> None: + """Retrieve and launch scheduled tasks that should be running at this time.""" # Don't bother trying to launch new tasks if we're already at capacity. if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: return @@ -326,20 +319,26 @@ class TaskScheduler: self._launching_new_tasks = True - try: - for task in await self.get_tasks( - statuses=[TaskStatus.ACTIVE], limit=self.MAX_CONCURRENT_RUNNING_TASKS - ): - await self._launch_task(task) - for task in await self.get_tasks( - statuses=[TaskStatus.SCHEDULED], - max_timestamp=self._clock.time_msec(), - limit=self.MAX_CONCURRENT_RUNNING_TASKS, - ): - await self._launch_task(task) + async def inner() -> None: + try: + for task in await self.get_tasks( + statuses=[TaskStatus.ACTIVE], + limit=self.MAX_CONCURRENT_RUNNING_TASKS, + ): + # _launch_task will ignore tasks that we're already running, and + # will also do nothing if we're already at the maximum capacity. + await self._launch_task(task) + for task in await self.get_tasks( + statuses=[TaskStatus.SCHEDULED], + max_timestamp=self._clock.time_msec(), + limit=self.MAX_CONCURRENT_RUNNING_TASKS, + ): + await self._launch_task(task) - finally: - self._launching_new_tasks = False + finally: + self._launching_new_tasks = False + + run_as_background_process("launch_scheduled_tasks", inner) @wrap_as_background_process("clean_scheduled_tasks") async def _clean_scheduled_tasks(self) -> None: diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index 30f0510c9f..9e403b948b 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -18,8 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # - -from typing import Optional, Tuple +from typing import List, Optional, Tuple from twisted.internet.task import deferLater from twisted.test.proto_helpers import MemoryReactor @@ -104,33 +103,43 @@ class TestTaskScheduler(HomeserverTestCase): ) ) - # This is to give the time to the active tasks to finish - self.reactor.advance(1) - - # Check that only MAX_CONCURRENT_RUNNING_TASKS tasks has run and that one - # is still scheduled. - tasks = [ - self.get_success(self.task_scheduler.get_task(task_id)) - for task_id in task_ids - ] + def get_tasks_of_status(status: TaskStatus) -> List[ScheduledTask]: + tasks = ( + self.get_success(self.task_scheduler.get_task(task_id)) + for task_id in task_ids + ) + return [t for t in tasks if t is not None and t.status == status] + # At this point, there should be MAX_CONCURRENT_RUNNING_TASKS active tasks and + # one scheduled task. self.assertEquals( - len( - [t for t in tasks if t is not None and t.status == TaskStatus.COMPLETE] - ), + len(get_tasks_of_status(TaskStatus.ACTIVE)), TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, ) + self.assertEquals( + len(get_tasks_of_status(TaskStatus.SCHEDULED)), + 1, + ) - scheduled_tasks = [ - t for t in tasks if t is not None and t.status == TaskStatus.ACTIVE - ] - self.assertEquals(len(scheduled_tasks), 1) - - # We need to wait for the next run of the scheduler loop - self.reactor.advance((TaskScheduler.SCHEDULE_INTERVAL_MS / 1000)) + # Give the time to the active tasks to finish self.reactor.advance(1) - # Check that the last task has been properly executed after the next scheduler loop run + # Check that MAX_CONCURRENT_RUNNING_TASKS tasks have run and that one + # is still scheduled. + self.assertEquals( + len(get_tasks_of_status(TaskStatus.COMPLETE)), + TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, + ) + scheduled_tasks = get_tasks_of_status(TaskStatus.SCHEDULED) + self.assertEquals(len(scheduled_tasks), 1) + + # The scheduled task should start 0.1s after the first of the active tasks + # finishes + self.reactor.advance(0.1) + self.assertEquals(len(get_tasks_of_status(TaskStatus.ACTIVE)), 1) + + # ... and should finally complete after another second + self.reactor.advance(1) prev_scheduled_task = self.get_success( self.task_scheduler.get_task(scheduled_tasks[0].id) ) From 6a909aade2eef99c2cd18a6f4c0e923b199600d6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 29 Nov 2024 11:26:37 -0600 Subject: [PATCH 012/691] Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})` (#17972) Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})` Spawning from https://github.com/element-hq/sbg/pull/421#discussion_r1859497330 where we have a proxy that intercepts responses to `/_matrix/client/v3/login/sso/redirect(/{idpId})` in order to upgrade them to use OAuth 2.0 Pushed Authorization Requests (PAR). Instead of needing to intercept multiple endpoints that redirect to the authorization endpoint, it seems better to just have Synapse consolidate to a single flow. ### Testing strategy 1. Create a new OAuth application. I'll be using GitHub for example but there are [many options](https://github.com/matrix-org/synapse/blob/be65a8ec0195955c15fdb179c9158b187638e39a/docs/openid.md). Visit https://github.com/settings/developers -> **New OAuth App** - Application name: `Synapse local testing` - Homepage URL: `http://localhost:8008` - Authorization callback URL: `http://localhost:8008/_synapse/client/oidc/callback` 1. Update your Synapse `homeserver.yaml` ```yaml server_name: "my.synapse.server" public_baseurl: http://localhost:8008/ listeners: - port: 8008 bind_addresses: [ #'::1', '127.0.0.1' ] tls: false type: http x_forwarded: true resources: - names: [client, federation, metrics] compress: false # SSO login testing oidc_providers: - idp_id: github idp_name: Github idp_brand: "github" # optional: styling hint for clients discover: false issuer: "https://github.com/" client_id: "xxx" # TO BE FILLED client_secret: "xxx" # TO BE FILLED authorization_endpoint: "https://github.com/login/oauth/authorize" token_endpoint: "https://github.com/login/oauth/access_token" userinfo_endpoint: "https://api.github.com/user" scopes: ["read:user"] user_mapping_provider: config: subject_claim: "id" localpart_template: "{{ user.login }}" display_name_template: "{{ user.name }}" ``` 1. Start Synapse: `poetry run synapse_homeserver --config-path homeserver.yaml` 1. Visit `http://localhost:8008/_synapse/client/pick_idp?redirectUrl=http%3A%2F%2Fexample.com` 1. Choose GitHub 1. Notice that you're redirected to GitHub to sign in (`https://github.com/login/oauth/authorize?...`) Tested locally and works: 1. `http://localhost:8008/_synapse/client/pick_idp?idp=oidc-github&redirectUrl=http%3A//example.com` -> 1. `http://localhost:8008/_matrix/client/v3/login/sso/redirect/oidc-github?redirectUrl=http://example.com` -> 1. `https://github.com/login/oauth/authorize?response_type=code&client_id=xxx&redirect_uri=http%3A%2F%2Flocalhost%3A8008%2F_synapse%2Fclient%2Foidc%2Fcallback&scope=read%3Auser&state=xxx&nonce=xxx` --- changelog.d/17972.misc | 1 + synapse/api/urls.py | 42 ++++++- synapse/config/cas.py | 6 +- synapse/config/server.py | 6 + synapse/rest/synapse/client/pick_idp.py | 29 ++--- tests/api/test_urls.py | 55 +++++++++ tests/rest/client/test_login.py | 144 ++++++++++++++++++++++-- tests/rest/client/utils.py | 7 +- 8 files changed, 262 insertions(+), 28 deletions(-) create mode 100644 changelog.d/17972.misc create mode 100644 tests/api/test_urls.py diff --git a/changelog.d/17972.misc b/changelog.d/17972.misc new file mode 100644 index 0000000000..e7f009d20d --- /dev/null +++ b/changelog.d/17972.misc @@ -0,0 +1 @@ +Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 03a3e96f28..655b5edd7a 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -23,7 +23,8 @@ import hmac from hashlib import sha256 -from urllib.parse import urlencode +from typing import Optional +from urllib.parse import urlencode, urljoin from synapse.config import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -66,3 +67,42 @@ class ConsentURIBuilder: urlencode({"u": user_id, "h": mac}), ) return consent_uri + + +class LoginSSORedirectURIBuilder: + def __init__(self, hs_config: HomeServerConfig): + self._public_baseurl = hs_config.server.public_baseurl + + def build_login_sso_redirect_uri( + self, *, idp_id: Optional[str], client_redirect_url: str + ) -> str: + """Build a `/login/sso/redirect` URI for the given identity provider. + + Builds `/_matrix/client/v3/login/sso/redirect/{idpId}?redirectUrl=xxx` when `idp_id` is specified. + Otherwise, builds `/_matrix/client/v3/login/sso/redirect?redirectUrl=xxx` when `idp_id` is `None`. + + Args: + idp_id: Optional ID of the identity provider + client_redirect_url: URL to redirect the user to after login + + Returns + The URI to follow when choosing a specific identity provider. + """ + base_url = urljoin( + self._public_baseurl, + f"{CLIENT_API_PREFIX}/v3/login/sso/redirect", + ) + + serialized_query_parameters = urlencode({"redirectUrl": client_redirect_url}) + + if idp_id: + resultant_url = urljoin( + # We have to add a trailing slash to the base URL to ensure that the + # last path segment is not stripped away when joining with another path. + f"{base_url}/", + f"{idp_id}?{serialized_query_parameters}", + ) + else: + resultant_url = f"{base_url}?{serialized_query_parameters}" + + return resultant_url diff --git a/synapse/config/cas.py b/synapse/config/cas.py index fa59c350c1..c32bf36951 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -20,7 +20,7 @@ # # -from typing import Any, List +from typing import Any, List, Optional from synapse.config.sso import SsoAttributeRequirement from synapse.types import JsonDict @@ -46,7 +46,9 @@ class CasConfig(Config): # TODO Update this to a _synapse URL. public_baseurl = self.root.server.public_baseurl - self.cas_service_url = public_baseurl + "_matrix/client/r0/login/cas/ticket" + self.cas_service_url: Optional[str] = ( + public_baseurl + "_matrix/client/r0/login/cas/ticket" + ) self.cas_protocol_version = cas_config.get("protocol_version") if ( diff --git a/synapse/config/server.py b/synapse/config/server.py index ad7331de42..6b29983617 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -332,8 +332,14 @@ class ServerConfig(Config): logger.info("Using default public_baseurl %s", public_baseurl) else: self.serve_client_wellknown = True + # Ensure that public_baseurl ends with a trailing slash if public_baseurl[-1] != "/": public_baseurl += "/" + + # Scrutinize user-provided config + if not isinstance(public_baseurl, str): + raise ConfigError("Must be a string", ("public_baseurl",)) + self.public_baseurl = public_baseurl # check that public_baseurl is valid diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py index f26929bd60..5e599f85b0 100644 --- a/synapse/rest/synapse/client/pick_idp.py +++ b/synapse/rest/synapse/client/pick_idp.py @@ -21,6 +21,7 @@ import logging from typing import TYPE_CHECKING +from synapse.api.urls import LoginSSORedirectURIBuilder from synapse.http.server import ( DirectServeHtmlResource, finish_request, @@ -49,6 +50,8 @@ class PickIdpResource(DirectServeHtmlResource): hs.config.sso.sso_login_idp_picker_template ) self._server_name = hs.hostname + self._public_baseurl = hs.config.server.public_baseurl + self._login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) async def _async_render_GET(self, request: SynapseRequest) -> None: client_redirect_url = parse_string( @@ -56,25 +59,23 @@ class PickIdpResource(DirectServeHtmlResource): ) idp = parse_string(request, "idp", required=False) - # if we need to pick an IdP, do so + # If we need to pick an IdP, do so if not idp: return await self._serve_id_picker(request, client_redirect_url) - # otherwise, redirect to the IdP's redirect URI - providers = self._sso_handler.get_identity_providers() - auth_provider = providers.get(idp) - if not auth_provider: - logger.info("Unknown idp %r", idp) - self._sso_handler.render_error( - request, "unknown_idp", "Unknown identity provider ID" + # Otherwise, redirect to the login SSO redirect endpoint for the given IdP + # (which will in turn take us to the the IdP's redirect URI). + # + # We could go directly to the IdP's redirect URI, but this way we ensure that + # the user goes through the same logic as normal flow. Additionally, if a proxy + # needs to intercept the request, it only needs to intercept the one endpoint. + sso_login_redirect_url = ( + self._login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id=idp, client_redirect_url=client_redirect_url ) - return - - sso_url = await auth_provider.handle_redirect_request( - request, client_redirect_url.encode("utf8") ) - logger.info("Redirecting to %s", sso_url) - request.redirect(sso_url) + logger.info("Redirecting to %s", sso_login_redirect_url) + request.redirect(sso_login_redirect_url) finish_request(request) async def _serve_id_picker( diff --git a/tests/api/test_urls.py b/tests/api/test_urls.py new file mode 100644 index 0000000000..ce156a05dc --- /dev/null +++ b/tests/api/test_urls.py @@ -0,0 +1,55 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.urls import LoginSSORedirectURIBuilder +from synapse.server import HomeServer +from synapse.util import Clock + +from tests.unittest import HomeserverTestCase + +# a (valid) url with some annoying characters in. %3D is =, %26 is &, %2B is + +TRICKY_TEST_CLIENT_REDIRECT_URL = 'https://x?&q"+%3D%2B"="fö%26=o"' + + +class LoginSSORedirectURIBuilderTestCase(HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) + + def test_no_idp_id(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id=None, client_redirect_url="http://example.com/redirect" + ), + "https://test/_matrix/client/v3/login/sso/redirect?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) + + def test_explicit_idp_id(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc-github", client_redirect_url="http://example.com/redirect" + ), + "https://test/_matrix/client/v3/login/sso/redirect/oidc-github?redirectUrl=http%3A%2F%2Fexample.com%2Fredirect", + ) + + def test_tricky_redirect_uri(self) -> None: + self.assertEqual( + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc-github", + client_redirect_url=TRICKY_TEST_CLIENT_REDIRECT_URL, + ), + "https://test/_matrix/client/v3/login/sso/redirect/oidc-github?redirectUrl=https%3A%2F%2Fx%3F%3Cab+c%3E%26q%22%2B%253D%252B%22%3D%22f%C3%B6%2526%3Do%22", + ) diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index cbd6d8d4bf..1451fd7c29 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -43,6 +43,7 @@ from twisted.web.resource import Resource import synapse.rest.admin from synapse.api.constants import ApprovalNoticeMedium, LoginType from synapse.api.errors import Codes +from synapse.api.urls import LoginSSORedirectURIBuilder from synapse.appservice import ApplicationService from synapse.http.client import RawHeaders from synapse.module_api import ModuleApi @@ -69,6 +70,10 @@ try: except ImportError: HAS_JWT = False +import logging + +logger = logging.getLogger(__name__) + # synapse server name: used to populate public_baseurl in some tests SYNAPSE_SERVER_PUBLIC_HOSTNAME = "synapse" @@ -77,7 +82,7 @@ SYNAPSE_SERVER_PUBLIC_HOSTNAME = "synapse" # FakeChannel.isSecure() returns False, so synapse will see the requested uri as # http://..., so using http in the public_baseurl stops Synapse trying to redirect to # https://.... -BASE_URL = "http://%s/" % (SYNAPSE_SERVER_PUBLIC_HOSTNAME,) +PUBLIC_BASEURL = "http://%s/" % (SYNAPSE_SERVER_PUBLIC_HOSTNAME,) # CAS server used in some tests CAS_SERVER = "https://fake.test" @@ -109,6 +114,23 @@ ADDITIONAL_LOGIN_FLOWS = [ ] +def get_relative_uri_from_absolute_uri(absolute_uri: str) -> str: + """ + Peels off the path and query string from an absolute URI. Useful when interacting + with `make_request(...)` util function which expects a relative path instead of a + full URI. + """ + parsed_uri = urllib.parse.urlparse(absolute_uri) + # Sanity check that we're working with an absolute URI + assert parsed_uri.scheme == "http" or parsed_uri.scheme == "https" + + relative_uri = parsed_uri.path + if parsed_uri.query: + relative_uri += "?" + parsed_uri.query + + return relative_uri + + class TestSpamChecker: def __init__(self, config: None, api: ModuleApi): api.register_spam_checker_callbacks( @@ -614,7 +636,7 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): def default_config(self) -> Dict[str, Any]: config = super().default_config() - config["public_baseurl"] = BASE_URL + config["public_baseurl"] = PUBLIC_BASEURL config["cas_config"] = { "enabled": True, @@ -653,6 +675,9 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): ] return config + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.login_sso_redirect_url_builder = LoginSSORedirectURIBuilder(hs.config) + def create_resource_dict(self) -> Dict[str, Resource]: d = super().create_resource_dict() d.update(build_synapse_client_resource_tree(self.hs)) @@ -725,6 +750,32 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + "&idp=cas", shorthand=False, ) + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="cas", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers @@ -750,6 +801,32 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL) + "&idp=saml", ) + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="saml", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers @@ -773,13 +850,38 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): # pick the default OIDC provider channel = self.make_request( "GET", - "/_synapse/client/pick_idp?redirectUrl=" - + urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL) - + "&idp=oidc", + f"/_synapse/client/pick_idp?redirectUrl={urllib.parse.quote_plus(TEST_CLIENT_REDIRECT_URL)}&idp=oidc", ) self.assertEqual(channel.code, 302, channel.result) location_headers = channel.headers.getRawHeaders("Location") assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="oidc", client_redirect_url=TEST_CLIENT_REDIRECT_URL + ), + ) + + with fake_oidc_server.patch_homeserver(hs=self.hs): + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers oidc_uri = location_headers[0] oidc_uri_path, oidc_uri_query = oidc_uri.split("?", 1) @@ -838,12 +940,38 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): self.assertEqual(chan.json_body["user_id"], "@user1:test") def test_multi_sso_redirect_to_unknown(self) -> None: - """An unknown IdP should cause a 400""" + """An unknown IdP should cause a 404""" channel = self.make_request( "GET", "/_synapse/client/pick_idp?redirectUrl=http://x&idp=xyz", ) - self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.code, 302, channel.result) + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + sso_login_redirect_uri = location_headers[0] + + # it should redirect us to the standard login SSO redirect flow + self.assertEqual( + sso_login_redirect_uri, + self.login_sso_redirect_url_builder.build_login_sso_redirect_uri( + idp_id="xyz", client_redirect_url="http://x" + ), + ) + + # follow the redirect + channel = self.make_request( + "GET", + # We have to make this relative to be compatible with `make_request(...)` + get_relative_uri_from_absolute_uri(sso_login_redirect_uri), + # We have to set the Host header to match the `public_baseurl` to avoid + # the extra redirect in the `SsoRedirectServlet` in order for the + # cookies to be visible. + custom_headers=[ + ("Host", SYNAPSE_SERVER_PUBLIC_HOSTNAME), + ], + ) + + self.assertEqual(channel.code, 404, channel.result) def test_client_idp_redirect_to_unknown(self) -> None: """If the client tries to pick an unknown IdP, return a 404""" @@ -1473,7 +1601,7 @@ class UsernamePickerTestCase(HomeserverTestCase): def default_config(self) -> Dict[str, Any]: config = super().default_config() - config["public_baseurl"] = BASE_URL + config["public_baseurl"] = PUBLIC_BASEURL config["oidc_config"] = {} config["oidc_config"].update(TEST_OIDC_CONFIG) diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index a1c284726a..dbd6049f9f 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -889,7 +889,7 @@ class RestHelper: "GET", uri, ) - assert channel.code == 302 + assert channel.code == 302, f"Expected 302 for {uri}, got {channel.code}" # hit the redirect url again with the right Host header, which should now issue # a cookie and redirect to the SSO provider. @@ -901,17 +901,18 @@ class RestHelper: location = get_location(channel) parts = urllib.parse.urlsplit(location) + next_uri = urllib.parse.urlunsplit(("", "") + parts[2:]) channel = make_request( self.reactor, self.site, "GET", - urllib.parse.urlunsplit(("", "") + parts[2:]), + next_uri, custom_headers=[ ("Host", parts[1]), ], ) - assert channel.code == 302 + assert channel.code == 302, f"Expected 302 for {next_uri}, got {channel.code}" channel.extract_cookies(cookies) return get_location(channel) From a82f5f206f7aeb18a48b7eb1aa6205d66d56e25b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Dec 2024 10:54:14 +0000 Subject: [PATCH 013/691] Fix release process to not create duplicate releases (#17970) This is to work around https://github.com/softprops/action-gh-release/issues/445 --------- Co-authored-by: Quentin Gliech --- .github/workflows/release-artifacts.yml | 3 ++- changelog.d/17970.bugfix | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17970.bugfix diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c0aff79141..42a374fa19 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -212,7 +212,8 @@ jobs: mv debs*/* debs/ tar -cvJf debs.tar.xz debs - name: Attach to release - uses: softprops/action-gh-release@v2 + # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 + uses: softprops/action-gh-release@v2.0.5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/changelog.d/17970.bugfix b/changelog.d/17970.bugfix new file mode 100644 index 0000000000..835079de3f --- /dev/null +++ b/changelog.d/17970.bugfix @@ -0,0 +1 @@ +Fix release process to not create duplicate releases. From a89b697209018cfb8792d22651b646cd154f1c71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:28:08 +0000 Subject: [PATCH 014/691] Bump pysaml2 from 7.3.1 to 7.5.0 (#17978) --- poetry.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index f43fe2489a..1a735e2fd6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1917,13 +1917,13 @@ test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] [[package]] name = "pysaml2" -version = "7.3.1" +version = "7.5.0" description = "Python implementation of SAML Version 2 Standard" optional = true -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.9,<4.0" files = [ - {file = "pysaml2-7.3.1-py3-none-any.whl", hash = "sha256:2cc66e7a371d3f5ff9601f0ed93b5276cca816fce82bb38447d5a0651f2f5193"}, - {file = "pysaml2-7.3.1.tar.gz", hash = "sha256:eab22d187c6dd7707c58b5bb1688f9b8e816427667fc99d77f54399e15cd0a0a"}, + {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, + {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, ] [package.dependencies] @@ -1933,7 +1933,7 @@ pyopenssl = "*" python-dateutil = "*" pytz = "*" requests = ">=2,<3" -xmlschema = ">=1.2.1" +xmlschema = ">=2,<3" [package.extras] s2repoze = ["paste", "repoze.who", "zope.interface"] From 9b2ae62d20a7862c58e1302fdfba03773caf2d83 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 2 Dec 2024 08:28:47 -0700 Subject: [PATCH 015/691] Use stable error code for account locking (#17965) --- changelog.d/17965.feature | 1 + synapse/api/errors.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/17965.feature diff --git a/changelog.d/17965.feature b/changelog.d/17965.feature new file mode 100644 index 0000000000..e447a58986 --- /dev/null +++ b/changelog.d/17965.feature @@ -0,0 +1 @@ +Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). \ No newline at end of file diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e6efa7a424..71e4bb4971 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -87,8 +87,7 @@ class Codes(str, Enum): WEAK_PASSWORD = "M_WEAK_PASSWORD" INVALID_SIGNATURE = "M_INVALID_SIGNATURE" USER_DEACTIVATED = "M_USER_DEACTIVATED" - # USER_LOCKED = "M_USER_LOCKED" - USER_LOCKED = "ORG_MATRIX_MSC3939_USER_LOCKED" + USER_LOCKED = "M_USER_LOCKED" NOT_YET_UPLOADED = "M_NOT_YET_UPLOADED" CANNOT_OVERWRITE_MEDIA = "M_CANNOT_OVERWRITE_MEDIA" From e5d3bfba30351f4f9c2bcf89a2b002c6be9ee099 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 2 Dec 2024 10:17:55 -0600 Subject: [PATCH 016/691] Sliding Sync: Include invite, ban, kick, targets when `$LAZY`-loading room members (#17947) Part of https://github.com/element-hq/synapse/issues/17929 --- changelog.d/17947.feature | 1 + synapse/api/constants.py | 2 + synapse/handlers/sliding_sync/__init__.py | 12 +- synapse/types/handlers/sliding_sync.py | 11 +- .../sliding_sync/test_rooms_required_state.py | 166 +++++++++++++++++- 5 files changed, 181 insertions(+), 11 deletions(-) create mode 100644 changelog.d/17947.feature diff --git a/changelog.d/17947.feature b/changelog.d/17947.feature new file mode 100644 index 0000000000..2d1b99cec2 --- /dev/null +++ b/changelog.d/17947.feature @@ -0,0 +1 @@ +Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 8db302b3d8..1206d1e00f 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -231,6 +231,8 @@ class EventContentFields: ROOM_NAME: Final = "name" MEMBERSHIP: Final = "membership" + MEMBERSHIP_DISPLAYNAME: Final = "displayname" + MEMBERSHIP_AVATAR_URL: Final = "avatar_url" # Used in m.room.guest_access events. GUEST_ACCESS: Final = "guest_access" diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 85cfbc6dbf..4f4faef524 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -955,15 +955,21 @@ class SlidingSyncHandler: and state_key == StateValues.LAZY ): lazy_load_room_members = True + # Everyone in the timeline is relevant - # - # FIXME: We probably also care about invite, ban, kick, targets, etc - # but the spec only mentions "senders". timeline_membership: Set[str] = set() if timeline_events is not None: for timeline_event in timeline_events: + # Anyone who sent a message is relevant timeline_membership.add(timeline_event.sender) + # We also care about invite, ban, kick, targets, + # etc. + if timeline_event.type == EventTypes.Member: + timeline_membership.add( + timeline_event.state_key + ) + # Update the required state filter so we pick up the new # membership for user_id in timeline_membership: diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index aae60fddea..3ebd334a6d 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -407,8 +407,8 @@ class StateValues: # Include all state events of the given type WILDCARD: Final = "*" # Lazy-load room membership events (include room membership events for any event - # `sender` in the timeline). We only give special meaning to this value when it's a - # `state_key`. + # `sender` or membership change target in the timeline). We only give special + # meaning to this value when it's a `state_key`. LAZY: Final = "$LAZY" # Subsitute with the requester's user ID. Typically used by clients to get # the user's membership. @@ -641,9 +641,10 @@ class RoomSyncConfig: if user_id == StateValues.ME: continue # We're lazy-loading membership so we can just return the state we have. - # Lazy-loading means we include membership for any event `sender` in the - # timeline but since we had to auth those timeline events, we will have the - # membership state for them (including from remote senders). + # Lazy-loading means we include membership for any event `sender` or + # membership change target in the timeline but since we had to auth those + # timeline events, we will have the membership state for them (including + # from remote senders). elif user_id == StateValues.LAZY: continue elif user_id == StateValues.WILDCARD: diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index ecea5f2d5b..b4869d5fa3 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -11,6 +11,7 @@ # See the GNU Affero General Public License for more details: # . # +import enum import logging from parameterized import parameterized, parameterized_class @@ -18,9 +19,9 @@ from parameterized import parameterized, parameterized_class from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventContentFields, EventTypes, JoinRules, Membership from synapse.handlers.sliding_sync import StateValues -from synapse.rest.client import login, room, sync +from synapse.rest.client import knock, login, room, sync from synapse.server import HomeServer from synapse.util import Clock @@ -30,6 +31,17 @@ from tests.test_utils.event_injection import mark_event_as_partial_state logger = logging.getLogger(__name__) +# Inherit from `str` so that they show up in the test description when we +# `@parameterized.expand(...)` the first parameter +class MembershipAction(str, enum.Enum): + INVITE = "invite" + JOIN = "join" + KNOCK = "knock" + LEAVE = "leave" + BAN = "ban" + KICK = "kick" + + # FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the # foreground update for # `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by @@ -52,6 +64,7 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, + knock.register_servlets, room.register_servlets, sync.register_servlets, ] @@ -496,6 +509,153 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): ) self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + @parameterized.expand( + [ + (MembershipAction.LEAVE,), + (MembershipAction.INVITE,), + (MembershipAction.KNOCK,), + (MembershipAction.JOIN,), + (MembershipAction.BAN,), + (MembershipAction.KICK,), + ] + ) + def test_rooms_required_state_changed_membership_in_timeline_lazy_loading_room_members_incremental_sync( + self, + room_membership_action: str, + ) -> None: + """ + On incremental sync, test `rooms.required_state` returns people relevant to the + timeline when lazy-loading room members, `["m.room.member","$LAZY"]` **including + changes to membership**. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + user3_tok = self.login(user3_id, "pass") + user4_id = self.register_user("user4", "pass") + user4_tok = self.login(user4_id, "pass") + user5_id = self.register_user("user5", "pass") + user5_tok = self.login(user5_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # If we're testing knocks, set the room to knock + if room_membership_action == MembershipAction.KNOCK: + self.helper.send_state( + room_id1, + EventTypes.JoinRules, + {"join_rule": JoinRules.KNOCK}, + tok=user2_tok, + ) + + # Join the test users to the room + self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.invite(room_id1, src=user2_id, targ=user3_id, tok=user2_tok) + self.helper.join(room_id1, user3_id, tok=user3_tok) + self.helper.invite(room_id1, src=user2_id, targ=user4_id, tok=user2_tok) + self.helper.join(room_id1, user4_id, tok=user4_tok) + if room_membership_action in ( + MembershipAction.LEAVE, + MembershipAction.BAN, + MembershipAction.JOIN, + ): + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + self.helper.join(room_id1, user5_id, tok=user5_tok) + + # Send some messages to fill up the space + self.helper.send(room_id1, "1", tok=user2_tok) + self.helper.send(room_id1, "2", tok=user2_tok) + self.helper.send(room_id1, "3", tok=user2_tok) + + # Make the Sliding Sync request with lazy loading for the room members + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 3, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Send more timeline events into the room + self.helper.send(room_id1, "4", tok=user2_tok) + self.helper.send(room_id1, "5", tok=user4_tok) + # The third event will be our membership event concerning user5 + if room_membership_action == MembershipAction.LEAVE: + # User 5 leaves + self.helper.leave(room_id1, user5_id, tok=user5_tok) + elif room_membership_action == MembershipAction.INVITE: + # User 5 is invited + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.KNOCK: + # User 5 knocks + self.helper.knock(room_id1, user5_id, tok=user5_tok) + # The admin of the room accepts the knock + self.helper.invite(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.JOIN: + # Update the display name of user5 (causing a membership change) + self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user5_id, + body={ + EventContentFields.MEMBERSHIP: Membership.JOIN, + EventContentFields.MEMBERSHIP_DISPLAYNAME: "quick changer", + }, + tok=user5_tok, + ) + elif room_membership_action == MembershipAction.BAN: + self.helper.ban(room_id1, src=user2_id, targ=user5_id, tok=user2_tok) + elif room_membership_action == MembershipAction.KICK: + # Kick user5 from the room + self.helper.change_membership( + room=room_id1, + src=user2_id, + targ=user5_id, + tok=user2_tok, + membership=Membership.LEAVE, + extra_data={ + "reason": "Bad manners", + }, + ) + else: + raise AssertionError( + f"Unknown room_membership_action: {room_membership_action}" + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # Only user2, user4, and user5 sent events in the last 3 events we see in the + # `timeline`. + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + # This appears because *some* membership in the room changed and the + # heroes are recalculated and is thrown in because we have it. But this + # is technically optional and not needed because we've already seen user2 + # in the last sync (and their membership hasn't changed). + state_map[(EventTypes.Member, user2_id)], + # Appears because there is a message in the timeline from this user + state_map[(EventTypes.Member, user4_id)], + # Appears because there is a membership event in the timeline from this user + state_map[(EventTypes.Member, user5_id)], + }, + exact=True, + ) + self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + def test_rooms_required_state_expand_lazy_loading_room_members_incremental_sync( self, ) -> None: @@ -1243,7 +1403,7 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): # Update the room name self.helper.send_state( - room_id1, "m.room.name", {"name": "Bar"}, state_key="", tok=user1_tok + room_id1, EventTypes.Name, {"name": "Bar"}, state_key="", tok=user1_tok ) # Update the sliding sync requests to exclude the room name again From 190c400a8379b99b7e5889d3cb2fb102995f6ab4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:55:40 +0000 Subject: [PATCH 017/691] Bump tomli from 2.1.0 to 2.2.1 (#17979) --- poetry.lock | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1a735e2fd6..af5d9a4406 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2514,13 +2514,43 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.1.0" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, - {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] From d648c8ce3f4cbf61191b9f5302e405f7b0288677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:55:53 +0000 Subject: [PATCH 018/691] Bump bytes from 1.8.0 to 1.9.0 (#17982) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7084165ee..e74e7226f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,9 +61,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cfg-if" From f3fd6852ac3dd8196511b97772285e06c301da31 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Dec 2024 10:54:14 +0000 Subject: [PATCH 019/691] Fix release process to not create duplicate releases (#17970) This is to work around https://github.com/softprops/action-gh-release/issues/445 --------- Co-authored-by: Quentin Gliech --- .github/workflows/release-artifacts.yml | 3 ++- changelog.d/17970.bugfix | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17970.bugfix diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c0aff79141..42a374fa19 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -212,7 +212,8 @@ jobs: mv debs*/* debs/ tar -cvJf debs.tar.xz debs - name: Attach to release - uses: softprops/action-gh-release@v2 + # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 + uses: softprops/action-gh-release@v2.0.5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/changelog.d/17970.bugfix b/changelog.d/17970.bugfix new file mode 100644 index 0000000000..835079de3f --- /dev/null +++ b/changelog.d/17970.bugfix @@ -0,0 +1 @@ +Fix release process to not create duplicate releases. From 4daa533e82f345ce87b9495d31781af570ba3ead Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 24 Oct 2024 14:54:09 -0500 Subject: [PATCH 020/691] Sliding Sync: Fix state leaking on incremental sync --- synapse/handlers/sliding_sync/__init__.py | 63 ++++++- .../storage/databases/main/state_deltas.py | 7 + .../sliding_sync/test_rooms_required_state.py | 160 +++++++++++++++++- 3 files changed, 226 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 85cfbc6dbf..0893017126 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -39,6 +39,7 @@ from synapse.logging.opentracing import ( trace, ) from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary +from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.databases.main.stream import PaginateFunction from synapse.storage.roommember import ( MemberSummary, @@ -48,6 +49,7 @@ from synapse.types import ( MutableStateMap, PersistedEventPosition, Requester, + RoomStreamToken, SlidingSyncStreamToken, StateMap, StrCollection, @@ -470,6 +472,64 @@ class SlidingSyncHandler: return state_map + @trace + async def get_current_state_deltas_for_room( + self, + room_id: str, + room_membership_for_user_at_to_token: RoomsForUserType, + from_token: RoomStreamToken, + to_token: RoomStreamToken, + ) -> List[StateDelta]: + """ + Get the state deltas between two tokens taking into account the user's + membership. If the user is LEAVE/BAN, we will only get the state deltas up to + their LEAVE/BAN event (inclusive). + + (> `from_token` and <= `to_token`) + """ + membership = room_membership_for_user_at_to_token.membership + # We don't know how to handle `membership` values other than these. The + # code below would need to be updated. + assert membership in ( + Membership.JOIN, + Membership.INVITE, + Membership.KNOCK, + Membership.LEAVE, + Membership.BAN, + ) + + # People shouldn't see past their leave/ban event + if membership in ( + Membership.LEAVE, + Membership.BAN, + ): + to_bound = ( + room_membership_for_user_at_to_token.event_pos.to_room_stream_token() + ) + # If we are participating in the room, we can get the latest current state in + # the room + elif membership == Membership.JOIN: + to_bound = to_token + # We can only rely on the stripped state included in the invite/knock event + # itself so there will never be any state deltas to send down. + elif membership in (Membership.INVITE, Membership.KNOCK): + return [] + else: + # We don't know how to handle this type of membership yet + # + # FIXME: We should use `assert_never` here but for some reason + # the exhaustive matching doesn't recognize the `Never` here. + # assert_never(membership) + raise AssertionError( + f"Unexpected membership {membership} that we don't know how to handle yet" + ) + + return await self.store.get_current_state_deltas_for_room( + room_id=room_id, + from_token=from_token, + to_token=to_bound, + ) + @trace async def get_room_sync_data( self, @@ -790,8 +850,9 @@ class SlidingSyncHandler: # TODO: Limit the number of state events we're about to send down # the room, if its too many we should change this to an # `initial=True`? - deltas = await self.store.get_current_state_deltas_for_room( + deltas = await self.get_current_state_deltas_for_room( room_id=room_id, + room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, from_token=from_bound, to_token=to_token.room_key, ) diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index 117ee89d0a..b90f667da8 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -243,6 +243,13 @@ class StateDeltasStore(SQLBaseStore): (> `from_token` and <= `to_token`) """ + # We can bail early if the `from_token` is after the `to_token` + if ( + to_token is not None + and from_token is not None + and to_token.is_before_or_eq(from_token) + ): + return [] if ( from_token is not None diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index ecea5f2d5b..be13f94538 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -751,9 +751,10 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) @parameterized.expand([(Membership.LEAVE,), (Membership.BAN,)]) - def test_rooms_required_state_leave_ban(self, stop_membership: str) -> None: + def test_rooms_required_state_leave_ban_initial(self, stop_membership: str) -> None: """ - Test `rooms.required_state` should not return state past a leave/ban event. + Test `rooms.required_state` should not return state past a leave/ban event when + it's the first "initial" time the room is being sent down the connection. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -788,6 +789,13 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): body={"foo": "bar"}, tok=user2_tok, ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "bar"}, + tok=user2_tok, + ) if stop_membership == Membership.LEAVE: # User 1 leaves @@ -796,6 +804,8 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): # User 1 is banned self.helper.ban(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + # Get the state_map before we change the state as this is the final state we + # expect User1 to be able to see state_map = self.get_success( self.storage_controllers.state.get_current_state(room_id1) ) @@ -808,12 +818,36 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): body={"foo": "qux"}, tok=user2_tok, ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "qux"}, + tok=user2_tok, + ) self.helper.leave(room_id1, user3_id, tok=user3_tok) # Make an incremental Sliding Sync request + # + # Also expand the required state to include the `org.matrix.bar_state` event. + # This is just an extra complication of the test. + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ["org.matrix.bar_state", ""], + ], + "timeline_limit": 3, + } + } + } response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) - # Only user2 and user3 sent events in the 3 events we see in the `timeline` + # We should only see the state up to the leave/ban event self._assertRequiredStateIncludes( response_body["rooms"][room_id1]["required_state"], { @@ -822,6 +856,126 @@ class SlidingSyncRoomsRequiredStateTestCase(SlidingSyncBase): state_map[(EventTypes.Member, user2_id)], state_map[(EventTypes.Member, user3_id)], state_map[("org.matrix.foo_state", "")], + state_map[("org.matrix.bar_state", "")], + }, + exact=True, + ) + self.assertIsNone(response_body["rooms"][room_id1].get("invite_state")) + + @parameterized.expand([(Membership.LEAVE,), (Membership.BAN,)]) + def test_rooms_required_state_leave_ban_incremental( + self, stop_membership: str + ) -> None: + """ + Test `rooms.required_state` should not return state past a leave/ban event on + incremental sync. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + user3_tok = self.login(user3_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user3_id, tok=user3_tok) + + self.helper.send_state( + room_id1, + event_type="org.matrix.foo_state", + state_key="", + body={"foo": "bar"}, + tok=user2_tok, + ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "bar"}, + tok=user2_tok, + ) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ], + "timeline_limit": 3, + } + } + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + if stop_membership == Membership.LEAVE: + # User 1 leaves + self.helper.leave(room_id1, user1_id, tok=user1_tok) + elif stop_membership == Membership.BAN: + # User 1 is banned + self.helper.ban(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + + # Get the state_map before we change the state as this is the final state we + # expect User1 to be able to see + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # Change the state after user 1 leaves + self.helper.send_state( + room_id1, + event_type="org.matrix.foo_state", + state_key="", + body={"foo": "qux"}, + tok=user2_tok, + ) + self.helper.send_state( + room_id1, + event_type="org.matrix.bar_state", + state_key="", + body={"bar": "qux"}, + tok=user2_tok, + ) + self.helper.leave(room_id1, user3_id, tok=user3_tok) + + # Make an incremental Sliding Sync request + # + # Also expand the required state to include the `org.matrix.bar_state` event. + # This is just an extra complication of the test. + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Create, ""], + [EventTypes.Member, "*"], + ["org.matrix.foo_state", ""], + ["org.matrix.bar_state", ""], + ], + "timeline_limit": 3, + } + } + } + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + # User1 should only see the state up to the leave/ban event + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + # User1 should see their leave/ban membership + state_map[(EventTypes.Member, user1_id)], + state_map[("org.matrix.bar_state", "")], + # The commented out state events were already returned in the initial + # sync so we shouldn't see them again on the incremental sync. And we + # shouldn't see the state events that changed after the leave/ban event. + # + # state_map[(EventTypes.Create, "")], + # state_map[(EventTypes.Member, user2_id)], + # state_map[(EventTypes.Member, user3_id)], + # state_map[("org.matrix.foo_state", "")], }, exact=True, ) From d82e1ed357b7ee21dff83d06cba7a67840cfd464 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Nov 2024 15:53:33 +0000 Subject: [PATCH 021/691] Handle null invite and knock room state --- .../federation/transport/server/federation.py | 3 +++ synapse/handlers/federation.py | 3 +++ synapse/handlers/sliding_sync/__init__.py | 16 +++++++++++----- synapse/push/push_tools.py | 8 ++++++-- synapse/rest/client/sync.py | 12 ++++++++++-- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index a05e5d5319..093ba30d31 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -509,6 +509,9 @@ class FederationV2InviteServlet(BaseFederationServerServlet): event = content["event"] invite_room_state = content.get("invite_room_state", []) + if not isinstance(invite_room_state, list): + invite_room_state = [] + # Synapse expects invite_room_state to be in unsigned, as it is in v1 # API diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2b7aad5b58..17dd4af13e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -880,6 +880,9 @@ class FederationHandler: if stripped_room_state is None: raise KeyError("Missing 'knock_room_state' field in send_knock response") + if not isinstance(stripped_room_state, list): + raise TypeError("'knock_room_state' has wrong type") + event.unsigned["knock_room_state"] = stripped_room_state context = EventContext.for_outlier(self._storage_controllers) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 0893017126..0175da1a13 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -815,13 +815,19 @@ class SlidingSyncHandler: stripped_state = [] if invite_or_knock_event.membership == Membership.INVITE: - stripped_state.extend( - invite_or_knock_event.unsigned.get("invite_room_state", []) + invite_state = invite_or_knock_event.unsigned.get( + "invite_room_state", [] ) + if not isinstance(invite_state, list): + invite_state = [] + + stripped_state.extend(invite_state) elif invite_or_knock_event.membership == Membership.KNOCK: - stripped_state.extend( - invite_or_knock_event.unsigned.get("knock_room_state", []) - ) + knock_state = invite_or_knock_event.unsigned.get("knock_room_state", []) + if not isinstance(knock_state, list): + knock_state = [] + + stripped_state.extend(knock_state) stripped_state.append(strip_event(invite_or_knock_event)) diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index 1ef881f702..3f3e4a9234 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -74,9 +74,13 @@ async def get_context_for_event( room_state = [] if ev.content.get("membership") == Membership.INVITE: - room_state = ev.unsigned.get("invite_room_state", []) + invite_room_state = ev.unsigned.get("invite_room_state", []) + if isinstance(invite_room_state, list): + room_state = invite_room_state elif ev.content.get("membership") == Membership.KNOCK: - room_state = ev.unsigned.get("knock_room_state", []) + knock_room_state = ev.unsigned.get("knock_room_state", []) + if isinstance(knock_room_state, list): + room_state = knock_room_state # Ideally we'd reuse the logic in `calculate_room_name`, but that gets # complicated to handle partial events vs pulling events from the DB. diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 5c62a74f41..f4ef84a038 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -436,7 +436,12 @@ class SyncRestServlet(RestServlet): ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned - invited_state = list(unsigned.pop("invite_room_state", [])) + + invited_state = unsigned.pop("invite_room_state", []) + if not isinstance(invited_state, list): + invited_state = [] + + invited_state = list(invited_state) invited_state.append(invite) invited[room.room_id] = {"invite_state": {"events": invited_state}} @@ -476,7 +481,10 @@ class SyncRestServlet(RestServlet): # Extract the stripped room state from the unsigned dict # This is for clients to get a little bit of information about # the room they've knocked on, without revealing any sensitive information - knocked_state = list(unsigned.pop("knock_room_state", [])) + knocked_state = unsigned.pop("knock_room_state", []) + if not isinstance(knocked_state, list): + knocked_state = [] + knocked_state = list(knocked_state) # Append the actual knock membership event itself as well. This provides # the client with: From 4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 5 Nov 2024 15:05:22 -0700 Subject: [PATCH 022/691] Don't allow unsupported content-type Co-authored-by: Eric Eastwood --- synapse/http/site.py | 36 ++++++++++++++++++++++++++++ tests/http/test_site.py | 53 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/synapse/http/site.py b/synapse/http/site.py index 1cd90cb9b7..e83a4447b2 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -21,6 +21,7 @@ import contextlib import logging import time +from http import HTTPStatus from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union import attr @@ -139,6 +140,41 @@ class SynapseRequest(Request): self.synapse_site.site_tag, ) + # Twisted machinery: this method is called by the Channel once the full request has + # been received, to dispatch the request to a resource. + # + # We're patching Twisted to bail/abort early when we see someone trying to upload + # `multipart/form-data` so we can avoid Twisted parsing the entire request body into + # in-memory (specific problem of this specific `Content-Type`). This protects us + # from an attacker uploading something bigger than the available RAM and crashing + # the server with a `MemoryError`, or carefully block just enough resources to cause + # all other requests to fail. + # + # FIXME: This can be removed once we Twisted releases a fix and we update to a + # version that is patched + def requestReceived(self, command: bytes, path: bytes, version: bytes) -> None: + if command == b"POST": + ctype = self.requestHeaders.getRawHeaders(b"content-type") + if ctype and b"multipart/form-data" in ctype[0]: + self.method, self.uri = command, path + self.clientproto = version + self.code = HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value + self.code_message = bytes( + HTTPStatus.UNSUPPORTED_MEDIA_TYPE.phrase, "ascii" + ) + self.responseHeaders.setRawHeaders(b"content-length", [b"0"]) + + logger.warning( + "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", + self.client, + command, + path, + ) + self.write(b"") + self.loseConnection() + return + return super().requestReceived(command, path, version) + def handleContentChunk(self, data: bytes) -> None: # we should have a `content` by now. assert self.content, "handleContentChunk() called before gotLength()" diff --git a/tests/http/test_site.py b/tests/http/test_site.py index bfa26a329c..fc620c705a 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -90,3 +90,56 @@ class SynapseRequestTestCase(HomeserverTestCase): # default max upload size is 50M, so it should drop on the next buffer after # that. self.assertEqual(sent, 50 * 1024 * 1024 + 1024) + + def test_content_type_multipart(self) -> None: + """HTTP POST requests with `content-type: multipart/form-data` should be rejected""" + self.hs.start_listening() + + # find the HTTP server which is configured to listen on port 0 + (port, factory, _backlog, interface) = self.reactor.tcpServers[0] + self.assertEqual(interface, "::") + self.assertEqual(port, 0) + + # as a control case, first send a regular request. + + # complete the connection and wire it up to a fake transport + client_address = IPv6Address("TCP", "::1", 2345) + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"0\r\n" + b"\r\n" + ) + + while not transport.disconnecting: + self.reactor.advance(1) + + # we should get a 404 + self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 404 ") + + # now send request with content-type header + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Transfer-Encoding: chunked\r\n" + b"Content-Type: multipart/form-data\r\n" + b"\r\n" + b"0\r\n" + b"\r\n" + ) + + while not transport.disconnecting: + self.reactor.advance(1) + + # we should get a 415 + self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 415 ") From b64a4e5fbbbf119b6c65aedf0d999b4237d55503 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 22 Nov 2024 15:30:29 +0000 Subject: [PATCH 023/691] Restrict which image formats we will decode in order to generate thumbnails --- synapse/media/thumbnailer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 3845067835..d6b8ce4a09 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -67,6 +67,11 @@ class ThumbnailError(Exception): class Thumbnailer: FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + # Which image formats we allow Pillow to open. + # This should intentionally be kept restrictive, because the decoder of any + # format in this list becomes part of our trusted computing base. + PILLOW_FORMATS = ("jpeg", "png", "webp", "gif") + @staticmethod def set_limits(max_image_pixels: int) -> None: Image.MAX_IMAGE_PIXELS = max_image_pixels @@ -76,7 +81,7 @@ class Thumbnailer: self._closed = False try: - self.image = Image.open(input_path) + self.image = Image.open(input_path, formats=self.PILLOW_FORMATS) except OSError as e: # If an error occurs opening the image, a thumbnail won't be able to # be generated. From fe3d88b833f76742874642c119b14b788341c905 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 3 Dec 2024 11:15:00 +0100 Subject: [PATCH 024/691] 1.120.1 --- CHANGES.md | 42 ++++++++++++++++++++++++++++++++++++++++ changelog.d/17970.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 49 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/17970.bugfix diff --git a/CHANGES.md b/CHANGES.md index 0caac3f89e..bac2b4c210 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,45 @@ +# Synapse 1.120.1 (2024-12-03) + +This patch release fixes multiple security vulnerabilities, some affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. + +Administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. + +### Security advisory + +The following issues are fixed in 1.120.1. + +- [GHSA-rfq8-j7rh-8hf2](https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2) / [CVE-2024-52805](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52805): **Unsupported content types can lead to memory exhaustion** + + Synapse instances which have a high `max_upload_size` and which don't have a reverse proxy in front of them that would otherwise limit upload size are affected. + + Fixed by [4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf](https://github.com/element-hq/synapse/commit/4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf). + +- [GHSA-f3r3-h2mq-hx2h](https://github.com/element-hq/synapse/security/advisories/GHSA-f3r3-h2mq-hx2h) / [CVE-2024-52815](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52815): **Malicious invites via federation can break a user's sync** + + Fixed by [d82e1ed357b7ee21dff83d06cba7a67840cfd464](https://github.com/element-hq/synapse/commit/d82e1ed357b7ee21dff83d06cba7a67840cfd464). + +- [GHSA-vp6v-whfm-rv3g](https://github.com/element-hq/synapse/security/advisories/GHSA-vp6v-whfm-rv3g) / [CVE-2024-53863](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53863): **Synapse can be forced to thumbnail unexpected file formats, invoking potentially untrustworthy decoders** + + Synapse instances can disable dynamic thumbnailing by setting `dynamic_thumbnails` to `false` in the configuration file. + + Fixed by [b64a4e5fbbbf119b6c65aedf0d999b4237d55503](https://github.com/element-hq/synapse/commit/b64a4e5fbbbf119b6c65aedf0d999b4237d55503). + +- [GHSA-56w4-5538-8v8h](https://github.com/element-hq/synapse/security/advisories/GHSA-56w4-5538-8v8h) / [CVE-2024-53867](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53867): **The Sliding Sync feature on Synapse versions between 1.113.0rc1 and 1.120.0 can leak partial room state changes to users no longer in a room** + + Non-state events, like messages, are unaffected. + + Synapse instances can disable the Sliding Sync feature by setting `experimental_features.msc3575_enabled` to `false` in the configuration file. + + Fixed by [4daa533e82f345ce87b9495d31781af570ba3ead](https://github.com/element-hq/synapse/commit/4daa533e82f345ce87b9495d31781af570ba3ead). + +See the advisories for more details. If you have any questions, email [security at element.io](mailto:security@element.io). + +### Bugfixes + +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970)) + + + # Synapse 1.120.0 (2024-11-26) ### Bugfixes diff --git a/changelog.d/17970.bugfix b/changelog.d/17970.bugfix deleted file mode 100644 index 835079de3f..0000000000 --- a/changelog.d/17970.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix release process to not create duplicate releases. diff --git a/debian/changelog b/debian/changelog index bd4466d7aa..936447b8b6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.120.1) stable; urgency=medium + + * New synapse release 1.120.1. + + -- Synapse Packaging team Tue, 03 Dec 2024 09:07:57 +0000 + matrix-synapse-py3 (1.120.0) stable; urgency=medium * New synapse release 1.120.0. diff --git a/pyproject.toml b/pyproject.toml index 5fd1d7c198..f838d4d7be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.120.0" +version = "1.120.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From b257c7ab199f8c800254764d2ac5d4a9708ceaa2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 3 Dec 2024 06:54:25 -0600 Subject: [PATCH 025/691] Be able to test `/login/sso/redirect` in Complement (#17986) Be able to test `/login/sso/redirect` in Complement Spawning from https://github.com/element-hq/sbg/pull/421#discussion_r1854926218 where we have a proxy that intercepts responses to `/_matrix/client/v3/login/sso/redirect(/{idpId})` in order to upgrade them to use OAuth 2.0 Pushed Authorization Requests (PAR). We have some Complement tests in that codebase that go over this flow and these changes are required [in order for the URL's to line up](https://github.com/element-hq/synapse/blob/d648c8ce3f4cbf61191b9f5302e405f7b0288677/synapse/rest/client/login.py#L652-L673). --- changelog.d/17986.misc | 1 + docker/complement/conf/workers-shared-extra.yaml.j2 | 1 + docker/conf-workers/nginx.conf.j2 | 2 +- scripts-dev/complement.sh | 4 ++++ synapse/config/logger.py | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17986.misc diff --git a/changelog.d/17986.misc b/changelog.d/17986.misc new file mode 100644 index 0000000000..c062f3ecdf --- /dev/null +++ b/changelog.d/17986.misc @@ -0,0 +1 @@ +Fix Docker and Complement config to be able to use `public_baseurl`. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9a74c617bc..c5228af72d 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -7,6 +7,7 @@ #} ## Server ## +public_baseurl: http://127.0.0.1:8008/ report_stats: False trusted_key_servers: [] enable_registration: true diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2 index d1e02af723..c3f9b584d2 100644 --- a/docker/conf-workers/nginx.conf.j2 +++ b/docker/conf-workers/nginx.conf.j2 @@ -42,6 +42,6 @@ server { {% endif %} proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $host; + proxy_set_header Host $host:$server_port; } } diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index b6dcb96e2c..6be9177f11 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -195,6 +195,10 @@ if [ -z "$skip_docker_build" ]; then # Build the unified Complement image (from the worker Synapse image we just built). echo_if_github "::group::Build Docker image: complement/Dockerfile" $CONTAINER_RUNTIME build -t complement-synapse \ + `# This is the tag we end up pushing to the registry (see` \ + `# .github/workflows/push_complement_image.yml) so let's just label it now` \ + `# so people can reference it by the same name locally.` \ + -t ghcr.io/element-hq/synapse/complement-synapse \ -f "docker/complement/Dockerfile" "docker/complement" echo_if_github "::endgroup::" diff --git a/synapse/config/logger.py b/synapse/config/logger.py index cfc1a57107..e5aca36b75 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -360,5 +360,6 @@ def setup_logging( "Licensed under the AGPL 3.0 license. Website: https://github.com/element-hq/synapse" ) logging.info("Server hostname: %s", config.server.server_name) + logging.info("Public Base URL: %s", config.server.public_baseurl) logging.info("Instance name: %s", hs.get_instance_name()) logging.info("Twisted reactor: %s", type(reactor).__name__) From 650492ed4d95ca8bf9a41eb0456c7e82b4616a22 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 3 Dec 2024 16:39:41 +0100 Subject: [PATCH 026/691] Stop building wheels for macOS --- .github/workflows/release-artifacts.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 42a374fa19..0c0e023c75 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -5,7 +5,7 @@ name: Build release artifacts on: # we build on PRs and develop to (hopefully) get early warning # of things breaking (but only build one set of debs). PRs skip - # building wheels on macOS & ARM. + # building wheels on ARM. pull_request: push: branches: ["develop", "release-*"] @@ -111,7 +111,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13] + os: [ubuntu-22.04] arch: [x86_64, aarch64] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. @@ -119,12 +119,6 @@ jobs: - ${{ startsWith(github.ref, 'refs/pull/') }} exclude: - # Don't build macos wheels on PR CI. - - is_pr: true - os: "macos-13" - # Don't build aarch64 wheels on mac. - - os: "macos-13" - arch: aarch64 # Don't build aarch64 wheels on PR CI. - is_pr: true arch: aarch64 From 6f689d452c5632df558e76bc5a24111e555a3c8a Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 3 Dec 2024 16:58:40 +0100 Subject: [PATCH 027/691] 1.120.2 --- CHANGES.md | 8 ++++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index bac2b4c210..535c41f6a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.120.2 (2024-12-03) + +This version has building of wheels for macOS disabled. +It is functionally identical to 1.120.1, which contains multiple security fixes. +If you are already using 1.120.1, there is no need to upgrade to this version. + + + # Synapse 1.120.1 (2024-12-03) This patch release fixes multiple security vulnerabilities, some affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. diff --git a/debian/changelog b/debian/changelog index 936447b8b6..3aa74d1d24 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.120.2) stable; urgency=medium + + * New synapse release 1.120.2. + + -- Synapse Packaging team Tue, 03 Dec 2024 15:43:37 +0000 + matrix-synapse-py3 (1.120.1) stable; urgency=medium * New synapse release 1.120.1. diff --git a/pyproject.toml b/pyproject.toml index f838d4d7be..a403952d89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.120.1" +version = "1.120.2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From abf44ad3244aa92273dcf79c53babd52363123f6 Mon Sep 17 00:00:00 2001 From: manuroe Date: Tue, 3 Dec 2024 23:58:43 +0100 Subject: [PATCH 028/691] MSC4076: Add disable_badge_count to pusher configuration (#17975) This PR implements [MSC4076: Let E2EE clients calculate app badge counts themselves (disable_badge_count)](https://github.com/matrix-org/matrix-spec-proposals/pull/4076). --- changelog.d/17975.feature | 1 + synapse/config/experimental.py | 3 ++ synapse/push/httppusher.py | 16 +++++-- tests/push/test_http.py | 84 +++++++++++++++++++++++++++++++++- 4 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 changelog.d/17975.feature diff --git a/changelog.d/17975.feature b/changelog.d/17975.feature new file mode 100644 index 0000000000..48f41bddad --- /dev/null +++ b/changelog.d/17975.feature @@ -0,0 +1 @@ +[MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 3411179a2a..57ac27697f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -448,3 +448,6 @@ class ExperimentalConfig(Config): # MSC4222: Adding `state_after` to sync v2 self.msc4222_enabled: bool = experimental.get("msc4222_enabled", False) + + # MSC4076: Add `disable_badge_count`` to pusher configuration + self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index dd9b64d6ef..69790ecab5 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -127,6 +127,11 @@ class HttpPusher(Pusher): if self.data is None: raise PusherConfigException("'data' key can not be null for HTTP pusher") + # Check if badge counts should be disabled for this push gateway + self.disable_badge_count = self.hs.config.experimental.msc4076_enabled and bool( + self.data.get("org.matrix.msc4076.disable_badge_count", False) + ) + self.name = "%s/%s/%s" % ( pusher_config.user_name, pusher_config.app_id, @@ -461,9 +466,10 @@ class HttpPusher(Pusher): content: JsonDict = { "event_id": event.event_id, "room_id": event.room_id, - "counts": {"unread": badge}, "prio": priority, } + if not self.disable_badge_count: + content["counts"] = {"unread": badge} # event_id_only doesn't include the tweaks, so override them. tweaks = {} else: @@ -478,11 +484,11 @@ class HttpPusher(Pusher): "type": event.type, "sender": event.user_id, "prio": priority, - "counts": { - "unread": badge, - # 'missed_calls': 2 - }, } + if not self.disable_badge_count: + content["counts"] = { + "unread": badge, + } if event.type == "m.room.member" and event.is_state(): content["membership"] = event.content["membership"] content["user_is_target"] = event.state_key == self.user_id diff --git a/tests/push/test_http.py b/tests/push/test_http.py index bcca472617..5c235bbe53 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -17,9 +17,11 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any, List, Tuple +from typing import Any, Dict, List, Tuple from unittest.mock import Mock +from parameterized import parameterized + from twisted.internet.defer import Deferred from twisted.test.proto_helpers import MemoryReactor @@ -1085,3 +1087,83 @@ class HTTPPusherTests(HomeserverTestCase): self.pump() self.assertEqual(len(self.push_attempts), 11) + + @parameterized.expand( + [ + # Badge count disabled + (True, True), + (True, False), + # Badge count enabled + (False, True), + (False, False), + ] + ) + @override_config({"experimental_features": {"msc4076_enabled": True}}) + def test_msc4076_badge_count( + self, disable_badge_count: bool, event_id_only: bool + ) -> None: + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Register the pusher with disable_badge_count set to True + user_tuple = self.get_success( + self.hs.get_datastores().main.get_user_by_access_token(access_token) + ) + assert user_tuple is not None + device_id = user_tuple.device_id + + # Set the push data dict based on test input parameters + push_data: Dict[str, Any] = { + "url": "http://example.com/_matrix/push/v1/notify", + } + if disable_badge_count: + push_data["org.matrix.msc4076.disable_badge_count"] = True + if event_id_only: + push_data["format"] = "event_id_only" + + self.get_success( + self.hs.get_pusherpool().add_or_update_pusher( + user_id=user_id, + device_id=device_id, + kind="http", + app_id="m.http", + app_display_name="HTTP Push Notifications", + device_display_name="pushy push", + pushkey="a@example.com", + lang=None, + data=push_data, + ) + ) + + # Create a room + room = self.helper.create_room_as(user_id, tok=access_token) + + # The other user joins + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends a message + self.helper.send(room, body="Hi!", tok=other_access_token) + + # Advance time a bit, so the pusher will register something has happened + self.pump() + + # One push was attempted to be sent + self.assertEqual(len(self.push_attempts), 1) + self.assertEqual( + self.push_attempts[0][1], "http://example.com/_matrix/push/v1/notify" + ) + + if disable_badge_count: + # Verify that the notification DOESN'T contain a counts field + self.assertNotIn("counts", self.push_attempts[0][2]["notification"]) + else: + # Ensure that the notification DOES contain a counts field + self.assertIn("counts", self.push_attempts[0][2]["notification"]) + self.assertEqual( + self.push_attempts[0][2]["notification"]["counts"]["unread"], 1 + ) From 23b626f2e68e985a3218abd0fc7d03b53bbcaf89 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 4 Dec 2024 12:04:49 +0100 Subject: [PATCH 029/691] Support for MSC4190: device management for application services (#17705) This is an implementation of MSC4190, which allows appservices to manage their user's devices without /login & /logout. --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/17705.feature | 1 + synapse/appservice/__init__.py | 2 + synapse/config/appservice.py | 13 ++ synapse/handlers/device.py | 34 +++++ synapse/handlers/register.py | 6 +- synapse/rest/client/devices.py | 62 +++++--- synapse/rest/client/register.py | 7 +- tests/handlers/test_appservice.py | 15 +- tests/handlers/test_oauth_delegation.py | 31 +++- tests/rest/client/test_devices.py | 181 ++++++++++++++++++++++++ tests/rest/client/test_register.py | 28 ++++ tests/unittest.py | 4 +- 12 files changed, 351 insertions(+), 33 deletions(-) create mode 100644 changelog.d/17705.feature diff --git a/changelog.d/17705.feature b/changelog.d/17705.feature new file mode 100644 index 0000000000..e2cd7bca4f --- /dev/null +++ b/changelog.d/17705.feature @@ -0,0 +1 @@ +Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index a96cdbf1e7..6ee5240c4e 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -87,6 +87,7 @@ class ApplicationService: ip_range_whitelist: Optional[IPSet] = None, supports_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, + msc4190_device_management: bool = False, ): self.token = token self.url = ( @@ -100,6 +101,7 @@ class ApplicationService: self.ip_range_whitelist = ip_range_whitelist self.supports_ephemeral = supports_ephemeral self.msc3202_transaction_extensions = msc3202_transaction_extensions + self.msc4190_device_management = msc4190_device_management if "|" in self.id: raise Exception("application service ID cannot contain '|' character") diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 6ff00e1ff8..dda6bcd1b7 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -183,6 +183,18 @@ def _load_appservice( "The `org.matrix.msc3202` option should be true or false if specified." ) + # Opt-in flag for the MSC4190 behaviours. + # When enabled, the following C-S API endpoints change for appservices: + # - POST /register does not return an access token + # - PUT /devices/{device_id} creates a new device if one does not exist + # - DELETE /devices/{device_id} no longer requires UIA + # - POST /delete_devices/{device_id} no longer requires UIA + msc4190_enabled = as_info.get("io.element.msc4190", False) + if not isinstance(msc4190_enabled, bool): + raise ValueError( + "The `io.element.msc4190` option should be true or false if specified." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -195,4 +207,5 @@ def _load_appservice( ip_range_whitelist=ip_range_whitelist, supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, + msc4190_device_management=msc4190_enabled, ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index d88660e273..d9622080b4 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -729,6 +729,40 @@ class DeviceHandler(DeviceWorkerHandler): await self.notify_device_update(user_id, device_ids) + async def upsert_device( + self, user_id: str, device_id: str, display_name: Optional[str] = None + ) -> bool: + """Create or update a device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + display_name: The new display name for this device. + + Returns: + True if the device was created, False if it was updated. + + """ + + # Reject a new displayname which is too long. + self._check_device_name_length(display_name) + + created = await self.store.store_device( + user_id, + device_id, + initial_device_display_name=display_name, + ) + + if not created: + await self.store.update_device( + user_id, + device_id, + new_display_name=display_name, + ) + + await self.notify_device_update(user_id, [device_id]) + return created + async def update_device(self, user_id: str, device_id: str, content: dict) -> None: """Update the given device diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c200e29569..c49db83ce7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -630,7 +630,9 @@ class RegistrationHandler: """ await self._auto_join_rooms(user_id) - async def appservice_register(self, user_localpart: str, as_token: str) -> str: + async def appservice_register( + self, user_localpart: str, as_token: str + ) -> Tuple[str, ApplicationService]: user = UserID(user_localpart, self.hs.hostname) user_id = user.to_string() service = self.store.get_app_service_by_token(as_token) @@ -653,7 +655,7 @@ class RegistrationHandler: appservice_id=service_id, create_profile_with_displayname=user.localpart, ) - return user_id + return (user_id, service) def check_user_id_not_appservice_exclusive( self, user_id: str, allowed_appservice: Optional[ApplicationService] = None diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 6a45a5d130..4607b23494 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -114,15 +114,19 @@ class DeleteDevicesRestServlet(RestServlet): else: raise e - await self.auth_handler.validate_user_via_ui_auth( - requester, - request, - body.dict(exclude_unset=True), - "remove device(s) from your account", - # Users might call this multiple times in a row while cleaning up - # devices, allow a single UI auth session to be re-used. - can_skip_ui_auth=True, - ) + if requester.app_service and requester.app_service.msc4190_device_management: + # MSC4190 can skip UIA for this endpoint + pass + else: + await self.auth_handler.validate_user_via_ui_auth( + requester, + request, + body.dict(exclude_unset=True), + "remove device(s) from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, + ) await self.device_handler.delete_devices( requester.user.to_string(), body.devices @@ -175,9 +179,6 @@ class DeviceRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - if self._msc3861_oauth_delegation_enabled: - raise UnrecognizedRequestError(code=404) - requester = await self.auth.get_user_by_req(request) try: @@ -192,15 +193,24 @@ class DeviceRestServlet(RestServlet): else: raise - await self.auth_handler.validate_user_via_ui_auth( - requester, - request, - body.dict(exclude_unset=True), - "remove a device from your account", - # Users might call this multiple times in a row while cleaning up - # devices, allow a single UI auth session to be re-used. - can_skip_ui_auth=True, - ) + if requester.app_service and requester.app_service.msc4190_device_management: + # MSC4190 allows appservices to delete devices through this endpoint without UIA + # It's also allowed with MSC3861 enabled + pass + + else: + if self._msc3861_oauth_delegation_enabled: + raise UnrecognizedRequestError(code=404) + + await self.auth_handler.validate_user_via_ui_auth( + requester, + request, + body.dict(exclude_unset=True), + "remove a device from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, + ) await self.device_handler.delete_devices( requester.user.to_string(), [device_id] @@ -216,6 +226,16 @@ class DeviceRestServlet(RestServlet): requester = await self.auth.get_user_by_req(request, allow_guest=True) body = parse_and_validate_json_object_from_request(request, self.PutBody) + + # MSC4190 allows appservices to create devices through this endpoint + if requester.app_service and requester.app_service.msc4190_device_management: + created = await self.device_handler.upsert_device( + user_id=requester.user.to_string(), + device_id=device_id, + display_name=body.display_name, + ) + return 201 if created else 200, {} + await self.device_handler.update_device( requester.user.to_string(), device_id, body.dict() ) diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 61e1436841..ad76f188ab 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -771,9 +771,12 @@ class RegisterRestServlet(RestServlet): body: JsonDict, should_issue_refresh_token: bool = False, ) -> JsonDict: - user_id = await self.registration_handler.appservice_register( + user_id, appservice = await self.registration_handler.appservice_register( username, as_token ) + if appservice.msc4190_device_management: + body["inhibit_login"] = True + return await self._create_registration_details( user_id, body, @@ -937,7 +940,7 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): as_token = self.auth.get_access_token_from_request(request) - user_id = await self.registration_handler.appservice_register( + user_id, _ = await self.registration_handler.appservice_register( desired_username, as_token ) return 200, {"user_id": user_id} diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1eec0d43b7..1db630e9e4 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -1165,12 +1165,23 @@ class ApplicationServicesHandlerOtkCountsTestCase(unittest.HomeserverTestCase): self.hs.get_datastores().main.services_cache = [self._service] # Register some appservice users - self._sender_user, self._sender_device = self.register_appservice_user( + user_id, device_id = self.register_appservice_user( "as.sender", self._service_token ) - self._namespaced_user, self._namespaced_device = self.register_appservice_user( + # With MSC4190 enabled, there will not be a device created + # during AS registration. However MSC4190 is not enabled + # in this test. It may become the default behaviour in the + # future, in which case this test will need to be updated. + assert device_id is not None + self._sender_user = user_id + self._sender_device = device_id + + user_id, device_id = self.register_appservice_user( "_as_user1", self._service_token ) + assert device_id is not None + self._namespaced_user = user_id + self._namespaced_device = device_id # Register a real user as well. self._real_user = self.register_user("real.user", "meow") diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 5b5dc713d1..5f73469daa 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -560,9 +560,15 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.assertEqual(channel.code, 401, channel.json_body) def expect_unrecognized( - self, method: str, path: str, content: Union[bytes, str, JsonDict] = "" + self, + method: str, + path: str, + content: Union[bytes, str, JsonDict] = "", + auth: bool = False, ) -> None: - channel = self.make_request(method, path, content) + channel = self.make_request( + method, path, content, access_token="token" if auth else None + ) self.assertEqual(channel.code, 404, channel.json_body) self.assertEqual( @@ -648,8 +654,25 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_device_management_endpoints_removed(self) -> None: """Test that device management endpoints that were removed in MSC2964 are no longer available.""" - self.expect_unrecognized("POST", "/_matrix/client/v3/delete_devices") - self.expect_unrecognized("DELETE", "/_matrix/client/v3/devices/{DEVICE}") + + # Because we still support those endpoints with ASes, it checks the + # access token before returning 404 + self.http_client.request = AsyncMock( + return_value=FakeResponse.json( + code=200, + payload={ + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + }, + ) + ) + + self.expect_unrecognized("POST", "/_matrix/client/v3/delete_devices", auth=True) + self.expect_unrecognized( + "DELETE", "/_matrix/client/v3/devices/{DEVICE}", auth=True + ) def test_openid_endpoints_removed(self) -> None: """Test that OpenID id_token endpoints that were removed in MSC2964 are no longer available.""" diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index a3ed12a38f..dd3abdebac 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -24,6 +24,7 @@ from twisted.internet.defer import ensureDeferred from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import NotFoundError +from synapse.appservice import ApplicationService from synapse.rest import admin, devices, sync from synapse.rest.client import keys, login, register from synapse.server import HomeServer @@ -455,3 +456,183 @@ class DehydratedDeviceTestCase(unittest.HomeserverTestCase): token, ) self.assertEqual(channel.json_body["device_keys"], {"@mikey:test": {}}) + + +class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): + servlets = [ + register.register_servlets, + devices.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.hs = self.setup_test_homeserver() + + # This application service uses the new MSC4190 behaviours + self.msc4190_service = ApplicationService( + id="msc4190", + token="some_token", + hs_token="some_token", + sender="@as:example.com", + namespaces={ + ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] + }, + msc4190_device_management=True, + ) + # This application service doesn't use the new MSC4190 behaviours + self.pre_msc_service = ApplicationService( + id="regular", + token="other_token", + hs_token="other_token", + sender="@as2:example.com", + namespaces={ + ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] + }, + msc4190_device_management=False, + ) + self.hs.get_datastores().main.services_cache.append(self.msc4190_service) + self.hs.get_datastores().main.services_cache.append(self.pre_msc_service) + return self.hs + + def test_PUT_device(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + self.register_appservice_user("bob", self.pre_msc_service.token) + + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={"display_name": "Alice's device"}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + self.assertEqual(channel.json_body["devices"][0]["device_id"], "AABBCCDD") + + # Doing a second time should return a 200 instead of a 201 + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={"display_name": "Alice's device"}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # On the regular service, that API should not allow for the + # creation of new devices. + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@bob:test", + content={"display_name": "Bob's device"}, + access_token=self.pre_msc_service.token, + ) + self.assertEqual(channel.code, 404, channel.json_body) + + def test_DELETE_device(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + + # There should be no device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + # Create a device + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + # There should be one device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + + # Delete the device. UIA should not be required. + channel = self.make_request( + "DELETE", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # There should be no device again + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + def test_POST_delete_devices(self) -> None: + self.register_appservice_user("alice", self.msc4190_service.token) + + # There should be no device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) + + # Create a device + channel = self.make_request( + "PUT", + "/_matrix/client/v3/devices/AABBCCDD?user_id=@alice:test", + content={}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 201, channel.json_body) + + # There should be one device + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(len(channel.json_body["devices"]), 1) + + # Delete the device with delete_devices + # UIA should not be required. + channel = self.make_request( + "POST", + "/_matrix/client/v3/delete_devices?user_id=@alice:test", + content={"devices": ["AABBCCDD"]}, + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # There should be no device again + channel = self.make_request( + "GET", + "/_matrix/client/v3/devices?user_id=@alice:test", + access_token=self.msc4190_service.token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {"devices": []}) diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index c091f403cc..b697bf6f67 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -120,6 +120,34 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 401, msg=channel.result) + def test_POST_appservice_msc4190_enabled(self) -> None: + # With MSC4190 enabled, the registration should *not* return an access token + user_id = "@as_user_kermit:test" + as_token = "i_am_an_app_service" + + appservice = ApplicationService( + as_token, + id="1234", + namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, + sender="@as:test", + msc4190_device_management=True, + ) + + self.hs.get_datastores().main.services_cache.append(appservice) + request_data = { + "username": "as_user_kermit", + "type": APP_SERVICE_REGISTRATION_TYPE, + } + + channel = self.make_request( + b"POST", self.url + b"?access_token=i_am_an_app_service", request_data + ) + + self.assertEqual(channel.code, 200, msg=channel.result) + det_data = {"user_id": user_id, "home_server": self.hs.hostname} + self.assertLessEqual(det_data.items(), channel.json_body.items()) + self.assertNotIn("access_token", channel.json_body) + def test_POST_bad_password(self) -> None: request_data = {"username": "kermit", "password": 666} channel = self.make_request(b"POST", self.url, request_data) diff --git a/tests/unittest.py b/tests/unittest.py index 614e805abd..6a32861a3e 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -781,7 +781,7 @@ class HomeserverTestCase(TestCase): self, username: str, appservice_token: str, - ) -> Tuple[str, str]: + ) -> Tuple[str, Optional[str]]: """Register an appservice user as an application service. Requires the client-facing registration API be registered. @@ -805,7 +805,7 @@ class HomeserverTestCase(TestCase): access_token=appservice_token, ) self.assertEqual(channel.code, 200, channel.json_body) - return channel.json_body["user_id"], channel.json_body["device_id"] + return channel.json_body["user_id"], channel.json_body.get("device_id") def login( self, From 05d58b86ac7062def93dd43e3eb794029c10e9e5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:53:51 +0000 Subject: [PATCH 030/691] Pin softprops/action-gh-release to v0.1.15 (#17995) We are still seeing duplicate releases on v2.0.5, so roll back further. [Other](https://github.com/Poko-Apps/curl-openssl-android/commit/f8a5a60b7c4b196c703d322bb3d11e9495807426#diff-88ab30345d9874c4336fe50b54b083ba5bdd925be961c34060e6a192b56b0433R72) [repositories](https://github.com/Glistix/glistix/commit/55fca4fec74aa114faf553b563ae5883b5d76be0#diff-e426ed45842837026e10e66af23d9c7077e89eacbe6958ce7cb991130ad05adaR105) seem to have settled on this version. Addresses https://github.com/element-hq/synapse/issues/17991 We're just going to test this during 1.121.0rc1. --- .github/workflows/release-artifacts.yml | 2 +- changelog.d/17995.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17995.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 0c0e023c75..deb5ec33e3 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -207,7 +207,7 @@ jobs: tar -cvJf debs.tar.xz debs - name: Attach to release # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 - uses: softprops/action-gh-release@v2.0.5 + uses: softprops/action-gh-release@v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/changelog.d/17995.misc b/changelog.d/17995.misc new file mode 100644 index 0000000000..d8e608c1ae --- /dev/null +++ b/changelog.d/17995.misc @@ -0,0 +1 @@ +Pin `softprops/action-gh-release` to v0.1.15 to work around https://github.com/softprops/action-gh-release/issues/445. \ No newline at end of file From 45ca6392f42c1cf609186c93131e18d22b04dab7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:58:26 +0000 Subject: [PATCH 031/691] Pin Rust to 1.82.0 when building Python wheels (#17993) Addresses step 1 of #17988. --- .github/workflows/release-artifacts.yml | 10 ++++++++-- changelog.d/17993.misc | 1 + pyproject.toml | 7 +++++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 changelog.d/17993.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index deb5ec33e3..5b5bfc1896 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -5,7 +5,7 @@ name: Build release artifacts on: # we build on PRs and develop to (hopefully) get early warning # of things breaking (but only build one set of debs). PRs skip - # building wheels on ARM. + # building wheels on macOS & ARM. pull_request: push: branches: ["develop", "release-*"] @@ -111,7 +111,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-22.04, macos-13] arch: [x86_64, aarch64] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. @@ -119,6 +119,12 @@ jobs: - ${{ startsWith(github.ref, 'refs/pull/') }} exclude: + # Don't build macos wheels on PR CI. + - is_pr: true + os: "macos-13" + # Don't build aarch64 wheels on mac. + - os: "macos-13" + arch: aarch64 # Don't build aarch64 wheels on PR CI. - is_pr: true arch: aarch64 diff --git a/changelog.d/17993.misc b/changelog.d/17993.misc new file mode 100644 index 0000000000..149cb358c3 --- /dev/null +++ b/changelog.d/17993.misc @@ -0,0 +1 @@ +Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6fd6a95219..60c62015b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -386,8 +386,11 @@ build-backend = "poetry.core.masonry.api" # c.f. https://github.com/matrix-org/synapse/pull/14259 skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" -# We need a rust compiler -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal" +# We need a rust compiler. +# +# We temporarily pin Rust to 1.82.0 to work around +# https://github.com/element-hq/synapse/issues/17988 +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.82.0 -y --profile minimal" environment= { PATH = "$PATH:$HOME/.cargo/bin" } # For some reason if we don't manually clean the build directory we From a00d0b3d0e72cd56733c30b1b52b5402c92f81cc Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 4 Dec 2024 14:49:28 +0000 Subject: [PATCH 032/691] 1.121.0rc1 --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++ changelog.d/17253.misc | 1 - changelog.d/17705.feature | 1 - changelog.d/17872.doc | 1 - changelog.d/17933.bugfix | 1 - changelog.d/17936.misc | 1 - changelog.d/17944.misc | 1 - changelog.d/17945.misc | 1 - changelog.d/17947.feature | 1 - changelog.d/17952.misc | 1 - changelog.d/17953.doc | 1 - changelog.d/17962.misc | 1 - changelog.d/17965.feature | 1 - changelog.d/17966.misc | 1 - changelog.d/17969.misc | 1 - changelog.d/17970.bugfix | 1 - changelog.d/17972.misc | 1 - changelog.d/17975.feature | 1 - changelog.d/17986.misc | 1 - changelog.d/17993.misc | 1 - changelog.d/17995.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- 23 files changed, 54 insertions(+), 21 deletions(-) delete mode 100644 changelog.d/17253.misc delete mode 100644 changelog.d/17705.feature delete mode 100644 changelog.d/17872.doc delete mode 100644 changelog.d/17933.bugfix delete mode 100644 changelog.d/17936.misc delete mode 100644 changelog.d/17944.misc delete mode 100644 changelog.d/17945.misc delete mode 100644 changelog.d/17947.feature delete mode 100644 changelog.d/17952.misc delete mode 100644 changelog.d/17953.doc delete mode 100644 changelog.d/17962.misc delete mode 100644 changelog.d/17965.feature delete mode 100644 changelog.d/17966.misc delete mode 100644 changelog.d/17969.misc delete mode 100644 changelog.d/17970.bugfix delete mode 100644 changelog.d/17972.misc delete mode 100644 changelog.d/17975.feature delete mode 100644 changelog.d/17986.misc delete mode 100644 changelog.d/17993.misc delete mode 100644 changelog.d/17995.misc diff --git a/CHANGES.md b/CHANGES.md index 535c41f6a6..20177bf00a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,50 @@ +# Synapse 1.121.0rc1 (2024-12-04) + +This release candidate contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). + +New changes listed below. + +### Features + +- Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. ([\#17705](https://github.com/element-hq/synapse/issues/17705)) +- Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. ([\#17947](https://github.com/element-hq/synapse/issues/17947)) +- Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). ([\#17965](https://github.com/element-hq/synapse/issues/17965)) +- [MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. ([\#17975](https://github.com/element-hq/synapse/issues/17975)) + +### Bugfixes + +- Fix long-standing bug where read receipts could get overly delayed being sent over federation. ([\#17933](https://github.com/element-hq/synapse/issues/17933)) + +### Improved Documentation + +- Add OIDC example configuration for Forgejo (fork of Gitea). ([\#17872](https://github.com/element-hq/synapse/issues/17872)) +- Link to element-docker-demo from contrib/docker*. ([\#17953](https://github.com/element-hq/synapse/issues/17953)) + +### Internal Changes + +- [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. ([\#17253](https://github.com/element-hq/synapse/issues/17253)) +- Fix incorrect comment in new schema delta. ([\#17936](https://github.com/element-hq/synapse/issues/17936)) +- Raise setuptools_rust version cap to 1.10.2. ([\#17944](https://github.com/element-hq/synapse/issues/17944)) +- Enable encrypted appservice related experimental features in the complement docker image. ([\#17945](https://github.com/element-hq/synapse/issues/17945)) +- Return whether the user is suspended when querying the user account in the Admin API. ([\#17952](https://github.com/element-hq/synapse/issues/17952)) +- Fix new scheduled tasks jumping the queue. ([\#17962](https://github.com/element-hq/synapse/issues/17962)) +- Bump pyo3 and dependencies to v0.23.2. ([\#17966](https://github.com/element-hq/synapse/issues/17966)) +- Update setuptools-rust and fix building abi3 wheels in latest version. ([\#17969](https://github.com/element-hq/synapse/issues/17969)) +- Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. ([\#17972](https://github.com/element-hq/synapse/issues/17972)) +- Fix Docker and Complement config to be able to use `public_baseurl`. ([\#17986](https://github.com/element-hq/synapse/issues/17986)) +- Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. ([\#17993](https://github.com/element-hq/synapse/issues/17993)) +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970), [\#17995](https://github.com/element-hq/synapse/issues/17995)) + + +### Updates to locked dependencies + +* Bump bytes from 1.8.0 to 1.9.0. ([\#17982](https://github.com/element-hq/synapse/issues/17982)) +* Bump pysaml2 from 7.3.1 to 7.5.0. ([\#17978](https://github.com/element-hq/synapse/issues/17978)) +* Bump serde_json from 1.0.132 to 1.0.133. ([\#17939](https://github.com/element-hq/synapse/issues/17939)) +* Bump tomli from 2.0.2 to 2.1.0. ([\#17959](https://github.com/element-hq/synapse/issues/17959)) +* Bump tomli from 2.1.0 to 2.2.1. ([\#17979](https://github.com/element-hq/synapse/issues/17979)) +* Bump tornado from 6.4.1 to 6.4.2. ([\#17955](https://github.com/element-hq/synapse/issues/17955)) + # Synapse 1.120.2 (2024-12-03) This version has building of wheels for macOS disabled. diff --git a/changelog.d/17253.misc b/changelog.d/17253.misc deleted file mode 100644 index 868691624d..0000000000 --- a/changelog.d/17253.misc +++ /dev/null @@ -1 +0,0 @@ -[MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. diff --git a/changelog.d/17705.feature b/changelog.d/17705.feature deleted file mode 100644 index e2cd7bca4f..0000000000 --- a/changelog.d/17705.feature +++ /dev/null @@ -1 +0,0 @@ -Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. diff --git a/changelog.d/17872.doc b/changelog.d/17872.doc deleted file mode 100644 index 7f8b2d3495..0000000000 --- a/changelog.d/17872.doc +++ /dev/null @@ -1 +0,0 @@ -Add OIDC example configuration for Forgejo (fork of Gitea). diff --git a/changelog.d/17933.bugfix b/changelog.d/17933.bugfix deleted file mode 100644 index 8d30ac587e..0000000000 --- a/changelog.d/17933.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix long-standing bug where read receipts could get overly delayed being sent over federation. diff --git a/changelog.d/17936.misc b/changelog.d/17936.misc deleted file mode 100644 index 91d976fbd9..0000000000 --- a/changelog.d/17936.misc +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect comment in new schema delta. diff --git a/changelog.d/17944.misc b/changelog.d/17944.misc deleted file mode 100644 index a8a645103f..0000000000 --- a/changelog.d/17944.misc +++ /dev/null @@ -1 +0,0 @@ -Raise setuptools_rust version cap to 1.10.2. \ No newline at end of file diff --git a/changelog.d/17945.misc b/changelog.d/17945.misc deleted file mode 100644 index eeebb92169..0000000000 --- a/changelog.d/17945.misc +++ /dev/null @@ -1 +0,0 @@ -Enable encrypted appservice related experimental features in the complement docker image. diff --git a/changelog.d/17947.feature b/changelog.d/17947.feature deleted file mode 100644 index 2d1b99cec2..0000000000 --- a/changelog.d/17947.feature +++ /dev/null @@ -1 +0,0 @@ -Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. diff --git a/changelog.d/17952.misc b/changelog.d/17952.misc deleted file mode 100644 index 84fc8bfc29..0000000000 --- a/changelog.d/17952.misc +++ /dev/null @@ -1 +0,0 @@ -Return whether the user is suspended when querying the user account in the Admin API. \ No newline at end of file diff --git a/changelog.d/17953.doc b/changelog.d/17953.doc deleted file mode 100644 index 10f5a27ba9..0000000000 --- a/changelog.d/17953.doc +++ /dev/null @@ -1 +0,0 @@ -Link to element-docker-demo from contrib/docker*. diff --git a/changelog.d/17962.misc b/changelog.d/17962.misc deleted file mode 100644 index adf6348707..0000000000 --- a/changelog.d/17962.misc +++ /dev/null @@ -1 +0,0 @@ -Fix new scheduled tasks jumping the queue. diff --git a/changelog.d/17965.feature b/changelog.d/17965.feature deleted file mode 100644 index e447a58986..0000000000 --- a/changelog.d/17965.feature +++ /dev/null @@ -1 +0,0 @@ -Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). \ No newline at end of file diff --git a/changelog.d/17966.misc b/changelog.d/17966.misc deleted file mode 100644 index c6d6e55fbf..0000000000 --- a/changelog.d/17966.misc +++ /dev/null @@ -1 +0,0 @@ -Bump pyo3 and dependencies to v0.23.2. \ No newline at end of file diff --git a/changelog.d/17969.misc b/changelog.d/17969.misc deleted file mode 100644 index 05506daaa0..0000000000 --- a/changelog.d/17969.misc +++ /dev/null @@ -1 +0,0 @@ -Update setuptools-rust and fix building abi3 wheels in latest version. diff --git a/changelog.d/17970.bugfix b/changelog.d/17970.bugfix deleted file mode 100644 index 835079de3f..0000000000 --- a/changelog.d/17970.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix release process to not create duplicate releases. diff --git a/changelog.d/17972.misc b/changelog.d/17972.misc deleted file mode 100644 index e7f009d20d..0000000000 --- a/changelog.d/17972.misc +++ /dev/null @@ -1 +0,0 @@ -Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. diff --git a/changelog.d/17975.feature b/changelog.d/17975.feature deleted file mode 100644 index 48f41bddad..0000000000 --- a/changelog.d/17975.feature +++ /dev/null @@ -1 +0,0 @@ -[MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. diff --git a/changelog.d/17986.misc b/changelog.d/17986.misc deleted file mode 100644 index c062f3ecdf..0000000000 --- a/changelog.d/17986.misc +++ /dev/null @@ -1 +0,0 @@ -Fix Docker and Complement config to be able to use `public_baseurl`. diff --git a/changelog.d/17993.misc b/changelog.d/17993.misc deleted file mode 100644 index 149cb358c3..0000000000 --- a/changelog.d/17993.misc +++ /dev/null @@ -1 +0,0 @@ -Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. \ No newline at end of file diff --git a/changelog.d/17995.misc b/changelog.d/17995.misc deleted file mode 100644 index d8e608c1ae..0000000000 --- a/changelog.d/17995.misc +++ /dev/null @@ -1 +0,0 @@ -Pin `softprops/action-gh-release` to v0.1.15 to work around https://github.com/softprops/action-gh-release/issues/445. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 3aa74d1d24..805c036c82 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.121.0~rc1) stable; urgency=medium + + * New Synapse release 1.121.0rc1. + + -- Synapse Packaging team Wed, 04 Dec 2024 14:47:23 +0000 + matrix-synapse-py3 (1.120.2) stable; urgency=medium * New synapse release 1.120.2. diff --git a/pyproject.toml b/pyproject.toml index 60c62015b7..e5051770f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.120.2" +version = "1.121.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From b90ad26ebc90ef286fad3b9a60699e6fdbbe2bd9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Dec 2024 16:56:42 -0700 Subject: [PATCH 033/691] Promote account suspension to stable (#17964) MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/3823 --- changelog.d/17964.feature | 1 + synapse/api/errors.py | 5 +++-- synapse/config/experimental.py | 4 ---- synapse/rest/admin/__init__.py | 3 +-- tests/rest/admin/test_user.py | 1 - tests/rest/client/test_rooms.py | 32 ++++++++------------------------ 6 files changed, 13 insertions(+), 33 deletions(-) create mode 100644 changelog.d/17964.feature diff --git a/changelog.d/17964.feature b/changelog.d/17964.feature new file mode 100644 index 0000000000..e2ae566eb9 --- /dev/null +++ b/changelog.d/17964.feature @@ -0,0 +1 @@ +Support stable account suspension from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). \ No newline at end of file diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 71e4bb4971..21989b6e0e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -100,8 +100,9 @@ class Codes(str, Enum): # The account has been suspended on the server. # By opposition to `USER_DEACTIVATED`, this is a reversible measure # that can possibly be appealed and reverted. - # Part of MSC3823. - USER_ACCOUNT_SUSPENDED = "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" + # Introduced by MSC3823 + # https://github.com/matrix-org/matrix-spec-proposals/pull/3823 + USER_ACCOUNT_SUSPENDED = "M_USER_SUSPENDED" BAD_ALIAS = "M_BAD_ALIAS" # For restricted join rules. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 57ac27697f..eb8d967e70 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -436,10 +436,6 @@ class ExperimentalConfig(Config): ("experimental", "msc4108_delegation_endpoint"), ) - self.msc3823_account_suspension = experimental.get( - "msc3823_account_suspension", False - ) - # MSC4151: Report room API (Client-Server API) self.msc4151_enabled: bool = experimental.get("msc4151_enabled", False) diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 4db8975674..c01282a43e 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -332,8 +332,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: BackgroundUpdateRestServlet(hs).register(http_server) BackgroundUpdateStartJobRestServlet(hs).register(http_server) ExperimentalFeaturesRestServlet(hs).register(http_server) - if hs.config.experimental.msc3823_account_suspension: - SuspendAccountRestServlet(hs).register(http_server) + SuspendAccountRestServlet(hs).register(http_server) def register_servlets_for_client_rest_resource( diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index fdb8fafa0e..9a0e90208d 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -5031,7 +5031,6 @@ class UserSuspensionTestCase(unittest.HomeserverTestCase): self.store = hs.get_datastores().main - @override_config({"experimental_features": {"msc3823_account_suspension": True}}) def test_suspend_user(self) -> None: # test that suspending user works channel = self.make_request( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 07600418ed..4cf1a3dc51 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -1337,17 +1337,13 @@ class RoomJoinTestCase(RoomBase): "POST", f"/join/{self.room1}", access_token=self.tok2 ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") channel = self.make_request( "POST", f"/rooms/{self.room1}/join", access_token=self.tok2 ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_knock_on_room(self) -> None: # set the user as suspended @@ -1361,9 +1357,7 @@ class RoomJoinTestCase(RoomBase): shorthand=False, ) self.assertEqual(channel.code, 403) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_invite_to_room(self) -> None: # set the user as suspended @@ -1376,9 +1370,7 @@ class RoomJoinTestCase(RoomBase): access_token=self.tok1, content={"user_id": self.user2}, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): @@ -4011,9 +4003,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): access_token=self.tok1, content={"body": "hello", "msgtype": "m.text"}, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_change_profile_data(self) -> None: # set the user as suspended @@ -4026,9 +4016,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"avatar_url": "mxc://matrix.org/wefh34uihSDRGhw34"}, shorthand=False, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") channel2 = self.make_request( "PUT", @@ -4037,9 +4025,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"displayname": "something offensive"}, shorthand=False, ) - self.assertEqual( - channel2.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel2.json_body["errcode"], "M_USER_SUSPENDED") def test_suspended_user_cannot_redact_messages_other_than_their_own(self) -> None: # first user sends message @@ -4073,9 +4059,7 @@ class UserSuspensionTests(unittest.HomeserverTestCase): content={"reason": "bogus"}, shorthand=False, ) - self.assertEqual( - channel.json_body["errcode"], "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" - ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") # but can redact their own channel = self.make_request( From 02d09e3f0c334d979d497bb2474fed36c149e137 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 5 Dec 2024 15:41:57 +0000 Subject: [PATCH 034/691] Add RoomID & EventID rust types (#17996) Adds the RoomID & EventID rust types to the rust lib. Also adds a Deserialize impl to the existing UserID type. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/17996.misc | 1 + rust/src/identifier.rs | 166 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 changelog.d/17996.misc diff --git a/changelog.d/17996.misc b/changelog.d/17996.misc new file mode 100644 index 0000000000..8664ee45ae --- /dev/null +++ b/changelog.d/17996.misc @@ -0,0 +1 @@ +Add `RoomID` & `EventID` rust types. diff --git a/rust/src/identifier.rs b/rust/src/identifier.rs index b199c5838e..b70f6a30c7 100644 --- a/rust/src/identifier.rs +++ b/rust/src/identifier.rs @@ -71,6 +71,34 @@ impl TryFrom<&str> for UserID { } } +impl TryFrom for UserID { + type Error = IdentifierError; + + /// Will try creating a `UserID` from the provided `&str`. + /// Can fail if the user_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('@') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(UserID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for UserID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + UserID::try_from(s).map_err(serde::de::Error::custom) + } +} + impl Deref for UserID { type Target = str; @@ -84,3 +112,141 @@ impl fmt::Display for UserID { write!(f, "{}", self.0) } } + +/// A Matrix room_id. +#[derive(Clone, Debug, PartialEq)] +pub struct RoomID(String); + +impl RoomID { + /// Returns the `localpart` of the room_id. + pub fn localpart(&self) -> &str { + &self[1..self.colon_pos()] + } + + /// Returns the `server_name` / `domain` of the room_id. + pub fn server_name(&self) -> &str { + &self[self.colon_pos() + 1..] + } + + /// Returns the position of the ':' inside of the room_id. + /// Used when splitting the room_id into it's respective parts. + fn colon_pos(&self) -> usize { + self.find(':').unwrap() + } +} + +impl TryFrom<&str> for RoomID { + type Error = IdentifierError; + + /// Will try creating a `RoomID` from the provided `&str`. + /// Can fail if the room_id is incorrectly formatted. + fn try_from(s: &str) -> Result { + if !s.starts_with('!') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(RoomID(s.to_string())) + } +} + +impl TryFrom for RoomID { + type Error = IdentifierError; + + /// Will try creating a `RoomID` from the provided `String`. + /// Can fail if the room_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('!') { + return Err(IdentifierError::IncorrectSigil); + } + + if s.find(':').is_none() { + return Err(IdentifierError::MissingColon); + } + + Ok(RoomID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for RoomID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + RoomID::try_from(s).map_err(serde::de::Error::custom) + } +} + +impl Deref for RoomID { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for RoomID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A Matrix event_id. +#[derive(Clone, Debug, PartialEq)] +pub struct EventID(String); + +impl TryFrom<&str> for EventID { + type Error = IdentifierError; + + /// Will try creating a `EventID` from the provided `&str`. + /// Can fail if the event_id is incorrectly formatted. + fn try_from(s: &str) -> Result { + if !s.starts_with('$') { + return Err(IdentifierError::IncorrectSigil); + } + + Ok(EventID(s.to_string())) + } +} + +impl TryFrom for EventID { + type Error = IdentifierError; + + /// Will try creating a `EventID` from the provided `String`. + /// Can fail if the event_id is incorrectly formatted. + fn try_from(s: String) -> Result { + if !s.starts_with('$') { + return Err(IdentifierError::IncorrectSigil); + } + + Ok(EventID(s)) + } +} + +impl<'de> serde::Deserialize<'de> for EventID { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: String = serde::Deserialize::deserialize(deserializer)?; + EventID::try_from(s).map_err(serde::de::Error::custom) + } +} + +impl Deref for EventID { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for EventID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} From 0db5d247f884a2df24548c066b48f8b0b4084a0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:07:40 +0000 Subject: [PATCH 035/691] Bump python-multipart from 0.0.16 to 0.0.18 (#17985) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index af5d9a4406..1c8d66a834 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1954,13 +1954,13 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.16" +version = "0.0.18" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.8" files = [ - {file = "python_multipart-0.0.16-py3-none-any.whl", hash = "sha256:c2759b7b976ef3937214dfb592446b59dfaa5f04682a076f78b117c94776d87a"}, - {file = "python_multipart-0.0.16.tar.gz", hash = "sha256:8dee37b88dab9b59922ca173c35acb627cc12ec74019f5cd4578369c6df36554"}, + {file = "python_multipart-0.0.18-py3-none-any.whl", hash = "sha256:efe91480f485f6a361427a541db4796f9e1591afc0fb8e7a4ba06bfbc6708996"}, + {file = "python_multipart-0.0.18.tar.gz", hash = "sha256:7a68db60c8bfb82e460637fa4750727b45af1d5e2ed215593f917f64694d34fe"}, ] [[package]] From ecbc0b740cc881c6094ef6ed22ab9d9e7b65e322 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:37:40 +0000 Subject: [PATCH 036/691] Bump dawidd6/action-download-artifact from 6 to 7 (#17981) --- .github/workflows/docs-pr-netlify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 6d184a21e0..3962f75055 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 with: workflow: docs-pr.yaml run_id: ${{ github.event.workflow_run.id }} From 790ce14e46478723caf47bcf2df6a8db1ec6119f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:54:53 +0000 Subject: [PATCH 037/691] Bump pyo3 from 0.23.2 to 0.23.3 (#18001) --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e74e7226f2..f97c48c6e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -272,9 +272,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54b3d09cbdd1f8c20650b28e7b09e338881482f4aa908a5f61a00c98fba2690" +checksum = "e484fd2c8b4cb67ab05a318f1fd6fa8f199fcc30819f08f07d200809dba26c15" dependencies = [ "anyhow", "cfg-if", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3015cf985888fe66cfb63ce0e321c603706cd541b7aec7ddd35c281390af45d8" +checksum = "dc0e0469a84f208e20044b98965e1561028180219e35352a2afaf2b942beff3b" dependencies = [ "once_cell", "target-lexicon", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fca7cd8fd809b5ac4eefb89c1f98f7a7651d3739dfb341ca6980090f554c270" +checksum = "eb1547a7f9966f6f1a0f0227564a9945fe36b90da5a93b3933fc3dc03fae372d" dependencies = [ "libc", "pyo3-build-config", @@ -322,9 +322,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e657fa5379a79151b6ff5328d9216a84f55dc93b17b08e7c3609a969b73aa0" +checksum = "fdb6da8ec6fa5cedd1626c886fc8749bdcbb09424a86461eb8cdf096b7c33257" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295548d5ffd95fd1981d2d3cf4458831b21d60af046b729b6fd143b0ba7aee2f" +checksum = "38a385202ff5a92791168b1136afae5059d3ac118457bb7bc304c197c2d33e7d" dependencies = [ "heck", "proc-macro2", From adce8a0111a37795bc4167afb2543a8877320b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Hirsch?= Date: Mon, 9 Dec 2024 17:30:03 +0100 Subject: [PATCH 038/691] Reorganize account data, receipts and presence request regexps in generic_worker docs (#17954) POST requests for account data, receipts and presence require the worker to be configured as a stream writer. The regular expressions in the default list don't assume any HTTP method, so if the worker is not a stream writer, the request fails. The stream writer section of the documentation lists the same regexps as the one I'm removing, so people configuring stream writers can still configure their routing properly. More context: https://github.com/element-hq/synapse/issues/17243#issuecomment-2493621645 --- changelog.d/17954.doc | 1 + docs/workers.md | 18 +++++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) create mode 100644 changelog.d/17954.doc diff --git a/changelog.d/17954.doc b/changelog.d/17954.doc new file mode 100644 index 0000000000..8986e41e58 --- /dev/null +++ b/changelog.d/17954.doc @@ -0,0 +1 @@ +Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL. diff --git a/docs/workers.md b/docs/workers.md index 0116c455bc..6c614bd6d0 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -273,17 +273,6 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/knock/ ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ - # Account data requests - ^/_matrix/client/(r0|v3|unstable)/.*/tags - ^/_matrix/client/(r0|v3|unstable)/.*/account_data - - # Receipts requests - ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt - ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers - - # Presence requests - ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ - # User directory search requests ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ @@ -292,6 +281,13 @@ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events + # Account data requests + ^/_matrix/client/(r0|v3|unstable)/.*/tags + ^/_matrix/client/(r0|v3|unstable)/.*/account_data + + # Presence requests + ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + Pagination requests can also be handled, but all requests for a given room must be routed to the same instance. Additionally, care must be taken to ensure that the purge history admin API is not used while pagination requests From 5cf74c2da0c9d94db2fe8bb91234890a14bb7b91 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Tue, 10 Dec 2024 14:26:38 +0000 Subject: [PATCH 039/691] Fix bug when rejecting withdrew invite with a third_party_rules module (#17930) When rejecting a withdrew invite through federation, an out of band event needs to be created. When doing so with a third_party_rules module installed, `get_prev_state_ids` [is called](https://github.com/element-hq/synapse/blob/e0fdb862cbbddc920a30233024eb99038ee2fb28/synapse/module_api/callbacks/third_party_event_rules_callbacks.py#L285) on the context to calculate the state to pass at `check_event_allowed` callbacks. The context for outliers is defined [here](https://github.com/element-hq/synapse/blob/e0fdb862cbbddc920a30233024eb99038ee2fb28/synapse/events/snapshot.py#L168), and `state_group_before_event` is None. This change makes the behavior of `get_prev_state_ids` and `get_current_state_ids` match the one presented in the docstring regarding null state_group. --- changelog.d/17930.bugfix | 1 + synapse/events/snapshot.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/17930.bugfix diff --git a/changelog.d/17930.bugfix b/changelog.d/17930.bugfix new file mode 100644 index 0000000000..2e37686857 --- /dev/null +++ b/changelog.d/17930.bugfix @@ -0,0 +1 @@ +Fix bug when rejecting withdrew invite with a third_party_rules module, where the invite would be stuck for the client. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index dd21a6136b..0bca4c188b 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -248,7 +248,7 @@ class EventContext(UnpersistedEventContextBase): @tag_args async def get_current_state_ids( self, state_filter: Optional["StateFilter"] = None - ) -> Optional[StateMap[str]]: + ) -> StateMap[str]: """ Gets the room state map, including this event - ie, the state in ``state_group`` @@ -256,13 +256,12 @@ class EventContext(UnpersistedEventContextBase): not make it into the room state. This method will raise an exception if ``rejected`` is set. + It is also an error to access this for an outlier event. + Arg: state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules Returns: - Returns None if state_group is None, which happens when the associated - event is an outlier. - Maps a (type, state_key) to the event ID of the state event matching this tuple. """ @@ -300,7 +299,8 @@ class EventContext(UnpersistedEventContextBase): this tuple. """ - assert self.state_group_before_event is not None + if self.state_group_before_event is None: + return {} return await self._storage.state.get_state_ids_for_group( self.state_group_before_event, state_filter ) From 5da7081197d4b65b3e3241b80726d95a2867eaff Mon Sep 17 00:00:00 2001 From: jahway603 <64485701+jahway603@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:24:03 -0500 Subject: [PATCH 040/691] Update Alpine Linux Synapse Package Maintainer within installation.md (#17846) Update Alpine Linux Synapse Package Maintainer within installation.md as it is outdated. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [N/A] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/17846.misc | 1 + docs/setup/installation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17846.misc diff --git a/changelog.d/17846.misc b/changelog.d/17846.misc new file mode 100644 index 0000000000..6d1f930f10 --- /dev/null +++ b/changelog.d/17846.misc @@ -0,0 +1 @@ +Update Alpine Linux Synapse Package Maintainer within installation.md. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index bfeacab375..d0081a88d9 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -157,7 +157,7 @@ sudo pip install py-bcrypt #### Alpine Linux -6543 maintains [Synapse packages for Alpine Linux](https://pkgs.alpinelinux.org/packages?name=synapse&branch=edge) in the community repository. Install with: +Jahway603 maintains [Synapse packages for Alpine Linux](https://pkgs.alpinelinux.org/packages?name=synapse&branch=edge) in the community repository. Install with: ```sh sudo apk add synapse From 5b0873516c5ecf86e6cc4dff95f087410599b1a1 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:40:36 +0100 Subject: [PATCH 041/691] Attempt to fix duplicate releases issue (#18025) This hopefully fixes https://github.com/element-hq/synapse/issues/17991, as we first upgraded to v2 and are now back to 0.1.15. (This was lost in https://github.com/element-hq/synapse/pull/17923, related https://github.com/element-hq/synapse/pull/17995) --- .github/workflows/release-artifacts.yml | 4 ++++ changelog.d/18025.misc | 1 + 2 files changed, 5 insertions(+) create mode 100644 changelog.d/18025.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 5b5bfc1896..10583bc060 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -221,3 +221,7 @@ jobs: Sdist/* Wheel*/* debs.tar.xz + # if it's not already published, keep the release as a draft. + draft: true + # mark it as a prerelease if the tag contains 'rc'. + prerelease: ${{ contains(github.ref, 'rc') }} diff --git a/changelog.d/18025.misc b/changelog.d/18025.misc new file mode 100644 index 0000000000..835079de3f --- /dev/null +++ b/changelog.d/18025.misc @@ -0,0 +1 @@ +Fix release process to not create duplicate releases. From ed6edc17d0bef167b0833d962e7abaf2449a21ef Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:12:50 +0100 Subject: [PATCH 042/691] 1.121.0 --- CHANGES.md | 9 +++++++++ changelog.d/18025.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18025.misc diff --git a/CHANGES.md b/CHANGES.md index 20177bf00a..6096463cc6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.121.0 (2024-12-11) + +### Internal Changes + +- Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) + + + + # Synapse 1.121.0rc1 (2024-12-04) This release candidate contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). diff --git a/changelog.d/18025.misc b/changelog.d/18025.misc deleted file mode 100644 index 835079de3f..0000000000 --- a/changelog.d/18025.misc +++ /dev/null @@ -1 +0,0 @@ -Fix release process to not create duplicate releases. diff --git a/debian/changelog b/debian/changelog index 805c036c82..8149cd7486 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.121.0) stable; urgency=medium + + * New Synapse release 1.121.0. + + -- Synapse Packaging team Wed, 11 Dec 2024 13:12:30 +0100 + matrix-synapse-py3 (1.121.0~rc1) stable; urgency=medium * New Synapse release 1.121.0rc1. diff --git a/pyproject.toml b/pyproject.toml index e5051770f5..a8a86b4d5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.121.0rc1" +version = "1.121.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 737f6c73f7ccb611c271fd568a6af0a7c705619d Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:20:39 +0100 Subject: [PATCH 043/691] Update changelog --- CHANGES.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6096463cc6..2ef27f6ade 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,18 +1,15 @@ # Synapse 1.121.0 (2024-12-11) +This release contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). + ### Internal Changes - Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) - # Synapse 1.121.0rc1 (2024-12-04) -This release candidate contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). - -New changes listed below. - ### Features - Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. ([\#17705](https://github.com/element-hq/synapse/issues/17705)) From 6c4037dcf3eaa75b55952cd589b0d72d7109c0a0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:27:56 +0100 Subject: [PATCH 044/691] Downgrade ubuntu to 22.04 when building docker images (#18026) As currently all docker builds are failing. https://github.blog/changelog/2024-12-05-notice-of-upcoming-releases-and-breaking-changes-for-github-actions/ https://github.com/actions/runner-images/issues/10636 --- .github/workflows/docker.yml | 2 +- changelog.d/18026.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18026.misc diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ebf866e3d5..82cacdfeb3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,7 +14,7 @@ permissions: id-token: write # needed for signing the images with GitHub OIDC Token jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Set up QEMU id: qemu diff --git a/changelog.d/18026.misc b/changelog.d/18026.misc new file mode 100644 index 0000000000..729f72c4a2 --- /dev/null +++ b/changelog.d/18026.misc @@ -0,0 +1 @@ +Downgrade the Ubuntu GHA runner when building docker images. From daa783f16c18ae59801de68d9c9d5d96b8bfb9f0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 11 Dec 2024 18:25:44 +0000 Subject: [PATCH 045/691] 1.121.1 --- CHANGES.md | 11 +++++++++++ changelog.d/18026.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18026.misc diff --git a/CHANGES.md b/CHANGES.md index 2ef27f6ade..7b124d8a40 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +# Synapse 1.121.1 (2024-12-11) + +This release contains a fix for our docker build CI. It is functionally identical to 1.121.0, whose changelog is below. + +### Internal Changes + +- Downgrade the Ubuntu GHA runner when building docker images. ([\#18026](https://github.com/element-hq/synapse/issues/18026)) + + + + # Synapse 1.121.0 (2024-12-11) This release contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). diff --git a/changelog.d/18026.misc b/changelog.d/18026.misc deleted file mode 100644 index 729f72c4a2..0000000000 --- a/changelog.d/18026.misc +++ /dev/null @@ -1 +0,0 @@ -Downgrade the Ubuntu GHA runner when building docker images. diff --git a/debian/changelog b/debian/changelog index 8149cd7486..fc34d8ac3f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.121.1) stable; urgency=medium + + * New Synapse release 1.121.1. + + -- Synapse Packaging team Wed, 11 Dec 2024 18:24:48 +0000 + matrix-synapse-py3 (1.121.0) stable; urgency=medium * New Synapse release 1.121.0. diff --git a/pyproject.toml b/pyproject.toml index a8a86b4d5c..0430371586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.121.0" +version = "1.121.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From ac429050bc2c97a7c6336e874ecb4283b44f0449 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 11 Dec 2024 18:28:45 +0000 Subject: [PATCH 046/691] Remove redundant security disclaimer --- CHANGES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7b124d8a40..116f48d2e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,8 +11,6 @@ This release contains a fix for our docker build CI. It is functionally identica # Synapse 1.121.0 (2024-12-11) -This release contains the security fixes from [v1.120.2](https://github.com/element-hq/synapse/releases/tag/v1.120.2). - ### Internal Changes - Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) From eedab12e6d794420b3b16e38cb4314dab6a9034c Mon Sep 17 00:00:00 2001 From: Wilson Date: Fri, 13 Dec 2024 09:17:41 -0500 Subject: [PATCH 047/691] forward requester id to check username for spam callbacks (#17916) --- changelog.d/17916.feature | 1 + docs/modules/spam_checker_callbacks.md | 4 ++- docs/spam_checker.md | 4 +-- synapse/handlers/user_directory.py | 2 +- .../callbacks/spamchecker_callbacks.py | 30 ++++++++++++++-- tests/handlers/test_user_directory.py | 36 +++++++++++++++++++ 6 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 changelog.d/17916.feature diff --git a/changelog.d/17916.feature b/changelog.d/17916.feature new file mode 100644 index 0000000000..118997c5e5 --- /dev/null +++ b/changelog.d/17916.feature @@ -0,0 +1 @@ +Module developers will have access to user id of requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. \ No newline at end of file diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index ec306d81ab..c7f8606fd0 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -245,7 +245,7 @@ this callback. _First introduced in Synapse v1.37.0_ ```python -async def check_username_for_spam(user_profile: synapse.module_api.UserProfile) -> bool +async def check_username_for_spam(user_profile: synapse.module_api.UserProfile, requester_id: str) -> bool ``` Called when computing search results in the user directory. The module must return a @@ -264,6 +264,8 @@ The profile is represented as a dictionary with the following keys: The module is given a copy of the original dictionary, so modifying it from within the module cannot modify a user's profile when included in user directory search results. +The requester_id parameter is the ID of the user that called the user directory API. + If multiple modules implement this callback, they will be considered in order. If a callback returns `False`, Synapse falls through to the next one. The value of the first callback that does not return `False` will be used. If this happens, Synapse will not call diff --git a/docs/spam_checker.md b/docs/spam_checker.md index 1b6d814937..4ace3512b3 100644 --- a/docs/spam_checker.md +++ b/docs/spam_checker.md @@ -72,8 +72,8 @@ class ExampleSpamChecker: async def user_may_publish_room(self, userid, room_id): return True # allow publishing of all rooms - async def check_username_for_spam(self, user_profile): - return False # allow all usernames + async def check_username_for_spam(self, user_profile, requester_id): + return False # allow all usernames regardless of requester async def check_registration_for_spam( self, diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index a343637b82..1281929d38 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -161,7 +161,7 @@ class UserDirectoryHandler(StateDeltasHandler): non_spammy_users = [] for user in results["results"]: if not await self._spam_checker_module_callbacks.check_username_for_spam( - user + user, user_id ): non_spammy_users.append(user) results["results"] = non_spammy_users diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index 17079ff781..a2f328cafe 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -31,6 +31,7 @@ from typing import ( Optional, Tuple, Union, + cast, ) # `Literal` appears with Python 3.8. @@ -168,7 +169,10 @@ USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[ ] ], ] -CHECK_USERNAME_FOR_SPAM_CALLBACK = Callable[[UserProfile], Awaitable[bool]] +CHECK_USERNAME_FOR_SPAM_CALLBACK = Union[ + Callable[[UserProfile], Awaitable[bool]], + Callable[[UserProfile, str], Awaitable[bool]], +] LEGACY_CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[ [ Optional[dict], @@ -716,7 +720,9 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM - async def check_username_for_spam(self, user_profile: UserProfile) -> bool: + async def check_username_for_spam( + self, user_profile: UserProfile, requester_id: str + ) -> bool: """Checks if a user ID or display name are considered "spammy" by this server. If the server considers a username spammy, then it will not be included in @@ -727,15 +733,33 @@ class SpamCheckerModuleApiCallbacks: * user_id * display_name * avatar_url + requester_id: The user ID of the user making the user directory search request. Returns: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + checker_args = inspect.signature(callback) # Make a copy of the user profile object to ensure the spam checker cannot # modify it. - res = await delay_cancellation(callback(user_profile.copy())) + # Also ensure backwards compatibility with spam checker callbacks + # that don't expect the requester_id argument. + if len(checker_args.parameters) == 2: + callback_with_requester_id = cast( + Callable[[UserProfile, str], Awaitable[bool]], callback + ) + res = await delay_cancellation( + callback_with_requester_id(user_profile.copy(), requester_id) + ) + else: + callback_without_requester_id = cast( + Callable[[UserProfile], Awaitable[bool]], callback + ) + res = await delay_cancellation( + callback_without_requester_id(user_profile.copy()) + ) + if res: return True diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index 878d9683b6..a75095a79f 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -796,6 +796,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 1) + # Kept old spam checker without `requester_id` tests for backwards compatibility. async def allow_all(user_profile: UserProfile) -> bool: # Allow all users. return False @@ -809,6 +810,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 1) + # Kept old spam checker without `requester_id` tests for backwards compatibility. # Configure a spam checker that filters all users. async def block_all(user_profile: UserProfile) -> bool: # All users are spammy. @@ -820,6 +822,40 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 0) + async def allow_all_expects_requester_id( + user_profile: UserProfile, requester_id: str + ) -> bool: + self.assertEqual(requester_id, u1) + # Allow all users. + return False + + # Configure a spam checker that does not filter any users. + spam_checker = self.hs.get_module_api_callbacks().spam_checker + spam_checker._check_username_for_spam_callbacks = [ + allow_all_expects_requester_id + ] + + # The results do not change: + # We get one search result when searching for user2 by user1. + s = self.get_success(self.handler.search_users(u1, "user2", 10)) + self.assertEqual(len(s["results"]), 1) + + # Configure a spam checker that filters all users. + async def block_all_expects_requester_id( + user_profile: UserProfile, requester_id: str + ) -> bool: + self.assertEqual(requester_id, u1) + # All users are spammy. + return True + + spam_checker._check_username_for_spam_callbacks = [ + block_all_expects_requester_id + ] + + # User1 now gets no search results for any of the other users. + s = self.get_success(self.handler.search_users(u1, "user2", 10)) + self.assertEqual(len(s["results"]), 0) + @override_config( { "spam_checker": { From eb019c03c4e991ec956a655f62d846d382a60ce3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:58:34 +0000 Subject: [PATCH 048/691] Bump anyhow from 1.0.93 to 1.0.94 (#18012) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f97c48c6e4..6825d965fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arc-swap" From 95853c5f3105e036f4e03c2e0ea3af6ca821c847 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:03:42 +0000 Subject: [PATCH 049/691] Bump pydantic from 2.9.2 to 2.10.3 (#18014) --- poetry.lock | 204 +++++++++++++++++++++++++++------------------------- 1 file changed, 106 insertions(+), 98 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1c8d66a834..f8eebba3bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1660,22 +1660,19 @@ files = [ [[package]] name = "pydantic" -version = "2.9.2" +version = "2.10.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, + {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, -] +pydantic-core = "2.27.1" +typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1683,100 +1680,111 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.27.1" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206"}, + {file = "pydantic_core-2.27.1-cp310-none-win32.whl", hash = "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c"}, + {file = "pydantic_core-2.27.1-cp310-none-win_amd64.whl", hash = "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc"}, + {file = "pydantic_core-2.27.1-cp311-none-win32.whl", hash = "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9"}, + {file = "pydantic_core-2.27.1-cp311-none-win_amd64.whl", hash = "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5"}, + {file = "pydantic_core-2.27.1-cp311-none-win_arm64.whl", hash = "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae"}, + {file = "pydantic_core-2.27.1-cp312-none-win32.whl", hash = "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c"}, + {file = "pydantic_core-2.27.1-cp312-none-win_amd64.whl", hash = "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16"}, + {file = "pydantic_core-2.27.1-cp312-none-win_arm64.whl", hash = "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23"}, + {file = "pydantic_core-2.27.1-cp313-none-win32.whl", hash = "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05"}, + {file = "pydantic_core-2.27.1-cp313-none-win_amd64.whl", hash = "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337"}, + {file = "pydantic_core-2.27.1-cp313-none-win_arm64.whl", hash = "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b"}, + {file = "pydantic_core-2.27.1-cp38-none-win32.whl", hash = "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618"}, + {file = "pydantic_core-2.27.1-cp38-none-win_amd64.whl", hash = "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131"}, + {file = "pydantic_core-2.27.1-cp39-none-win32.whl", hash = "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3"}, + {file = "pydantic_core-2.27.1-cp39-none-win_amd64.whl", hash = "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840"}, + {file = "pydantic_core-2.27.1.tar.gz", hash = "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235"}, ] [package.dependencies] From 35c361c0d9bc4aa2c3018356387b752bd6e92d00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:19:09 +0000 Subject: [PATCH 050/691] Bump pillow from 10.4.0 to 11.0.0 (#18015) --- poetry.lock | 161 +++++++++++++++++++++++++--------------------------- 1 file changed, 78 insertions(+), 83 deletions(-) diff --git a/poetry.lock b/poetry.lock index f8eebba3bd..c4b34c17f6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1454,95 +1454,90 @@ files = [ [[package]] name = "pillow" -version = "10.4.0" +version = "11.0.0" description = "Python Imaging Library (Fork)" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"}, - {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b"}, - {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc"}, - {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e"}, - {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46"}, - {file = "pillow-10.4.0-cp310-cp310-win32.whl", hash = "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984"}, - {file = "pillow-10.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141"}, - {file = "pillow-10.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1"}, - {file = "pillow-10.4.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c"}, - {file = "pillow-10.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe"}, - {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319"}, - {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d"}, - {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696"}, - {file = "pillow-10.4.0-cp311-cp311-win32.whl", hash = "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496"}, - {file = "pillow-10.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91"}, - {file = "pillow-10.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22"}, - {file = "pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94"}, - {file = "pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef"}, - {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a"}, - {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b"}, - {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9"}, - {file = "pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42"}, - {file = "pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a"}, - {file = "pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9"}, - {file = "pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3"}, - {file = "pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0"}, - {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc"}, - {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a"}, - {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309"}, - {file = "pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060"}, - {file = "pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea"}, - {file = "pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d"}, - {file = "pillow-10.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736"}, - {file = "pillow-10.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b"}, - {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd"}, - {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84"}, - {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0"}, - {file = "pillow-10.4.0-cp38-cp38-win32.whl", hash = "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e"}, - {file = "pillow-10.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab"}, - {file = "pillow-10.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d"}, - {file = "pillow-10.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b"}, - {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c"}, - {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1"}, - {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df"}, - {file = "pillow-10.4.0-cp39-cp39-win32.whl", hash = "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef"}, - {file = "pillow-10.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5"}, - {file = "pillow-10.4.0-cp39-cp39-win_arm64.whl", hash = "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885"}, - {file = "pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27"}, - {file = "pillow-10.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3"}, - {file = "pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06"}, + {file = "pillow-11.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6619654954dc4936fcff82db8eb6401d3159ec6be81e33c6000dfd76ae189947"}, + {file = "pillow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3c5ac4bed7519088103d9450a1107f76308ecf91d6dabc8a33a2fcfb18d0fba"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a65149d8ada1055029fcb665452b2814fe7d7082fcb0c5bed6db851cb69b2086"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88a58d8ac0cc0e7f3a014509f0455248a76629ca9b604eca7dc5927cc593c5e9"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c26845094b1af3c91852745ae78e3ea47abf3dbcd1cf962f16b9a5fbe3ee8488"}, + {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1a61b54f87ab5786b8479f81c4b11f4d61702830354520837f8cc791ebba0f5f"}, + {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:674629ff60030d144b7bca2b8330225a9b11c482ed408813924619c6f302fdbb"}, + {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:598b4e238f13276e0008299bd2482003f48158e2b11826862b1eb2ad7c768b97"}, + {file = "pillow-11.0.0-cp310-cp310-win32.whl", hash = "sha256:9a0f748eaa434a41fccf8e1ee7a3eed68af1b690e75328fd7a60af123c193b50"}, + {file = "pillow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a5629742881bcbc1f42e840af185fd4d83a5edeb96475a575f4da50d6ede337c"}, + {file = "pillow-11.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:ee217c198f2e41f184f3869f3e485557296d505b5195c513b2bfe0062dc537f1"}, + {file = "pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1c1d72714f429a521d8d2d018badc42414c3077eb187a59579f28e4270b4b0fc"}, + {file = "pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:499c3a1b0d6fc8213519e193796eb1a86a1be4b1877d678b30f83fd979811d1a"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8b2351c85d855293a299038e1f89db92a2f35e8d2f783489c6f0b2b5f3fe8a3"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4dba50cfa56f910241eb7f883c20f1e7b1d8f7d91c750cd0b318bad443f4d5"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5ddbfd761ee00c12ee1be86c9c0683ecf5bb14c9772ddbd782085779a63dd55b"}, + {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:45c566eb10b8967d71bf1ab8e4a525e5a93519e29ea071459ce517f6b903d7fa"}, + {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b4fd7bd29610a83a8c9b564d457cf5bd92b4e11e79a4ee4716a63c959699b306"}, + {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb929ca942d0ec4fac404cbf520ee6cac37bf35be479b970c4ffadf2b6a1cad9"}, + {file = "pillow-11.0.0-cp311-cp311-win32.whl", hash = "sha256:006bcdd307cc47ba43e924099a038cbf9591062e6c50e570819743f5607404f5"}, + {file = "pillow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:52a2d8323a465f84faaba5236567d212c3668f2ab53e1c74c15583cf507a0291"}, + {file = "pillow-11.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:16095692a253047fe3ec028e951fa4221a1f3ed3d80c397e83541a3037ff67c9"}, + {file = "pillow-11.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923"}, + {file = "pillow-11.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8069c5179902dcdce0be9bfc8235347fdbac249d23bd90514b7a47a72d9fecf4"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f02541ef64077f22bf4924f225c0fd1248c168f86e4b7abdedd87d6ebaceab0f"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fcb4621042ac4b7865c179bb972ed0da0218a076dc1820ffc48b1d74c1e37fe9"}, + {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7"}, + {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8853a3bf12afddfdf15f57c4b02d7ded92c7a75a5d7331d19f4f9572a89c17e6"}, + {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3107c66e43bda25359d5ef446f59c497de2b5ed4c7fdba0894f8d6cf3822dafc"}, + {file = "pillow-11.0.0-cp312-cp312-win32.whl", hash = "sha256:86510e3f5eca0ab87429dd77fafc04693195eec7fd6a137c389c3eeb4cfb77c6"}, + {file = "pillow-11.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:8ec4a89295cd6cd4d1058a5e6aec6bf51e0eaaf9714774e1bfac7cfc9051db47"}, + {file = "pillow-11.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:27a7860107500d813fcd203b4ea19b04babe79448268403172782754870dac25"}, + {file = "pillow-11.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bcd1fb5bb7b07f64c15618c89efcc2cfa3e95f0e3bcdbaf4642509de1942a699"}, + {file = "pillow-11.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0e038b0745997c7dcaae350d35859c9715c71e92ffb7e0f4a8e8a16732150f38"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ae08bd8ffc41aebf578c2af2f9d8749d91f448b3bfd41d7d9ff573d74f2a6b2"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d69bfd8ec3219ae71bcde1f942b728903cad25fafe3100ba2258b973bd2bc1b2"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:61b887f9ddba63ddf62fd02a3ba7add935d053b6dd7d58998c630e6dbade8527"}, + {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c6a660307ca9d4867caa8d9ca2c2658ab685de83792d1876274991adec7b93fa"}, + {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:73e3a0200cdda995c7e43dd47436c1548f87a30bb27fb871f352a22ab8dcf45f"}, + {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fba162b8872d30fea8c52b258a542c5dfd7b235fb5cb352240c8d63b414013eb"}, + {file = "pillow-11.0.0-cp313-cp313-win32.whl", hash = "sha256:f1b82c27e89fffc6da125d5eb0ca6e68017faf5efc078128cfaa42cf5cb38798"}, + {file = "pillow-11.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ba470552b48e5835f1d23ecb936bb7f71d206f9dfeee64245f30c3270b994de"}, + {file = "pillow-11.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:846e193e103b41e984ac921b335df59195356ce3f71dcfd155aa79c603873b84"}, + {file = "pillow-11.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4ad70c4214f67d7466bea6a08061eba35c01b1b89eaa098040a35272a8efb22b"}, + {file = "pillow-11.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6ec0d5af64f2e3d64a165f490d96368bb5dea8b8f9ad04487f9ab60dc4bb6003"}, + {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c809a70e43c7977c4a42aefd62f0131823ebf7dd73556fa5d5950f5b354087e2"}, + {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:4b60c9520f7207aaf2e1d94de026682fc227806c6e1f55bba7606d1c94dd623a"}, + {file = "pillow-11.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1e2688958a840c822279fda0086fec1fdab2f95bf2b717b66871c4ad9859d7e8"}, + {file = "pillow-11.0.0-cp313-cp313t-win32.whl", hash = "sha256:607bbe123c74e272e381a8d1957083a9463401f7bd01287f50521ecb05a313f8"}, + {file = "pillow-11.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c39ed17edea3bc69c743a8dd3e9853b7509625c2462532e62baa0732163a904"}, + {file = "pillow-11.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:75acbbeb05b86bc53cbe7b7e6fe00fbcf82ad7c684b3ad82e3d711da9ba287d3"}, + {file = "pillow-11.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2e46773dc9f35a1dd28bd6981332fd7f27bec001a918a72a79b4133cf5291dba"}, + {file = "pillow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2679d2258b7f1192b378e2893a8a0a0ca472234d4c2c0e6bdd3380e8dfa21b6a"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda2616eb2313cbb3eebbe51f19362eb434b18e3bb599466a1ffa76a033fb916"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ec184af98a121fb2da42642dea8a29ec80fc3efbaefb86d8fdd2606619045d"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:8594f42df584e5b4bb9281799698403f7af489fba84c34d53d1c4bfb71b7c4e7"}, + {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:c12b5ae868897c7338519c03049a806af85b9b8c237b7d675b8c5e089e4a618e"}, + {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:70fbbdacd1d271b77b7721fe3cdd2d537bbbd75d29e6300c672ec6bb38d9672f"}, + {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5178952973e588b3f1360868847334e9e3bf49d19e169bbbdfaf8398002419ae"}, + {file = "pillow-11.0.0-cp39-cp39-win32.whl", hash = "sha256:8c676b587da5673d3c75bd67dd2a8cdfeb282ca38a30f37950511766b26858c4"}, + {file = "pillow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:94f3e1780abb45062287b4614a5bc0874519c86a777d4a7ad34978e86428b8dd"}, + {file = "pillow-11.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:290f2cc809f9da7d6d622550bbf4c1e57518212da51b6a30fe8e0a270a5b78bd"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1187739620f2b365de756ce086fdb3604573337cc28a0d3ac4a01ab6b2d2a6d2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbbcb7b57dc9c794843e3d1258c0fbf0f48656d46ffe9e09b63bbd6e8cd5d0a2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d203af30149ae339ad1b4f710d9844ed8796e97fda23ffbc4cc472968a47d0b"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a0d3b115009ebb8ac3d2ebec5c2982cc693da935f4ab7bb5c8ebe2f47d36f2"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:73853108f56df97baf2bb8b522f3578221e56f646ba345a372c78326710d3830"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e58876c91f97b0952eb766123bfef372792ab3f4e3e1f1a2267834c2ab131734"}, + {file = "pillow-11.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:224aaa38177597bb179f3ec87eeefcce8e4f85e608025e9cfac60de237ba6316"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5bd2d3bdb846d757055910f0a59792d33b555800813c3b39ada1829c372ccb06"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:375b8dd15a1f5d2feafff536d47e22f69625c1aa92f12b339ec0b2ca40263273"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:daffdf51ee5db69a82dd127eabecce20729e21f7a3680cf7cbb23f0829189790"}, + {file = "pillow-11.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7326a1787e3c7b0429659e0a944725e1b03eeaa10edd945a86dead1913383944"}, + {file = "pillow-11.0.0.tar.gz", hash = "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739"}, ] [package.extras] -docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] From 512c9efcb37687b7f5e51cad59565b5b4f57de92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:20:16 +0000 Subject: [PATCH 051/691] Bump serde from 1.0.215 to 1.0.216 (#18031) --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6825d965fc..5dd4122d9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -431,18 +431,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", From 29d586311d2425b30a8a83ce120fa7511050f89f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:23:11 +0000 Subject: [PATCH 052/691] Bump http from 1.1.0 to 1.2.0 (#18013) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5dd4122d9e..3487064451 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,9 +168,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", From 8208186e3cc597fbf27862a1df816d209ccd21bd Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 16 Dec 2024 11:27:34 -0800 Subject: [PATCH 053/691] Add some useful endpoints to Admin API (#17948) - Fetch the number of invites the provided user has sent after a given timestamp - Fetch the number of rooms the provided user has joined after a given timestamp, regardless if they have left/been banned from the rooms subsequently - Get report IDs of event reports where the provided user was the sender of the reported event --- changelog.d/17948.feature | 3 + docs/admin_api/event_reports.md | 9 +- docs/admin_api/user_admin_api.md | 75 +++++- synapse/rest/admin/__init__.py | 4 + synapse/rest/admin/event_reports.py | 9 +- synapse/rest/admin/users.py | 54 +++- .../storage/databases/main/events_worker.py | 48 ++++ synapse/storage/databases/main/room.py | 11 +- synapse/storage/databases/main/roommember.py | 34 +++ .../delta/88/06_events_received_ts_index.sql | 17 ++ tests/rest/admin/test_event_reports.py | 35 +++ tests/rest/admin/test_user.py | 251 ++++++++++++++++++ 12 files changed, 535 insertions(+), 15 deletions(-) create mode 100644 changelog.d/17948.feature create mode 100644 synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql diff --git a/changelog.d/17948.feature b/changelog.d/17948.feature new file mode 100644 index 0000000000..d404996cd6 --- /dev/null +++ b/changelog.d/17948.feature @@ -0,0 +1,3 @@ +Add endpoints to Admin API to fetch the number of invites the provided user has sent after a given timestamp, +fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event +reports against a provided user (ie where the user was the sender of the reported event). diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md index 83f7dc37f4..9075e92882 100644 --- a/docs/admin_api/event_reports.md +++ b/docs/admin_api/event_reports.md @@ -60,10 +60,11 @@ paginate through. anything other than the return value of `next_token` from a previous call. Defaults to `0`. * `dir`: string - Direction of event report order. Whether to fetch the most recent first (`b`) or the oldest first (`f`). Defaults to `b`. -* `user_id`: string - Is optional and filters to only return users with user IDs that - contain this value. This is the user who reported the event and wrote the reason. -* `room_id`: string - Is optional and filters to only return rooms with room IDs that - contain this value. +* `user_id`: optional string - Filter by the user ID of the reporter. This is the user who reported the event + and wrote the reason. +* `room_id`: optional string - Filter by room id. +* `event_sender_user_id`: optional string - Filter by the sender of the reported event. This is the user who + the report was made against. **Response** diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index a6e2e0a153..c63b7068c5 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -477,9 +477,9 @@ with a body of: } ``` -## List room memberships of a user +## List joined rooms of a user -Gets a list of all `room_id` that a specific `user_id` is member. +Gets a list of all `room_id` that a specific `user_id` is joined to and is a member of (participating in). The API is: @@ -516,6 +516,73 @@ The following fields are returned in the JSON response body: - `joined_rooms` - An array of `room_id`. - `total` - Number of rooms. +## Get the number of invites sent by the user + +Fetches the number of invites sent by the provided user ID across all rooms +after the given timestamp. + +``` +GET /_synapse/admin/v1/users/$user_id/sent_invite_count +``` + +**Parameters** + +The following parameters should be set in the URL: + +* `user_id`: fully qualified: for example, `@user:server.com` + +The following should be set as query parameters in the URL: + +* `from_ts`: int, required. A timestamp in ms from the unix epoch. Only + invites sent at or after the provided timestamp will be returned. + This works by comparing the provided timestamp to the `received_ts` + column in the `events` table. + Note: https://currentmillis.com/ is a useful tool for converting dates + into timestamps and vice versa. + +A response body like the following is returned: + +```json +{ + "invite_count": 30 +} +``` + +_Added in Synapse 1.122.0_ + +## Get the cumulative number of rooms a user has joined after a given timestamp + +Fetches the number of rooms that the user joined after the given timestamp, even +if they have subsequently left/been banned from those rooms. + +``` +GET /_synapse/admin/v1/users/$ None: UserByThreePid(hs).register(http_server) RedactUser(hs).register(http_server) RedactUserStatus(hs).register(http_server) + UserInvitesCount(hs).register(http_server) + UserJoinedRoomCount(hs).register(http_server) DeviceRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/event_reports.py b/synapse/rest/admin/event_reports.py index 9fb68bfa46..ff1abc0697 100644 --- a/synapse/rest/admin/event_reports.py +++ b/synapse/rest/admin/event_reports.py @@ -50,8 +50,10 @@ class EventReportsRestServlet(RestServlet): The parameters `from` and `limit` are required only for pagination. By default, a `limit` of 100 is used. The parameter `dir` can be used to define the order of results. - The parameter `user_id` can be used to filter by user id. - The parameter `room_id` can be used to filter by room id. + The `user_id` query parameter filters by the user ID of the reporter of the event. + The `room_id` query parameter filters by room id. + The `event_sender_user_id` query parameter can be used to filter by the user id + of the sender of the reported event. Returns: A list of reported events and an integer representing the total number of reported events that exist given this query @@ -71,6 +73,7 @@ class EventReportsRestServlet(RestServlet): direction = parse_enum(request, "dir", Direction, Direction.BACKWARDS) user_id = parse_string(request, "user_id") room_id = parse_string(request, "room_id") + event_sender_user_id = parse_string(request, "event_sender_user_id") if start < 0: raise SynapseError( @@ -87,7 +90,7 @@ class EventReportsRestServlet(RestServlet): ) event_reports, total = await self._store.get_event_reports_paginate( - start, limit, direction, user_id, room_id + start, limit, direction, user_id, room_id, event_sender_user_id ) ret = {"event_reports": event_reports, "total": total} if (start + limit) < total: diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index b146c2754d..7b8f1d1b2a 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -983,7 +983,7 @@ class UserAdminServlet(RestServlet): class UserMembershipRestServlet(RestServlet): """ - Get room list of an user. + Get list of joined room ID's for a user. """ PATTERNS = admin_patterns("/users/(?P[^/]*)/joined_rooms$") @@ -999,8 +999,9 @@ class UserMembershipRestServlet(RestServlet): await assert_requester_is_admin(self.auth, request) room_ids = await self.store.get_rooms_for_user(user_id) - ret = {"joined_rooms": list(room_ids), "total": len(room_ids)} - return HTTPStatus.OK, ret + rooms_response = {"joined_rooms": list(room_ids), "total": len(room_ids)} + + return HTTPStatus.OK, rooms_response class PushersRestServlet(RestServlet): @@ -1501,3 +1502,50 @@ class RedactUserStatus(RestServlet): } else: raise NotFoundError("redact id '%s' not found" % redact_id) + + +class UserInvitesCount(RestServlet): + """ + Return the count of invites that the user has sent after the given timestamp + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + from_ts = parse_integer(request, "from_ts", required=True) + + sent_invite_count = await self.store.get_sent_invite_count_by_user( + user_id, from_ts + ) + + return HTTPStatus.OK, {"invite_count": sent_invite_count} + + +class UserJoinedRoomCount(RestServlet): + """ + Return the count of rooms that the user has joined at or after the given timestamp, even + if they have subsequently left/been banned from those rooms. + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + from_ts = parse_integer(request, "from_ts", required=True) + + joined_rooms = await self.store.get_rooms_for_user_by_date(user_id, from_ts) + + return HTTPStatus.OK, {"cumulative_joined_room_count": len(joined_rooms)} diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 825fd00993..222df8757a 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -339,6 +339,16 @@ class EventsWorkerStore(SQLBaseStore): writers=["master"], ) + # Added to accommodate some queries for the admin API in order to fetch/filter + # membership events by when it was received + self.db_pool.updates.register_background_index_update( + update_name="events_received_ts_index", + index_name="received_ts_idx", + table="events", + columns=("received_ts",), + where_clause="type = 'm.room.member'", + ) + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( @@ -2589,6 +2599,44 @@ class EventsWorkerStore(SQLBaseStore): ) ) + async def get_sent_invite_count_by_user(self, user_id: str, from_ts: int) -> int: + """ + Get the number of invites sent by the given user at or after the provided timestamp. + + Args: + user_id: user ID to search against + from_ts: a timestamp in milliseconds from the unix epoch. Filters against + `events.received_ts` + + """ + + def _get_sent_invite_count_by_user_txn( + txn: LoggingTransaction, user_id: str, from_ts: int + ) -> int: + sql = """ + SELECT COUNT(rm.event_id) + FROM room_memberships AS rm + INNER JOIN events AS e USING(event_id) + WHERE rm.sender = ? + AND rm.membership = 'invite' + AND e.type = 'm.room.member' + AND e.received_ts >= ? + """ + + txn.execute(sql, (user_id, from_ts)) + res = txn.fetchone() + + if res is None: + return 0 + return int(res[0]) + + return await self.db_pool.runInteraction( + "_get_sent_invite_count_by_user_txn", + _get_sent_invite_count_by_user_txn, + user_id, + from_ts, + ) + @cached(tree=True) async def get_metadata_for_event( self, room_id: str, event_id: str diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index cc3ce0951e..2522bebd72 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1586,6 +1586,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): direction: Direction = Direction.BACKWARDS, user_id: Optional[str] = None, room_id: Optional[str] = None, + event_sender_user_id: Optional[str] = None, ) -> Tuple[List[Dict[str, Any]], int]: """Retrieve a paginated list of event reports @@ -1596,6 +1597,8 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): oldest first (forwards) user_id: search for user_id. Ignored if user_id is None room_id: search for room_id. Ignored if room_id is None + event_sender_user_id: search for the sender of the reported event. Ignored if + event_sender_user_id is None Returns: Tuple of: json list of event reports @@ -1615,6 +1618,10 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): filters.append("er.room_id LIKE ?") args.extend(["%" + room_id + "%"]) + if event_sender_user_id: + filters.append("events.sender = ?") + args.extend([event_sender_user_id]) + if direction == Direction.BACKWARDS: order = "DESC" else: @@ -1630,6 +1637,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): sql = """ SELECT COUNT(*) as total_event_reports FROM event_reports AS er + LEFT JOIN events USING(event_id) JOIN room_stats_state ON room_stats_state.room_id = er.room_id {} """.format(where_clause) @@ -1648,8 +1656,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): room_stats_state.canonical_alias, room_stats_state.name FROM event_reports AS er - LEFT JOIN events - ON events.event_id = er.event_id + LEFT JOIN events USING(event_id) JOIN room_stats_state ON room_stats_state.room_id = er.room_id {where_clause} diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 4249cf77e5..50ed6a28bf 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1572,6 +1572,40 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): get_sliding_sync_room_for_user_batch_txn, ) + async def get_rooms_for_user_by_date( + self, user_id: str, from_ts: int + ) -> FrozenSet[str]: + """ + Fetch a list of rooms that the user has joined at or after the given timestamp, including + those they subsequently have left/been banned from. + + Args: + user_id: user ID of the user to search for + from_ts: a timestamp in ms from the unix epoch at which to begin the search at + """ + + def _get_rooms_for_user_by_join_date_txn( + txn: LoggingTransaction, user_id: str, timestamp: int + ) -> frozenset: + sql = """ + SELECT rm.room_id + FROM room_memberships AS rm + INNER JOIN events AS e USING (event_id) + WHERE rm.user_id = ? + AND rm.membership = 'join' + AND e.type = 'm.room.member' + AND e.received_ts >= ? + """ + txn.execute(sql, (user_id, timestamp)) + return frozenset([r[0] for r in txn]) + + return await self.db_pool.runInteraction( + "_get_rooms_for_user_by_join_date_txn", + _get_rooms_for_user_by_join_date_txn, + user_id, + from_ts, + ) + class RoomMemberBackgroundUpdateStore(SQLBaseStore): def __init__( diff --git a/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql b/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql new file mode 100644 index 0000000000..d70a4a8dbc --- /dev/null +++ b/synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add an index on `events.received_ts` for `m.room.member` events to allow for +-- efficient lookup of events by timestamp in some Admin API's +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8806, 'events_received_ts_index', '{}'); diff --git a/tests/rest/admin/test_event_reports.py b/tests/rest/admin/test_event_reports.py index feb410a11d..6047ce1f4a 100644 --- a/tests/rest/admin/test_event_reports.py +++ b/tests/rest/admin/test_event_reports.py @@ -378,6 +378,41 @@ class EventReportsTestCase(unittest.HomeserverTestCase): self.assertEqual(len(channel.json_body["event_reports"]), 1) self.assertNotIn("next_token", channel.json_body) + def test_filter_against_event_sender(self) -> None: + """ + Tests filtering by the sender of the reported event + """ + # first grab all the reports + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200) + + # filter out set of report ids of events sent by one of the users + locally_filtered_report_ids = set() + for event_report in channel.json_body["event_reports"]: + if event_report["sender"] == self.other_user: + locally_filtered_report_ids.add(event_report["id"]) + + # grab the report ids by sender and compare to filtered report ids + channel = self.make_request( + "GET", + f"{self.url}?event_sender_user_id={self.other_user}", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + self.assertEqual(channel.json_body["total"], len(locally_filtered_report_ids)) + + event_reports = channel.json_body["event_reports"] + server_filtered_report_ids = set() + for event_report in event_reports: + server_filtered_report_ids.add(event_report["id"]) + self.assertIncludes( + locally_filtered_report_ids, server_filtered_report_ids, exact=True + ) + def _create_event_and_report(self, room_id: str, user_tok: str) -> None: """Create and report events""" resp = self.helper.send(room_id, tok=user_tok) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 9a0e90208d..b517aefd0c 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -5502,3 +5502,254 @@ class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): redaction_ids.add(event["redacts"]) self.assertIncludes(redaction_ids, original_event_ids, exact=True) + + +class GetInvitesFromUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + admin.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin = self.register_user("thomas", "pass", True) + self.admin_tok = self.login("thomas", "pass") + + self.bad_user = self.register_user("teresa", "pass") + self.bad_user_tok = self.login("teresa", "pass") + + self.random_users = [] + for i in range(4): + self.random_users.append(self.register_user(f"user{i}", f"pass{i}")) + + self.room1 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + self.room2 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + self.room3 = self.helper.create_room_as(self.bad_user, tok=self.bad_user_tok) + + @unittest.override_config( + {"rc_invites": {"per_issuer": {"per_second": 1000, "burst_count": 1000}}} + ) + def test_get_user_invite_count_new_invites_test_case(self) -> None: + """ + Test that new invites that arrive after a provided timestamp are counted + """ + # grab a current timestamp + before_invites_sent_ts = self.hs.get_clock().time_msec() + + # bad user sends some invites + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite(room_id, self.bad_user, user, tok=self.bad_user_tok) + + # fetch using timestamp, all should be returned + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + # send some more invites, they should show up in addition to original 8 using same timestamp + for user in self.random_users: + self.helper.invite( + self.room3, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 12) + + def test_get_user_invite_count_invites_before_ts_test_case(self) -> None: + """ + Test that invites sent before provided ts are not counted + """ + # bad user sends some invites + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite(room_id, self.bad_user, user, tok=self.bad_user_tok) + + # add a msec between last invite and ts + after_invites_sent_ts = self.hs.get_clock().time_msec() + 1 + + # fetch invites with timestamp, none should be returned + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={after_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 0) + + def test_user_invite_count_kick_ban_not_counted(self) -> None: + """ + Test that kicks and bans are not counted in invite count + """ + to_kick_user_id = self.register_user("kick_me", "pass") + to_kick_tok = self.login("kick_me", "pass") + + self.helper.join(self.room1, to_kick_user_id, tok=to_kick_tok) + + # grab a current timestamp + before_invites_sent_ts = self.hs.get_clock().time_msec() + + # bad user sends some invites (8) + for room_id in [self.room1, self.room2]: + for user in self.random_users: + self.helper.invite( + room_id, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + # fetch using timestamp, all invites sent should be counted + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + # send a kick and some bans and make sure these aren't counted against invite total + for user in self.random_users: + self.helper.ban( + self.room1, src=self.bad_user, targ=user, tok=self.bad_user_tok + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/kick", + content={"user_id": to_kick_user_id}, + access_token=self.bad_user_tok, + ) + self.assertEqual(channel.code, 200) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/sent_invite_count?from_ts={before_invites_sent_ts}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["invite_count"], 8) + + +class GetCumulativeJoinedRoomCountForUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + admin.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin = self.register_user("thomas", "pass", True) + self.admin_tok = self.login("thomas", "pass") + + self.bad_user = self.register_user("teresa", "pass") + self.bad_user_tok = self.login("teresa", "pass") + + def test_user_cumulative_joined_room_count(self) -> None: + """ + Tests proper count returned from /cumulative_joined_room_count endpoint + """ + # Create rooms and join, grab timestamp before room creation + before_room_creation_timestamp = self.hs.get_clock().time_msec() + + joined_rooms = [] + for _ in range(3): + room = self.helper.create_room_as(self.admin, tok=self.admin_tok) + self.helper.join( + room, user=self.bad_user, expect_code=200, tok=self.bad_user_tok + ) + joined_rooms.append(room) + + # get a timestamp after room creation and join, add a msec between last join and ts + after_room_creation = self.hs.get_clock().time_msec() + 1 + + # Get rooms using this timestamp, there should be none since all rooms were created and joined + # before provided timestamp + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(after_room_creation)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(0, channel.json_body["cumulative_joined_room_count"]) + + # fetch rooms with the older timestamp before they were created and joined, this should + # return the rooms + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) + + def test_user_joined_room_count_includes_left_and_banned_rooms(self) -> None: + """ + Tests proper count returned from /joined_room_count endpoint when user has left + or been banned from joined rooms + """ + # Create rooms and join, grab timestamp before room creation + before_room_creation_timestamp = self.hs.get_clock().time_msec() + + joined_rooms = [] + for _ in range(3): + room = self.helper.create_room_as(self.admin, tok=self.admin_tok) + self.helper.join( + room, user=self.bad_user, expect_code=200, tok=self.bad_user_tok + ) + joined_rooms.append(room) + + # fetch rooms with the older timestamp before they were created and joined + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) + + # have the user banned from/leave the joined rooms + self.helper.ban( + joined_rooms[0], + src=self.admin, + targ=self.bad_user, + expect_code=200, + tok=self.admin_tok, + ) + self.helper.change_membership( + joined_rooms[1], + src=self.bad_user, + targ=self.bad_user, + membership="leave", + expect_code=200, + tok=self.bad_user_tok, + ) + self.helper.ban( + joined_rooms[2], + src=self.admin, + targ=self.bad_user, + expect_code=200, + tok=self.admin_tok, + ) + + # fetch the joined room count again, the number should remain the same as the collected joined rooms + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.bad_user}/cumulative_joined_room_count?from_ts={int(before_room_creation_timestamp)}", + access_token=self.admin_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + len(joined_rooms), channel.json_body["cumulative_joined_room_count"] + ) From 3d60a58ad6a7db2ff06a06c6b95012466ee586f4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 16 Dec 2024 16:12:40 -0700 Subject: [PATCH 054/691] Add `last_seen_ts` to query user example (#17976) This section could probably do with a lot more editorial attention, but for now this is all there is in terms of documentation. The field is already returned by Synapse: https://github.com/element-hq/synapse/blob/4587decd678300217969f1d2f69b226421a33ced/synapse/handlers/admin.py#L150 `last_seen_ts` was introduced in https://github.com/matrix-org/synapse/pull/16218 --- changelog.d/17976.doc | 1 + docs/admin_api/user_admin_api.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/17976.doc diff --git a/changelog.d/17976.doc b/changelog.d/17976.doc new file mode 100644 index 0000000000..c45ead4cf9 --- /dev/null +++ b/changelog.d/17976.doc @@ -0,0 +1 @@ +Add previously-undocumented `last_seen_ts` to query user admin API. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index c63b7068c5..2742d2d2cd 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -40,6 +40,7 @@ It returns a JSON body like the following: "erased": false, "shadow_banned": 0, "creation_ts": 1560432506, + "last_seen_ts": 1732919539393, "appservice_id": null, "consent_server_notice_sent": null, "consent_version": null, From 57bf44941e52f09dc7ea21acdbe20633b7449f5a Mon Sep 17 00:00:00 2001 From: V02460 Date: Tue, 17 Dec 2024 01:01:33 +0100 Subject: [PATCH 055/691] Add `macaroon_secret_key_path` config option (#17983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Another config option on my quest to a `*_path` variant for every secret. This time it’s `macaroon_secret_key_path`. Reading secrets from files has the security advantage of separating the secrets from the config. It also simplifies secrets management in Kubernetes. Also useful to NixOS users. --- changelog.d/17983.feature | 1 + .../configuration/config_documentation.md | 16 +++++++++++++ synapse/config/key.py | 21 +++++++++++++---- tests/config/test_load.py | 23 ++++++++++++------- tests/config/utils.py | 5 ++-- 5 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 changelog.d/17983.feature diff --git a/changelog.d/17983.feature b/changelog.d/17983.feature new file mode 100644 index 0000000000..2c54c80c40 --- /dev/null +++ b/changelog.d/17983.feature @@ -0,0 +1 @@ +Add `macaroon_secret_key_path` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7a48d76bbb..98ceb878e2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3091,6 +3091,22 @@ Example configuration: ```yaml macaroon_secret_key: ``` +--- +### `macaroon_secret_key_path` + +An alternative to [`macaroon_secret_key`](#macaroon_secret_key): +allows the secret key to be specified in an external file. + +The file should be a plain text file, containing only the secret key. +Synapse reads the secret key from the given file once at startup. + +Example configuration: +```yaml +macaroon_secret_key_path: /path/to/secrets/file +``` + +_Added in Synapse 1.121.0._ + --- ### `form_secret` diff --git a/synapse/config/key.py b/synapse/config/key.py index bc96888967..01aae09c13 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -43,7 +43,7 @@ from unpaddedbase64 import decode_base64 from synapse.types import JsonDict from synapse.util.stringutils import random_string, random_string_with_symbols -from ._base import Config, ConfigError +from ._base import Config, ConfigError, read_file if TYPE_CHECKING: from signedjson.key import VerifyKeyWithExpiry @@ -91,6 +91,11 @@ To suppress this warning and continue using 'matrix.org', admins should set 'suppress_key_server_warning' to 'true' in homeserver.yaml. --------------------------------------------------------------------------------""" +CONFLICTING_MACAROON_SECRET_KEY_OPTS_ERROR = """\ +Conflicting options 'macaroon_secret_key' and 'macaroon_secret_key_path' are +both defined in config file. +""" + logger = logging.getLogger(__name__) @@ -166,10 +171,16 @@ class KeyConfig(Config): ) ) - macaroon_secret_key: Optional[str] = config.get( - "macaroon_secret_key", self.root.registration.registration_shared_secret - ) - + macaroon_secret_key = config.get("macaroon_secret_key") + macaroon_secret_key_path = config.get("macaroon_secret_key_path") + if macaroon_secret_key_path: + if macaroon_secret_key: + raise ConfigError(CONFLICTING_MACAROON_SECRET_KEY_OPTS_ERROR) + macaroon_secret_key = read_file( + macaroon_secret_key_path, "macaroon_secret_key_path" + ).strip() + if not macaroon_secret_key: + macaroon_secret_key = self.root.registration.registration_shared_secret if not macaroon_secret_key: # Unfortunately, there are people out there that don't have this # set. Lets just be "nice" and derive one from their secret key. diff --git a/tests/config/test_load.py b/tests/config/test_load.py index c5dee06af5..f8f7b72e40 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -39,7 +39,7 @@ except ImportError: class ConfigLoadingFileTestCase(ConfigFileTestCase): def test_load_fails_if_server_name_missing(self) -> None: - self.generate_config_and_remove_lines_containing("server_name") + self.generate_config_and_remove_lines_containing(["server_name"]) with self.assertRaises(ConfigError): HomeServerConfig.load_config("", ["-c", self.config_file]) with self.assertRaises(ConfigError): @@ -76,7 +76,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): ) def test_load_succeeds_if_macaroon_secret_key_missing(self) -> None: - self.generate_config_and_remove_lines_containing("macaroon") + self.generate_config_and_remove_lines_containing(["macaroon"]) config1 = HomeServerConfig.load_config("", ["-c", self.config_file]) config2 = HomeServerConfig.load_config("", ["-c", self.config_file]) config3 = HomeServerConfig.load_or_generate_config("", ["-c", self.config_file]) @@ -111,7 +111,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): self.assertTrue(config3.registration.enable_registration) def test_stats_enabled(self) -> None: - self.generate_config_and_remove_lines_containing("enable_metrics") + self.generate_config_and_remove_lines_containing(["enable_metrics"]) self.add_lines_to_config(["enable_metrics: true"]) # The default Metrics Flags are off by default. @@ -131,6 +131,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): [ "turn_shared_secret_path: /does/not/exist", "registration_shared_secret_path: /does/not/exist", + "macaroon_secret_key_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] * (hiredis is not None), ] @@ -146,16 +147,20 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): [ ( "turn_shared_secret_path: {}", - lambda c: c.voip.turn_shared_secret, + lambda c: c.voip.turn_shared_secret.encode("utf-8"), ), ( "registration_shared_secret_path: {}", - lambda c: c.registration.registration_shared_secret, + lambda c: c.registration.registration_shared_secret.encode("utf-8"), + ), + ( + "macaroon_secret_key_path: {}", + lambda c: c.key.macaroon_secret_key, ), *[ ( "redis:\n enabled: true\n password_path: {}", - lambda c: c.redis.redis_password, + lambda c: c.redis.redis_password.encode("utf-8"), ) ] * (hiredis is not None), @@ -164,11 +169,13 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): def test_secret_files_existing( self, config_line: str, get_secret: Callable[[RootConfig], str] ) -> None: - self.generate_config_and_remove_lines_containing("registration_shared_secret") + self.generate_config_and_remove_lines_containing( + ["registration_shared_secret", "macaroon_secret_key"] + ) with tempfile.NamedTemporaryFile(buffering=0) as secret_file: secret_file.write(b"53C237") self.add_lines_to_config(["", config_line.format(secret_file.name)]) config = HomeServerConfig.load_config("", ["-c", self.config_file]) - self.assertEqual(get_secret(config), "53C237") + self.assertEqual(get_secret(config), b"53C237") diff --git a/tests/config/utils.py b/tests/config/utils.py index 11140ff979..3cba4ac588 100644 --- a/tests/config/utils.py +++ b/tests/config/utils.py @@ -51,12 +51,13 @@ class ConfigFileTestCase(unittest.TestCase): ], ) - def generate_config_and_remove_lines_containing(self, needle: str) -> None: + def generate_config_and_remove_lines_containing(self, needles: list[str]) -> None: self.generate_config() with open(self.config_file) as f: contents = f.readlines() - contents = [line for line in contents if needle not in line] + for needle in needles: + contents = [line for line in contents if needle not in line] with open(self.config_file, "w") as f: f.write("".join(contents)) From f1ecf4664780068056dc7ede16d3c2ac4f20b909 Mon Sep 17 00:00:00 2001 From: cynhr Date: Wed, 18 Dec 2024 01:05:38 +0100 Subject: [PATCH 056/691] Add email.tlsname config option (#17849) The existing `email.smtp_host` config option is used for two distinct purposes: it is resolved into the IP address to connect to, and used to (request via SNI and) validate the server's certificate if TLS is enabled. This new option allows specifying a different name for the second purpose. This is especially helpful, if `email.smtp_host` isn't a global FQDN, but something that resolves only locally (e.g. "localhost" to connect through the loopback interface, or some other internally routed name), that one cannot get a valid certificate for. Alternatives would of course be to specify a global FQDN as `email.smtp_host`, or to disable TLS entirely, both of which might be undesirable, depending on the SMTP server configuration. --- changelog.d/17849.feature | 1 + .../configuration/config_documentation.md | 4 +- synapse/config/emailconfig.py | 1 + synapse/handlers/send_email.py | 91 ++++++++++++------- tests/handlers/test_send_email.py | 8 +- 5 files changed, 68 insertions(+), 37 deletions(-) create mode 100644 changelog.d/17849.feature diff --git a/changelog.d/17849.feature b/changelog.d/17849.feature new file mode 100644 index 0000000000..4de580f9ed --- /dev/null +++ b/changelog.d/17849.feature @@ -0,0 +1 @@ +Added the `email.tlsname` config option. This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 98ceb878e2..74522a1e4e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -673,8 +673,9 @@ This setting has the following sub-options: TLS via STARTTLS *if the SMTP server supports it*. If this option is set, Synapse will refuse to connect unless the server supports STARTTLS. * `enable_tls`: By default, if the server supports TLS, it will be used, and the server - must present a certificate that is valid for 'smtp_host'. If this option + must present a certificate that is valid for `tlsname`. If this option is set to false, TLS will not be used. +* `tlsname`: The domain name the SMTP server's TLS certificate must be valid for, defaulting to `smtp_host`. * `notif_from`: defines the "From" address to use when sending emails. It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, which is normally set in `app_name`, but may be overridden by the @@ -741,6 +742,7 @@ email: force_tls: true require_transport_security: true enable_tls: false + tlsname: mail.server.example.com notif_from: "Your Friendly %(app)s homeserver " app_name: my_branded_matrix_server enable_notifs: true diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 8033fa2e52..c3a3e05a82 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -110,6 +110,7 @@ class EmailConfig(Config): raise ConfigError( "email.require_transport_security requires email.enable_tls to be true" ) + self.email_tlsname = email_config.get("tlsname", None) if "app_name" in email_config: self.email_app_name = email_config["app_name"] diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 70cdb0721c..8cf8d2bada 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -47,15 +47,45 @@ logger = logging.getLogger(__name__) _is_old_twisted = parse_version(twisted.__version__) < parse_version("21") -class _NoTLSESMTPSender(ESMTPSender): - """Extend ESMTPSender to disable TLS +class _BackportESMTPSender(ESMTPSender): + """Extend old versions of ESMTPSender to configure TLS. - Unfortunately, before Twisted 21.2, ESMTPSender doesn't give an easy way to disable - TLS, so we override its internal method which it uses to generate a context factory. + Unfortunately, before Twisted 21.2, ESMTPSender doesn't give an easy way to + disable TLS, or to configure the hostname used for TLS certificate validation. + This backports the `hostname` parameter for that functionality. """ + __hostname: Optional[str] + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """""" + self.__hostname = kwargs.pop("hostname", None) + super().__init__(*args, **kwargs) + def _getContextFactory(self) -> Optional[IOpenSSLContextFactory]: - return None + if self.context is not None: + return self.context + elif self.__hostname is None: + return None # disable TLS if hostname is None + return optionsForClientTLS(self.__hostname) + + +class _BackportESMTPSenderFactory(ESMTPSenderFactory): + """An ESMTPSenderFactory for _BackportESMTPSender. + + This backports the `hostname` parameter, to disable or configure TLS. + """ + + __hostname: Optional[str] + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.__hostname = kwargs.pop("hostname", None) + super().__init__(*args, **kwargs) + + def protocol(self, *args: Any, **kwargs: Any) -> ESMTPSender: # type: ignore + # this overrides ESMTPSenderFactory's `protocol` attribute, with a Callable + # instantiating our _BackportESMTPSender, providing the hostname parameter + return _BackportESMTPSender(*args, **kwargs, hostname=self.__hostname) async def _sendmail( @@ -71,6 +101,7 @@ async def _sendmail( require_tls: bool = False, enable_tls: bool = True, force_tls: bool = False, + tlsname: Optional[str] = None, ) -> None: """A simple wrapper around ESMTPSenderFactory, to allow substitution in tests @@ -88,39 +119,33 @@ async def _sendmail( enable_tls: True to enable STARTTLS. If this is False and require_tls is True, the request will fail. force_tls: True to enable Implicit TLS. + tlsname: the domain name expected as the TLS certificate's commonname, + defaults to smtphost. """ msg = BytesIO(msg_bytes) d: "Deferred[object]" = Deferred() + if not enable_tls: + tlsname = None + elif tlsname is None: + tlsname = smtphost - def build_sender_factory(**kwargs: Any) -> ESMTPSenderFactory: - return ESMTPSenderFactory( - username, - password, - from_addr, - to_addr, - msg, - d, - heloFallback=True, - requireAuthentication=require_auth, - requireTransportSecurity=require_tls, - **kwargs, - ) - - factory: IProtocolFactory - if _is_old_twisted: - # before twisted 21.2, we have to override the ESMTPSender protocol to disable - # TLS - factory = build_sender_factory() - - if not enable_tls: - factory.protocol = _NoTLSESMTPSender - else: - # for twisted 21.2 and later, there is a 'hostname' parameter which we should - # set to enable TLS. - factory = build_sender_factory(hostname=smtphost if enable_tls else None) + factory: IProtocolFactory = ( + _BackportESMTPSenderFactory if _is_old_twisted else ESMTPSenderFactory + )( + username, + password, + from_addr, + to_addr, + msg, + d, + heloFallback=True, + requireAuthentication=require_auth, + requireTransportSecurity=require_tls, + hostname=tlsname, + ) if force_tls: - factory = TLSMemoryBIOFactory(optionsForClientTLS(smtphost), True, factory) + factory = TLSMemoryBIOFactory(optionsForClientTLS(tlsname), True, factory) endpoint = HostnameEndpoint( reactor, smtphost, smtpport, timeout=30, bindAddress=None @@ -148,6 +173,7 @@ class SendEmailHandler: self._require_transport_security = hs.config.email.require_transport_security self._enable_tls = hs.config.email.enable_smtp_tls self._force_tls = hs.config.email.force_tls + self._tlsname = hs.config.email.email_tlsname self._sendmail = _sendmail @@ -227,4 +253,5 @@ class SendEmailHandler: require_tls=self._require_transport_security, enable_tls=self._enable_tls, force_tls=self._force_tls, + tlsname=self._tlsname, ) diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py index cedcea27d9..5f7839c82c 100644 --- a/tests/handlers/test_send_email.py +++ b/tests/handlers/test_send_email.py @@ -163,6 +163,7 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): "email": { "notif_from": "noreply@test", "force_tls": True, + "tlsname": "example.org", }, } ) @@ -186,10 +187,9 @@ class SendEmailHandlerTestCaseIPv4(HomeserverTestCase): self.assertEqual(host, self.reactor.lookups["localhost"]) self.assertEqual(port, 465) # We need to make sure that TLS is happenning - self.assertIsInstance( - client_factory._wrappedFactory._testingContextFactory, - ClientTLSOptions, - ) + context_factory = client_factory._wrappedFactory._testingContextFactory + self.assertIsInstance(context_factory, ClientTLSOptions) + self.assertEqual(context_factory._hostname, "example.org") # tlsname # And since we use endpoints, they go through reactor.connectTCP # which works differently to connectSSL on the testing reactor From f1b0f9a4ef4507fe32c0ac6396dfb2aa6c5317c0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:42:17 +0000 Subject: [PATCH 057/691] Bump mypy from 1.11.2 to 1.12.1 and fix new typechecking errors (#17999) Supersedes https://github.com/element-hq/synapse/pull/17958. Awkwardly, the changes made to fix the mypy errors in 1.12.1 cause errors in 1.11.2. So you'll need to update your mypy version to 1.12.1 to eliminate typechecking errors during developing. --- changelog.d/17999.misc | 1 + poetry.lock | 65 +++++++++++++----------- synapse/config/repository.py | 2 +- synapse/handlers/appservice.py | 8 +-- synapse/push/bulk_push_rule_evaluator.py | 8 +-- synapse/rest/client/room.py | 4 +- tests/replication/http/test__base.py | 4 +- 7 files changed, 49 insertions(+), 43 deletions(-) create mode 100644 changelog.d/17999.misc diff --git a/changelog.d/17999.misc b/changelog.d/17999.misc new file mode 100644 index 0000000000..9cabc5b15d --- /dev/null +++ b/changelog.d/17999.misc @@ -0,0 +1 @@ +Bump mypy from 1.11.2 to 1.12.1. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index c4b34c17f6..a0804b9b69 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1314,38 +1314,43 @@ files = [ [[package]] name = "mypy" -version = "1.11.2" +version = "1.12.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, - {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, - {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, - {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, - {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, - {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, - {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, - {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, - {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, - {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, - {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, - {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, - {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, - {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, - {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, - {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, - {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, - {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, - {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, + {file = "mypy-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3d7d4371829184e22fda4015278fbfdef0327a4b955a483012bd2d423a788801"}, + {file = "mypy-1.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f59f1dfbf497d473201356966e353ef09d4daec48caeacc0254db8ef633a28a5"}, + {file = "mypy-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b947097fae68004b8328c55161ac9db7d3566abfef72d9d41b47a021c2fba6b1"}, + {file = "mypy-1.12.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96af62050971c5241afb4701c15189ea9507db89ad07794a4ee7b4e092dc0627"}, + {file = "mypy-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:d90da248f4c2dba6c44ddcfea94bb361e491962f05f41990ff24dbd09969ce20"}, + {file = "mypy-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1230048fec1380faf240be6385e709c8570604d2d27ec6ca7e573e3bc09c3735"}, + {file = "mypy-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:02dcfe270c6ea13338210908f8cadc8d31af0f04cee8ca996438fe6a97b4ec66"}, + {file = "mypy-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5a437c9102a6a252d9e3a63edc191a3aed5f2fcb786d614722ee3f4472e33f6"}, + {file = "mypy-1.12.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:186e0c8346efc027ee1f9acf5ca734425fc4f7dc2b60144f0fbe27cc19dc7931"}, + {file = "mypy-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:673ba1140a478b50e6d265c03391702fa11a5c5aff3f54d69a62a48da32cb811"}, + {file = "mypy-1.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9fb83a7be97c498176fb7486cafbb81decccaef1ac339d837c377b0ce3743a7f"}, + {file = "mypy-1.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:389e307e333879c571029d5b93932cf838b811d3f5395ed1ad05086b52148fb0"}, + {file = "mypy-1.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b2048a95a21f7a9ebc9fbd075a4fcd310410d078aa0228dbbad7f71335e042"}, + {file = "mypy-1.12.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ee5932370ccf7ebf83f79d1c157a5929d7ea36313027b0d70a488493dc1b179"}, + {file = "mypy-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:19bf51f87a295e7ab2894f1d8167622b063492d754e69c3c2fed6563268cb42a"}, + {file = "mypy-1.12.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d34167d43613ffb1d6c6cdc0cc043bb106cac0aa5d6a4171f77ab92a3c758bcc"}, + {file = "mypy-1.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:427878aa54f2e2c5d8db31fa9010c599ed9f994b3b49e64ae9cd9990c40bd635"}, + {file = "mypy-1.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5fcde63ea2c9f69d6be859a1e6dd35955e87fa81de95bc240143cf00de1f7f81"}, + {file = "mypy-1.12.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d54d840f6c052929f4a3d2aab2066af0f45a020b085fe0e40d4583db52aab4e4"}, + {file = "mypy-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:20db6eb1ca3d1de8ece00033b12f793f1ea9da767334b7e8c626a4872090cf02"}, + {file = "mypy-1.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b16fe09f9c741d85a2e3b14a5257a27a4f4886c171d562bc5a5e90d8591906b8"}, + {file = "mypy-1.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dcc1e843d58f444fce19da4cce5bd35c282d4bde232acdeca8279523087088a"}, + {file = "mypy-1.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e10ba7de5c616e44ad21005fa13450cd0de7caaa303a626147d45307492e4f2d"}, + {file = "mypy-1.12.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e6fe449223fa59fbee351db32283838a8fee8059e0028e9e6494a03802b4004"}, + {file = "mypy-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:dc6e2a2195a290a7fd5bac3e60b586d77fc88e986eba7feced8b778c373f9afe"}, + {file = "mypy-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:de5b2a8988b4e1269a98beaf0e7cc71b510d050dce80c343b53b4955fff45f19"}, + {file = "mypy-1.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843826966f1d65925e8b50d2b483065c51fc16dc5d72647e0236aae51dc8d77e"}, + {file = "mypy-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fe20f89da41a95e14c34b1ddb09c80262edcc295ad891f22cc4b60013e8f78d"}, + {file = "mypy-1.12.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8135ffec02121a75f75dc97c81af7c14aa4ae0dda277132cfcd6abcd21551bfd"}, + {file = "mypy-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:a7b76fa83260824300cc4834a3ab93180db19876bce59af921467fd03e692810"}, + {file = "mypy-1.12.1-py3-none-any.whl", hash = "sha256:ce561a09e3bb9863ab77edf29ae3a50e65685ad74bba1431278185b7e5d5486e"}, + {file = "mypy-1.12.1.tar.gz", hash = "sha256:f5b3936f7a6d0e8280c9bdef94c7ce4847f5cdfc258fbb2c29a8c1711e8bb96d"}, ] [package.dependencies] @@ -3133,4 +3138,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.0" python-versions = "^3.9.0" -content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d" +content-hash = "170c8bc97d4cd16a38a98ed9d48bef0ef09c994d8b129160ef262306cf689db7" diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 27860154e1..fc5a90c85a 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,7 +22,7 @@ import logging import os from typing import Any, Dict, List, Tuple -from urllib.request import getproxies_environment # type: ignore +from urllib.request import getproxies_environment import attr diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 4b33e1330d..b7d1033351 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -896,10 +896,10 @@ class ApplicationServicesHandler: results = await make_deferred_yieldable( defer.DeferredList( [ - run_in_background( + run_in_background( # type: ignore[call-overload] self.appservice_api.claim_client_keys, # We know this must be an app service. - self.store.get_app_service_by_id(service_id), # type: ignore[arg-type] + self.store.get_app_service_by_id(service_id), service_query, ) for service_id, service_query in query_by_appservice.items() @@ -952,10 +952,10 @@ class ApplicationServicesHandler: results = await make_deferred_yieldable( defer.DeferredList( [ - run_in_background( + run_in_background( # type: ignore[call-overload] self.appservice_api.query_keys, # We know this must be an app service. - self.store.get_app_service_by_id(service_id), # type: ignore[arg-type] + self.store.get_app_service_by_id(service_id), service_query, ) for service_id, service_query in query_by_appservice.items() diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 9c0592a902..8c106f9649 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -371,7 +371,7 @@ class BulkPushRuleEvaluator: "Deferred[Tuple[int, Tuple[dict, Optional[int]], Dict[str, Dict[str, JsonValue]], Mapping[str, ProfileInfo]]]", gather_results( ( - run_in_background( # type: ignore[call-arg] + run_in_background( # type: ignore[call-overload] self.store.get_number_joined_users_in_room, event.room_id, # type: ignore[arg-type] ), @@ -382,10 +382,10 @@ class BulkPushRuleEvaluator: event_id_to_event, ), run_in_background(self._related_events, event), - run_in_background( # type: ignore[call-arg] + run_in_background( # type: ignore[call-overload] self.store.get_subset_users_in_room_with_profiles, - event.room_id, # type: ignore[arg-type] - rules_by_user.keys(), # type: ignore[arg-type] + event.room_id, + rules_by_user.keys(), ), ), consumeErrors=True, diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 8883cd6bc0..03e7bc0a24 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -783,9 +783,9 @@ class RoomMessageListRestServlet(RestServlet): # decorator on `get_number_joined_users_in_room` doesn't play well with # the type system. Maybe in the future, it can use some ParamSpec # wizardry to fix it up. - room_member_count_deferred = run_in_background( # type: ignore[call-arg] + room_member_count_deferred = run_in_background( # type: ignore[call-overload] self.store.get_number_joined_users_in_room, - room_id, # type: ignore[arg-type] + room_id, ) requester = await self.auth.get_user_by_req(request, allow_guest=True) diff --git a/tests/replication/http/test__base.py b/tests/replication/http/test__base.py index 2eaad3707a..31d3163c01 100644 --- a/tests/replication/http/test__base.py +++ b/tests/replication/http/test__base.py @@ -46,7 +46,7 @@ class CancellableReplicationEndpoint(ReplicationEndpoint): self.clock = hs.get_clock() @staticmethod - async def _serialize_payload() -> JsonDict: + async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: return {} @cancellable @@ -68,7 +68,7 @@ class UncancellableReplicationEndpoint(ReplicationEndpoint): self.clock = hs.get_clock() @staticmethod - async def _serialize_payload() -> JsonDict: + async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: return {} async def _handle_request( # type: ignore[override] From 09f377fa5240f666c958f6634f443b1d298d4a70 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:42:34 +0000 Subject: [PATCH 058/691] Wording improvements for the `TaskScheduler` (#17992) As I found the current docstrings a bit unclear while trying to wrap my head around this class. --- changelog.d/17992.doc | 1 + synapse/util/task_scheduler.py | 80 ++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 changelog.d/17992.doc diff --git a/changelog.d/17992.doc b/changelog.d/17992.doc new file mode 100644 index 0000000000..74afabe40f --- /dev/null +++ b/changelog.d/17992.doc @@ -0,0 +1 @@ +Improve documentation for the `TaskScheduler` class. \ No newline at end of file diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 3ed457bd30..4683d09cd7 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -46,33 +46,43 @@ logger = logging.getLogger(__name__) class TaskScheduler: """ - This is a simple task sheduler aimed at resumable tasks: usually we use `run_in_background` - to launch a background task, or Twisted `deferLater` if we want to do so later on. + This is a simple task scheduler designed for resumable tasks. Normally, + you'd use `run_in_background` to start a background task or Twisted's + `deferLater` if you want to run it later. - The problem with that is that the tasks will just stop and never be resumed if synapse - is stopped for whatever reason. + The issue is that these tasks stop completely and won't resume if Synapse is + shut down for any reason. - How this works: - - A function mapped to a named action should first be registered with `register_action`. - This function will be called when trying to resuming tasks after a synapse shutdown, - so this registration should happen when synapse is initialised, NOT right before scheduling - a task. - - A task can then be launched using this named action with `schedule_task`. A `params` dict - can be passed, and it will be available to the registered function when launched. This task - can be launch either now-ish, or later on by giving a `timestamp` parameter. + Here's how it works: - The function may call `update_task` at any time to update the `result` of the task, - and this can be used to resume the task at a specific point and/or to convey a result to - the code launching the task. - You can also specify the `result` (and/or an `error`) when returning from the function. + - Register an Action: First, you need to register a function to a named + action using `register_action`. This function will be called to resume tasks + after a Synapse shutdown. Make sure to register it when Synapse initializes, + not right before scheduling the task. - The reconciliation loop runs every minute, so this is not a precise scheduler. - There is a limit of 10 concurrent tasks, so tasks may be delayed if the pool is already - full. In this regard, please take great care that scheduled tasks can actually finished. - For now there is no mechanism to stop a running task if it is stuck. + - Schedule a Task: You can launch a task linked to the named action + using `schedule_task`. You can pass a `params` dictionary, which will be + passed to the registered function when it's executed. Tasks can be scheduled + to run either immediately or later by specifying a `timestamp`. - Tasks will be run on the worker specified with `run_background_tasks_on` config, - or the main one by default. + - Update Task: The function handling the task can call `update_task` at + any point to update the task's `result`. This lets you resume the task from + a specific point or pass results back to the code that scheduled it. When + the function completes, you can also return a `result` or an `error`. + + Things to keep in mind: + + - The reconciliation loop runs every minute, so this is not a high-precision + scheduler. + + - Only 10 tasks can run at the same time. If the pool is full, tasks may be + delayed. Make sure your scheduled tasks can actually finish. + + - Currently, there's no way to stop a task if it gets stuck. + + - Tasks will run on the worker defined by the `run_background_tasks_on` + setting in your configuration. If no worker is specified, they'll run on + the main one by default. """ # Precision of the scheduler, evaluation of tasks to run will only happen @@ -157,7 +167,7 @@ class TaskScheduler: params: Optional[JsonMapping] = None, ) -> str: """Schedule a new potentially resumable task. A function matching the specified - `action` should have be registered with `register_action` before the task is run. + `action` should've been registered with `register_action` before the task is run. Args: action: the name of a previously registered action @@ -210,15 +220,15 @@ class TaskScheduler: result: Optional[JsonMapping] = None, error: Optional[str] = None, ) -> bool: - """Update some task associated values. This is exposed publicly so it can - be used inside task functions, mainly to update the result and be able to - resume a task at a specific step after a restart of synapse. + """Update some task-associated values. This is exposed publicly so it can + be used inside task functions, mainly to update the result or resume + a task at a specific step after a restart of synapse. It can also be used to stage a task, by setting the `status` to `SCHEDULED` with a new timestamp. - The `status` can only be set to `ACTIVE` or `SCHEDULED`, `COMPLETE` and `FAILED` - are terminal status and can only be set by returning it in the function. + The `status` can only be set to `ACTIVE` or `SCHEDULED`. `COMPLETE` and `FAILED` + are terminal statuses and can only be set by returning them from the function. Args: id: the id of the task to update @@ -226,6 +236,12 @@ class TaskScheduler: status: the new `TaskStatus` of the task result: the new result of the task error: the new error of the task + + Returns: + True if the update was successful, False otherwise. + + Raises: + Exception: If a status other than `ACTIVE` or `SCHEDULED` was passed. """ if status == TaskStatus.COMPLETE or status == TaskStatus.FAILED: raise Exception( @@ -263,9 +279,9 @@ class TaskScheduler: max_timestamp: Optional[int] = None, limit: Optional[int] = None, ) -> List[ScheduledTask]: - """Get a list of tasks. Returns all the tasks if no args is provided. + """Get a list of tasks. Returns all the tasks if no args are provided. - If an arg is `None` all tasks matching the other args will be selected. + If an arg is `None`, all tasks matching the other args will be selected. If an arg is an empty list, the corresponding value of the task needs to be `None` to be selected. @@ -277,8 +293,8 @@ class TaskScheduler: a timestamp inferior to the specified one limit: Only return `limit` number of rows if set. - Returns - A list of `ScheduledTask`, ordered by increasing timestamps + Returns: + A list of `ScheduledTask`, ordered by increasing timestamps. """ return await self._store.get_scheduled_tasks( actions=actions, From 3eb92369ca14012a07da2fbf9250e66f66afb710 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:49:38 +0000 Subject: [PATCH 059/691] Fix mypy errors on Twisted 24.11.0 (#17998) Fixes various `mypy` errors associated with Twisted `24.11.0`. Hopefully addresses https://github.com/element-hq/synapse/issues/17075, though I've yet to test against `trunk`. Changes should be compatible with our currently pinned Twisted version of `24.7.0`. --- changelog.d/17998.misc | 1 + synapse/http/client.py | 7 ++++--- synapse/http/proxyagent.py | 13 +++++++++---- synapse/http/replicationagent.py | 4 +++- tests/http/test_proxyagent.py | 6 +++--- 5 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 changelog.d/17998.misc diff --git a/changelog.d/17998.misc b/changelog.d/17998.misc new file mode 100644 index 0000000000..5ddd54cd08 --- /dev/null +++ b/changelog.d/17998.misc @@ -0,0 +1 @@ +Fix various type errors across the codebase. \ No newline at end of file diff --git a/synapse/http/client.py b/synapse/http/client.py index 85923d956b..559b1febf0 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -41,7 +41,7 @@ from canonicaljson import encode_canonical_json from netaddr import AddrFormatError, IPAddress, IPSet from prometheus_client import Counter from typing_extensions import Protocol -from zope.interface import implementer, provider +from zope.interface import implementer from OpenSSL import SSL from OpenSSL.SSL import VERIFY_NONE @@ -225,7 +225,7 @@ class _IPBlockingResolver: recv.addressResolved(address) recv.resolutionComplete() - @provider(IResolutionReceiver) + @implementer(IResolutionReceiver) class EndpointReceiver: @staticmethod def resolutionBegan(resolutionInProgress: IHostResolution) -> None: @@ -239,8 +239,9 @@ class _IPBlockingResolver: def resolutionComplete() -> None: _callback() + endpoint_receiver_wrapper = EndpointReceiver() self._reactor.nameResolver.resolveHostName( - EndpointReceiver, hostname, portNumber=portNumber + endpoint_receiver_wrapper, hostname, portNumber=portNumber ) return recv diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index f80f67acc6..c91cf30fd1 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -21,7 +21,7 @@ import logging import random import re -from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple +from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple, Union from urllib.parse import urlparse from urllib.request import ( # type: ignore[attr-defined] getproxies_environment, @@ -351,7 +351,9 @@ def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], - **kwargs: object, + timeout: float = 30, + bindAddress: Optional[Union[bytes, str, tuple[Union[bytes, str], int]]] = None, + attemptDelay: Optional[float] = None, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy @@ -382,12 +384,15 @@ def http_proxy_endpoint( # 3.9+) on scheme-less proxies, e.g. host:port. scheme, host, port, credentials = parse_proxy(proxy) - proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs) + proxy_endpoint = HostnameEndpoint( + reactor, host, port, timeout, bindAddress, attemptDelay + ) if scheme == b"https": if tls_options_factory: tls_options = tls_options_factory.creatorForNetloc(host, port) - proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) + wrapped_proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) + return wrapped_proxy_endpoint, credentials else: raise RuntimeError( f"No TLS options for a https connection via proxy {proxy!s}" diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index ee8c707062..4eabbc8af9 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -89,7 +89,7 @@ class ReplicationEndpointFactory: location_config.port, ) if scheme == "https": - endpoint = wrapClientTLS( + wrapped_endpoint = wrapClientTLS( # The 'port' argument below isn't actually used by the function self.context_factory.creatorForNetloc( location_config.host.encode("utf-8"), @@ -97,6 +97,8 @@ class ReplicationEndpointFactory: ), endpoint, ) + return wrapped_endpoint + return endpoint elif isinstance(location_config, InstanceUnixLocationConfig): return UNIXClientEndpoint(self.reactor, location_config.path) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index f71e4c2b8f..80b0856a56 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -854,7 +854,7 @@ class MatrixFederationAgentTests(TestCase): def test_proxy_with_no_scheme(self) -> None: http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._hostStr, "proxy.com") + self.assertEqual(proxy_ep._hostText, "proxy.com") self.assertEqual(proxy_ep._port, 8888) @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"}) @@ -866,14 +866,14 @@ class MatrixFederationAgentTests(TestCase): def test_proxy_with_http_scheme(self) -> None: http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._hostStr, "proxy.com") + self.assertEqual(proxy_ep._hostText, "proxy.com") self.assertEqual(proxy_ep._port, 8888) @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"}) def test_proxy_with_https_scheme(self) -> None: https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) proxy_ep = checked_cast(_WrapperEndpoint, https_proxy_agent.http_proxy_endpoint) - self.assertEqual(proxy_ep._wrappedEndpoint._hostStr, "proxy.com") + self.assertEqual(proxy_ep._wrappedEndpoint._hostText, "proxy.com") self.assertEqual(proxy_ep._wrappedEndpoint._port, 8888) From bd9a1079bc6fe2d0a2c75acce1771324a52e0ab7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 19 Dec 2024 08:00:50 -0600 Subject: [PATCH 060/691] Update reverse proxy docs with what we've learned from #17986 (#17994) Update reverse proxy docs with what we've learned from https://github.com/element-hq/synapse/pull/17986 Also vice versa and update our nginx config with what I learned from the reverse proxy docs. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/17994.doc | 1 + docker/conf-workers/nginx.conf.j2 | 3 +++ docs/reverse_proxy.md | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17994.doc diff --git a/changelog.d/17994.doc b/changelog.d/17994.doc new file mode 100644 index 0000000000..54b7cf1000 --- /dev/null +++ b/changelog.d/17994.doc @@ -0,0 +1 @@ +Fix example in reverse proxy docs to include server port. diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2 index c3f9b584d2..95d2f760d2 100644 --- a/docker/conf-workers/nginx.conf.j2 +++ b/docker/conf-workers/nginx.conf.j2 @@ -38,6 +38,9 @@ server { {% if using_unix_sockets %} proxy_pass http://unix:/run/main_public.sock; {% else %} + # note: do not add a path (even a single /) after the port in `proxy_pass`, + # otherwise nginx will canonicalise the URI and cause signature verification + # errors. proxy_pass http://localhost:8080; {% endif %} proxy_set_header X-Forwarded-For $remote_addr; diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 7128af114e..45de2b1f65 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -74,7 +74,7 @@ server { proxy_pass http://localhost:8008; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $host; + proxy_set_header Host $host:$server_port; # Nginx by default only allows file uploads up to 1M in size # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml From 234d07eb09946543546ce63463c3bfd40465e1d6 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Thu, 19 Dec 2024 14:02:06 +0000 Subject: [PATCH 061/691] Disable statement timeout during room purge (#18017) This is already done for `purge_history` but seems to have been forgotten for `purge_room`. --- changelog.d/18017.misc | 1 + synapse/storage/databases/main/purge_events.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/18017.misc diff --git a/changelog.d/18017.misc b/changelog.d/18017.misc new file mode 100644 index 0000000000..6b943a5bed --- /dev/null +++ b/changelog.d/18017.misc @@ -0,0 +1 @@ +Disable DB statement timeout when doing a purge room since it can be quite long. diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index 08244153a3..c195685af8 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -376,6 +376,11 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): (room_id,), ) + if isinstance(self.database_engine, PostgresEngine): + # Disable statement timeouts for this transaction; purging rooms can + # take a while! + txn.execute("SET LOCAL statement_timeout = 0") + # First, fetch all the state groups that should be deleted, before # we delete that information. txn.execute( From 2d23250da719e5b81e5793e525171469958466f7 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 19 Dec 2024 12:02:47 -0500 Subject: [PATCH 062/691] Remove support for PostgreSQL 11 and 12 (#18034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is essentially matrix-org/synapse#14392. I didn't see anything in there about updating sytest or complement. The main driver of this is so that I can use `jsonb_path_exists` in #17488. 😄 --- .ci/scripts/calculate_jobs.py | 2 +- .github/workflows/tests.yml | 2 +- changelog.d/18034.removal | 1 + docs/upgrade.md | 8 ++++++++ synapse/storage/engines/postgres.py | 4 ++-- 5 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18034.removal diff --git a/.ci/scripts/calculate_jobs.py b/.ci/scripts/calculate_jobs.py index ea278173db..5249acdc5d 100755 --- a/.ci/scripts/calculate_jobs.py +++ b/.ci/scripts/calculate_jobs.py @@ -60,7 +60,7 @@ trial_postgres_tests = [ { "python-version": "3.9", "database": "postgres", - "postgres-version": "11", + "postgres-version": "13", "extras": "all", } ] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d91f9c2918..084b08b249 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -581,7 +581,7 @@ jobs: matrix: include: - python-version: "3.9" - postgres-version: "11" + postgres-version: "13" - python-version: "3.13" postgres-version: "17" diff --git a/changelog.d/18034.removal b/changelog.d/18034.removal new file mode 100644 index 0000000000..303b442fd4 --- /dev/null +++ b/changelog.d/18034.removal @@ -0,0 +1 @@ +Remove support for PostgreSQL 11 and 12. Contributed by @clokep. diff --git a/docs/upgrade.md b/docs/upgrade.md index 45e63b0c5d..6c96cb91a3 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.122.0 + +## Dropping support for PostgreSQL 11 and 12 + +In line with our [deprecation policy](deprecation_policy.md), we've dropped +support for PostgreSQL 11 and 12, as they are no longer supported upstream. +This release of Synapse requires PostgreSQL 13+. + # Upgrading to v1.120.0 ## Removal of experimental MSC3886 feature diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 8c8c6d0414..e4cd359201 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -99,8 +99,8 @@ class PostgresEngine( allow_unsafe_locale = self.config.get("allow_unsafe_locale", False) # Are we on a supported PostgreSQL version? - if not allow_outdated_version and self._version < 110000: - raise RuntimeError("Synapse requires PostgreSQL 11 or above.") + if not allow_outdated_version and self._version < 130000: + raise RuntimeError("Synapse requires PostgreSQL 13 or above.") with db_conn.cursor() as txn: txn.execute("SHOW SERVER_ENCODING") From d69c00b5a19ba45645665afa532421b25c74407a Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Fri, 20 Dec 2024 10:57:59 +0000 Subject: [PATCH 063/691] Stop using twisted.internet.defer.returnValue (#18020) `defer.returnValue` was only needed in Python 2; in Python 3, a simple `return` is fine. `twisted.internet.defer.returnValue` is deprecated as of Twisted 24.7.0. Most uses of `returnValue` in synapse were removed a while back; this cleans up some remaining bits. --- changelog.d/18020.misc | 1 + contrib/cmdclient/console.py | 6 +++--- contrib/cmdclient/http.py | 8 ++++---- synapse/logging/scopecontextmanager.py | 19 +------------------ synapse/util/patch_inline_callbacks.py | 4 ++-- 5 files changed, 11 insertions(+), 27 deletions(-) create mode 100644 changelog.d/18020.misc diff --git a/changelog.d/18020.misc b/changelog.d/18020.misc new file mode 100644 index 0000000000..8d2dd883b9 --- /dev/null +++ b/changelog.d/18020.misc @@ -0,0 +1 @@ +Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson. diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py index ca2e72b5e8..9b5d33d2b1 100755 --- a/contrib/cmdclient/console.py +++ b/contrib/cmdclient/console.py @@ -245,7 +245,7 @@ class SynapseCmd(cmd.Cmd): if "flows" not in json_res: print("Failed to find any login flows.") - defer.returnValue(False) + return False flow = json_res["flows"][0] # assume first is the one we want. if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow: @@ -254,8 +254,8 @@ class SynapseCmd(cmd.Cmd): "Unable to login via the command line client. Please visit " "%s to login." % fallback_url ) - defer.returnValue(False) - defer.returnValue(True) + return False + return True def do_emailrequest(self, line): """Requests the association of a third party identifier diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py index e6a10b5f32..54363e4259 100644 --- a/contrib/cmdclient/http.py +++ b/contrib/cmdclient/http.py @@ -78,7 +78,7 @@ class TwistedHttpClient(HttpClient): url, data, headers_dict={"Content-Type": ["application/json"]} ) body = yield readBody(response) - defer.returnValue((response.code, body)) + return response.code, body @defer.inlineCallbacks def get_json(self, url, args=None): @@ -88,7 +88,7 @@ class TwistedHttpClient(HttpClient): url = "%s?%s" % (url, qs) response = yield self._create_get_request(url) body = yield readBody(response) - defer.returnValue(json.loads(body)) + return json.loads(body) def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None): """Wrapper of _create_request to issue a PUT request""" @@ -134,7 +134,7 @@ class TwistedHttpClient(HttpClient): response = yield self._create_request(method, url) body = yield readBody(response) - defer.returnValue(json.loads(body)) + return json.loads(body) @defer.inlineCallbacks def _create_request( @@ -173,7 +173,7 @@ class TwistedHttpClient(HttpClient): if self.verbose: print("Status %s %s" % (response.code, response.phrase)) print(pformat(list(response.headers.getAllRawHeaders()))) - defer.returnValue(response) + return response def sleep(self, seconds): d = defer.Deferred() diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py index 581e6d6411..feaadc4d87 100644 --- a/synapse/logging/scopecontextmanager.py +++ b/synapse/logging/scopecontextmanager.py @@ -20,13 +20,10 @@ # import logging -from types import TracebackType -from typing import Optional, Type +from typing import Optional from opentracing import Scope, ScopeManager, Span -import twisted - from synapse.logging.context import ( LoggingContext, current_context, @@ -112,9 +109,6 @@ class _LogContextScope(Scope): """ A custom opentracing scope, associated with a LogContext - * filters out _DefGen_Return exceptions which arise from calling - `defer.returnValue` in Twisted code - * When the scope is closed, the logcontext's active scope is reset to None. and - if enter_logcontext was set - the logcontext is finished too. """ @@ -146,17 +140,6 @@ class _LogContextScope(Scope): self._finish_on_close = finish_on_close self._enter_logcontext = enter_logcontext - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - if exc_type == twisted.internet.defer._DefGen_Return: - # filter out defer.returnValue() calls - exc_type = value = traceback = None - super().__exit__(exc_type, value, traceback) - def __str__(self) -> str: return f"Scope<{self.span}>" diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py index 56bdf451da..beea4d2888 100644 --- a/synapse/util/patch_inline_callbacks.py +++ b/synapse/util/patch_inline_callbacks.py @@ -162,7 +162,7 @@ def _check_yield_points( d = result.throwExceptionIntoGenerator(gen) else: d = gen.send(result) - except (StopIteration, defer._DefGen_Return) as e: + except StopIteration as e: if current_context() != expected_context: # This happens when the context is lost sometime *after* the # final yield and returning. E.g. we forgot to yield on a @@ -183,7 +183,7 @@ def _check_yield_points( ) ) changes.append(err) - # The `StopIteration` or `_DefGen_Return` contains the return value from the + # The `StopIteration` contains the return value from the # generator. return cast(T, e.value) From 7c2284b2f22def70e46448d035969951a0ce2f4c Mon Sep 17 00:00:00 2001 From: morguldir Date: Mon, 23 Dec 2024 12:19:35 +0100 Subject: [PATCH 064/691] Make admin api redactions use the requester to send the redaction (#18029) --- changelog.d/18029.bugfix | 1 + synapse/handlers/admin.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18029.bugfix diff --git a/changelog.d/18029.bugfix b/changelog.d/18029.bugfix new file mode 100644 index 0000000000..f7036fe9fc --- /dev/null +++ b/changelog.d/18029.bugfix @@ -0,0 +1 @@ +Fix a bug preventing the admin redaction endpoint from working on messages from remote users. diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index d1194545ae..f3e7790d43 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -473,7 +473,7 @@ class AdminHandler: "type": EventTypes.Redaction, "content": {"reason": reason} if reason else {}, "room_id": room, - "sender": user_id, + "sender": requester.user.to_string(), } if room_version.updated_redaction_rules: event_dict["content"]["redacts"] = event.event_id From b1d030a1079779d8441a48accbf66923e3ceb589 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:52:41 +0000 Subject: [PATCH 065/691] Bump serde_json from 1.0.133 to 1.0.134 (#18044) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3487064451..6752854d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,9 +451,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", From 2b59e738ee79287b731b8bd17f263e890d3a4c80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:57:22 +0000 Subject: [PATCH 066/691] Bump authlib from 1.3.2 to 1.4.0 (#18048) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index a0804b9b69..b594727df9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "annotated-types" @@ -32,13 +32,13 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.3.2" +version = "1.4.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"}, - {file = "authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2"}, + {file = "Authlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"}, + {file = "authlib-1.4.0.tar.gz", hash = "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2"}, ] [package.dependencies] @@ -3138,4 +3138,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.0" python-versions = "^3.9.0" -content-hash = "170c8bc97d4cd16a38a98ed9d48bef0ef09c994d8b129160ef262306cf689db7" +content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d" From 66b24d3d009e8d9e8507c5b778bd28ebaa0f674d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:03:10 +0000 Subject: [PATCH 067/691] Bump anyhow from 1.0.94 to 1.0.95 (#18045) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6752854d0c..50481282e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arc-swap" From e5a53819fcaf73bbd20e30310e6b72d5bcbdc543 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:03:55 +0000 Subject: [PATCH 068/691] Bump mypy-zope from 1.0.8 to 1.0.9 (#18047) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index b594727df9..3811d384e4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1377,17 +1377,17 @@ files = [ [[package]] name = "mypy-zope" -version = "1.0.8" +version = "1.0.9" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" files = [ - {file = "mypy_zope-1.0.8-py3-none-any.whl", hash = "sha256:8794a77dae0c7e2f28b8ac48569091310b3ee45bb9d6cd4797dcb837c40f9976"}, - {file = "mypy_zope-1.0.8.tar.gz", hash = "sha256:854303a95aefc4289e8a0796808e002c2c7ecde0a10a8f7b8f48092f94ef9b9f"}, + {file = "mypy_zope-1.0.9-py3-none-any.whl", hash = "sha256:6666c1556891a3cb186137519dbd7a58cb30fb72b2504798cad47b35391921ba"}, + {file = "mypy_zope-1.0.9.tar.gz", hash = "sha256:37d6985dfb05a4c27b35cff47577fd5bad878db4893ddedf54d165f7389a1cdb"}, ] [package.dependencies] -mypy = ">=1.0.0,<1.13.0" +mypy = ">=1.0.0,<1.14.0" "zope.interface" = "*" "zope.schema" = "*" From ebc21a8c67ed7a05af010e329b6402cd41229ebe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:09:51 +0000 Subject: [PATCH 069/691] Bump twine from 5.1.1 to 6.0.1 (#18049) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3811d384e4..b5e5bcbd0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2627,19 +2627,20 @@ docs = ["sphinx (<7.0.0)"] [[package]] name = "twine" -version = "5.1.1" +version = "6.0.1" description = "Collection of utilities for publishing packages on PyPI" optional = false python-versions = ">=3.8" files = [ - {file = "twine-5.1.1-py3-none-any.whl", hash = "sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997"}, - {file = "twine-5.1.1.tar.gz", hash = "sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db"}, + {file = "twine-6.0.1-py3-none-any.whl", hash = "sha256:9c6025b203b51521d53e200f4a08b116dee7500a38591668c6a6033117bdc218"}, + {file = "twine-6.0.1.tar.gz", hash = "sha256:36158b09df5406e1c9c1fb8edb24fc2be387709443e7376689b938531582ee27"}, ] [package.dependencies] -importlib-metadata = ">=3.6" -keyring = ">=15.1" -pkginfo = ">=1.8.1,<1.11" +importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} +keyring = {version = ">=15.1", markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +packaging = "*" +pkginfo = ">=1.8.1" readme-renderer = ">=35.0" requests = ">=2.20" requests-toolbelt = ">=0.8.0,<0.9.0 || >0.9.0" @@ -2647,6 +2648,9 @@ rfc3986 = ">=1.4.0" rich = ">=12.0.0" urllib3 = ">=1.26.0" +[package.extras] +keyring = ["keyring (>=15.1)"] + [[package]] name = "twisted" version = "24.7.0" From b5267678d250ea2f3b020de0c45206341e2da096 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 3 Jan 2025 04:52:42 -0800 Subject: [PATCH 070/691] Add a test to verify remote user messages can be redacted via admin api redaction endpoint if requester is admin in room (#18043) --- changelog.d/18043.bugfix | 1 + tests/rest/admin/test_user.py | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 changelog.d/18043.bugfix diff --git a/changelog.d/18043.bugfix b/changelog.d/18043.bugfix new file mode 100644 index 0000000000..05f82a4487 --- /dev/null +++ b/changelog.d/18043.bugfix @@ -0,0 +1 @@ +Fix a bug preventing the admin redaction endpoint from working on messages from remote users. \ No newline at end of file diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index b517aefd0c..a35a250975 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -60,6 +60,7 @@ from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.test_utils import SMALL_PNG +from tests.test_utils.event_injection import inject_event from tests.unittest import override_config @@ -5408,6 +5409,64 @@ class UserRedactionTestCase(unittest.HomeserverTestCase): # we redacted 6 messages self.assertEqual(len(matches), 6) + def test_redactions_for_remote_user_succeed_with_admin_priv_in_room(self) -> None: + """ + Test that if the admin requester has privileges in a room, redaction requests + succeed for a remote user + """ + + # inject some messages from remote user and collect event ids + original_message_ids = [] + for i in range(5): + event = self.get_success( + inject_event( + self.hs, + room_id=self.rm1, + type="m.room.message", + sender="@remote:remote_server", + content={"msgtype": "m.text", "body": f"nefarious_chatter{i}"}, + ) + ) + original_message_ids.append(event.event_id) + + # send a request to redact a remote user's messages in a room. + # the server admin created this room and has admin privilege in room + channel = self.make_request( + "POST", + "/_synapse/admin/v1/user/@remote:remote_server/redact", + content={"rooms": [self.rm1]}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + id = channel.json_body.get("redact_id") + + # check that there were no failed redactions + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/user/redact_status/{id}", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body.get("status"), "complete") + failed_redactions = channel.json_body.get("failed_redactions") + self.assertEqual(failed_redactions, {}) + + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{self.rm1}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + for event in channel.json_body["chunk"]: + for event_id in original_message_ids: + if event["type"] == "m.room.redaction" and event["redacts"] == event_id: + original_message_ids.remove(event_id) + break + # we originally sent 5 messages so 5 should be redacted + self.assertEqual(len(original_message_ids), 0) + class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): servlets = [ From 6306de8e162479c51032cb8440ced9b125ed0927 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 3 Jan 2025 12:23:29 -0500 Subject: [PATCH 071/691] Refactor get_profile: do not return missing fields. (#18063) Refactor `get_profile` to avoid returning "empty" (`None` / `null`) fields. Currently this is not very important, but will be more useful once #17488 lands. It does update the servlet to use this now which has a minor change in behavior: additional fields served over federation will now be properly sent back to clients. It also adds constants for `avatar_url` / `displayname` although I did not attempt to use it everywhere possible. --- changelog.d/18063.misc | 1 + synapse/api/constants.py | 5 +++++ synapse/handlers/profile.py | 14 +++++++++----- synapse/handlers/sso.py | 9 +++++---- synapse/handlers/user_directory.py | 16 +++++++++++++--- synapse/module_api/__init__.py | 18 ++++++++++-------- synapse/rest/client/profile.py | 9 +-------- 7 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 changelog.d/18063.misc diff --git a/changelog.d/18063.misc b/changelog.d/18063.misc new file mode 100644 index 0000000000..424bf25408 --- /dev/null +++ b/changelog.d/18063.misc @@ -0,0 +1 @@ +Refactor `get_profile` to no longer include fields with a value of `None`. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 1206d1e00f..9806e2b0fe 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -320,3 +320,8 @@ class ApprovalNoticeMedium: class Direction(enum.Enum): BACKWARDS = "b" FORWARDS = "f" + + +class ProfileFields: + DISPLAYNAME: Final = "displayname" + AVATAR_URL: Final = "avatar_url" diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ac4544ca4c..22eedcb54f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -22,6 +22,7 @@ import logging import random from typing import TYPE_CHECKING, List, Optional, Union +from synapse.api.constants import ProfileFields from synapse.api.errors import ( AuthError, Codes, @@ -83,7 +84,7 @@ class ProfileHandler: Returns: A JSON dictionary. For local queries this will include the displayname and avatar_url - fields. For remote queries it may contain arbitrary information. + fields, if set. For remote queries it may contain arbitrary information. """ target_user = UserID.from_string(user_id) @@ -92,10 +93,13 @@ class ProfileHandler: if profileinfo.display_name is None and profileinfo.avatar_url is None: raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) - return { - "displayname": profileinfo.display_name, - "avatar_url": profileinfo.avatar_url, - } + # Do not include display name or avatar if unset. + ret = {} + if profileinfo.display_name is not None: + ret[ProfileFields.DISPLAYNAME] = profileinfo.display_name + if profileinfo.avatar_url is not None: + ret[ProfileFields.AVATAR_URL] = profileinfo.avatar_url + return ret else: try: result = await self.federation.make_query( diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index ee74289b6c..cee2eefbb3 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -43,7 +43,7 @@ from typing_extensions import Protocol from twisted.web.iweb import IRequest from twisted.web.server import Request -from synapse.api.constants import LoginType +from synapse.api.constants import LoginType, ProfileFields from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError from synapse.config.sso import SsoAttributeRequirement from synapse.handlers.device import DeviceHandler @@ -813,9 +813,10 @@ class SsoHandler: # bail if user already has the same avatar profile = await self._profile_handler.get_profile(user_id) - if profile["avatar_url"] is not None: - server_name = profile["avatar_url"].split("/")[-2] - media_id = profile["avatar_url"].split("/")[-1] + if ProfileFields.AVATAR_URL in profile: + avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/") + server_name = avatar_url_parts[-2] + media_id = avatar_url_parts[-1] if self._is_mine_server_name(server_name): media = await self._media_repo.store.get_local_media(media_id) # type: ignore[has-type] if media is not None and upload_name == media.upload_name: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 1281929d38..f88d39b38f 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -26,7 +26,13 @@ from typing import TYPE_CHECKING, List, Optional, Set, Tuple from twisted.internet.interfaces import IDelayedCall import synapse.metrics -from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules, Membership +from synapse.api.constants import ( + EventTypes, + HistoryVisibility, + JoinRules, + Membership, + ProfileFields, +) from synapse.api.errors import Codes, SynapseError from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler from synapse.metrics.background_process_metrics import run_as_background_process @@ -756,6 +762,10 @@ class UserDirectoryHandler(StateDeltasHandler): await self.store.update_profile_in_user_dir( user_id, - display_name=non_null_str_or_none(profile.get("displayname")), - avatar_url=non_null_str_or_none(profile.get("avatar_url")), + display_name=non_null_str_or_none( + profile.get(ProfileFields.DISPLAYNAME) + ), + avatar_url=non_null_str_or_none( + profile.get(ProfileFields.AVATAR_URL) + ), ) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index f6bfd93d3c..2a2f821427 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -45,6 +45,7 @@ from twisted.internet.interfaces import IDelayedCall from twisted.web.resource import Resource from synapse.api import errors +from synapse.api.constants import ProfileFields from synapse.api.errors import SynapseError from synapse.api.presence import UserPresenceState from synapse.config import ConfigError @@ -1086,7 +1087,10 @@ class ModuleApi: content = {} # Set the profile if not already done by the module. - if "avatar_url" not in content or "displayname" not in content: + if ( + ProfileFields.AVATAR_URL not in content + or ProfileFields.DISPLAYNAME not in content + ): try: # Try to fetch the user's profile. profile = await self._hs.get_profile_handler().get_profile( @@ -1095,8 +1099,8 @@ class ModuleApi: except SynapseError as e: # If the profile couldn't be found, use default values. profile = { - "displayname": target_user_id.localpart, - "avatar_url": None, + ProfileFields.DISPLAYNAME: target_user_id.localpart, + ProfileFields.AVATAR_URL: None, } if e.code != 404: @@ -1109,11 +1113,9 @@ class ModuleApi: ) # Set the profile where it needs to be set. - if "avatar_url" not in content: - content["avatar_url"] = profile["avatar_url"] - - if "displayname" not in content: - content["displayname"] = profile["displayname"] + for field_name in [ProfileFields.AVATAR_URL, ProfileFields.DISPLAYNAME]: + if field_name not in content and field_name in profile: + content[field_name] = profile[field_name] event_id, _ = await self._hs.get_room_member_handler().update_membership( requester=requester, diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 7a95b9445d..ef59582865 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -227,14 +227,7 @@ class ProfileRestServlet(RestServlet): user = UserID.from_string(user_id) await self.profile_handler.check_profile_query_allowed(user, requester_user) - displayname = await self.profile_handler.get_displayname(user) - avatar_url = await self.profile_handler.get_avatar_url(user) - - ret = {} - if displayname is not None: - ret["displayname"] = displayname - if avatar_url is not None: - ret["avatar_url"] = avatar_url + ret = await self.profile_handler.get_profile(user_id) return 200, ret From b3ba501c526bf15240bb8cc635858d05cd702c2c Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 6 Jan 2025 16:32:18 +0100 Subject: [PATCH 072/691] Properly purge state groups tables when purging a room (#18024) Currently purging a complex room can lead to a lot of orphaned rows left behind in the state groups tables. It seems it is because we are loosing track of state groups sometimes. This change uses the `room_id` indexed column of `state_groups` table to decide what to delete instead of doing an indirection through `event_to_state_groups`. Related to https://github.com/element-hq/synapse/issues/3364. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Erik Johnston --- changelog.d/18024.bugfix | 1 + synapse/storage/controllers/purge_events.py | 4 +- .../storage/databases/main/purge_events.py | 36 +++-------- synapse/storage/databases/state/store.py | 62 +++++++------------ tests/rest/admin/test_room.py | 2 +- 5 files changed, 35 insertions(+), 70 deletions(-) create mode 100644 changelog.d/18024.bugfix diff --git a/changelog.d/18024.bugfix b/changelog.d/18024.bugfix new file mode 100644 index 0000000000..956f43f036 --- /dev/null +++ b/changelog.d/18024.bugfix @@ -0,0 +1 @@ +Properly purge state groups tables when purging a room with the admin API. diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index e794b370c2..15c04ffef8 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -42,8 +42,8 @@ class PurgeEventsStorageController: """Deletes all record of a room""" with nested_logging_context(room_id): - state_groups_to_delete = await self.stores.main.purge_room(room_id) - await self.stores.state.purge_room_state(room_id, state_groups_to_delete) + await self.stores.main.purge_room(room_id) + await self.stores.state.purge_room_state(room_id) async def purge_history( self, room_id: str, token: str, delete_local_events: bool diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index c195685af8..ebdeb8fbd7 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -20,7 +20,7 @@ # import logging -from typing import Any, List, Set, Tuple, cast +from typing import Any, Set, Tuple, cast from synapse.api.errors import SynapseError from synapse.storage.database import LoggingTransaction @@ -332,7 +332,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): return referenced_state_groups - async def purge_room(self, room_id: str) -> List[int]: + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room Args: @@ -348,7 +348,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # purge any of those rows which were added during the first. logger.info("[purge] Starting initial main purge of [1/2]") - state_groups_to_delete = await self.db_pool.runInteraction( + await self.db_pool.runInteraction( "purge_room", self._purge_room_txn, room_id=room_id, @@ -356,18 +356,15 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): ) logger.info("[purge] Starting secondary main purge of [2/2]") - state_groups_to_delete.extend( - await self.db_pool.runInteraction( - "purge_room", - self._purge_room_txn, - room_id=room_id, - ), + await self.db_pool.runInteraction( + "purge_room", + self._purge_room_txn, + room_id=room_id, ) + logger.info("[purge] Done with main purge") - return state_groups_to_delete - - def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: + def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> None: # This collides with event persistence so we cannot write new events and metadata into # a room while deleting it or this transaction will fail. if isinstance(self.database_engine, PostgresEngine): @@ -381,19 +378,6 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # take a while! txn.execute("SET LOCAL statement_timeout = 0") - # First, fetch all the state groups that should be deleted, before - # we delete that information. - txn.execute( - """ - SELECT DISTINCT state_group FROM events - INNER JOIN event_to_state_groups USING(event_id) - WHERE events.room_id = ? - """, - (room_id,), - ) - - state_groups = [row[0] for row in txn] - # Get all the auth chains that are referenced by events that are to be # deleted. txn.execute( @@ -513,5 +497,3 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # periodically anyway (https://github.com/matrix-org/synapse/issues/5888) self._invalidate_caches_for_room_and_stream(txn, room_id) - - return state_groups diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index f7a59c8992..9944f90015 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -840,60 +840,42 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): return dict(rows) - async def purge_room_state( - self, room_id: str, state_groups_to_delete: Collection[int] - ) -> None: - """Deletes all record of a room from state tables - - Args: - room_id: - state_groups_to_delete: State groups to delete - """ - - logger.info("[purge] Starting state purge") - await self.db_pool.runInteraction( + async def purge_room_state(self, room_id: str) -> None: + return await self.db_pool.runInteraction( "purge_room_state", self._purge_room_state_txn, room_id, - state_groups_to_delete, ) - logger.info("[purge] Done with state purge") def _purge_room_state_txn( self, txn: LoggingTransaction, room_id: str, - state_groups_to_delete: Collection[int], ) -> None: - # first we have to delete the state groups states - logger.info("[purge] removing %s from state_groups_state", room_id) - - self.db_pool.simple_delete_many_txn( - txn, - table="state_groups_state", - column="state_group", - values=state_groups_to_delete, - keyvalues={}, - ) - - # ... and the state group edges + # Delete all edges that reference a state group linked to room_id logger.info("[purge] removing %s from state_group_edges", room_id) - - self.db_pool.simple_delete_many_txn( - txn, - table="state_group_edges", - column="state_group", - values=state_groups_to_delete, - keyvalues={}, + txn.execute( + """ + DELETE FROM state_group_edges AS sge WHERE sge.state_group IN ( + SELECT id FROM state_groups AS sg WHERE sg.room_id = ? + )""", + (room_id,), ) - # ... and the state groups - logger.info("[purge] removing %s from state_groups", room_id) + # state_groups_state table has a room_id column but no index on it, unlike state_groups, + # so we delete them by matching the room_id through the state_groups table. + logger.info("[purge] removing %s from state_groups_state", room_id) + txn.execute( + """ + DELETE FROM state_groups_state AS sgs WHERE sgs.state_group IN ( + SELECT id FROM state_groups AS sg WHERE sg.room_id = ? + )""", + (room_id,), + ) - self.db_pool.simple_delete_many_txn( + logger.info("[purge] removing %s from state_groups", room_id) + self.db_pool.simple_delete_txn( txn, table="state_groups", - column="id", - values=state_groups_to_delete, - keyvalues={}, + keyvalues={"room_id": room_id}, ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 95ed736451..99df591529 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -3050,7 +3050,7 @@ PURGE_TABLES = [ "pusher_throttle", "room_account_data", "room_tags", - # "state_groups", # Current impl leaves orphaned state groups around. + "state_groups", "state_groups_state", "federation_inbound_events_staging", ] From 49fcda31f6eb0076e09e00083612afb642dfd988 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:23:12 +0000 Subject: [PATCH 073/691] Bump serde from 1.0.216 to 1.0.217 (#18059) --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50481282e6..859dd2086d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -431,18 +431,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", From a0a4a36891f35393b8d4fb840929eb3b833df616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:24:49 +0000 Subject: [PATCH 074/691] Bump pyicu from 2.13.1 to 2.14 (#18060) --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index b5e5bcbd0e..4846f7e750 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1825,12 +1825,12 @@ plugins = ["importlib-metadata"] [[package]] name = "pyicu" -version = "2.13.1" +version = "2.14" description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" files = [ - {file = "PyICU-2.13.1.tar.gz", hash = "sha256:d4919085eaa07da12bade8ee721e7bbf7ade0151ca0f82946a26c8f4b98cdceb"}, + {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] [[package]] From 9fb7333a7cda5e7bf288ca8b044878749f0c74a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:27:05 +0000 Subject: [PATCH 075/691] Bump sentry-sdk from 2.17.0 to 2.19.2 (#18061) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4846f7e750..fb02363e8a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2321,13 +2321,13 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.17.0" +version = "2.19.2" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" files = [ - {file = "sentry_sdk-2.17.0-py2.py3-none-any.whl", hash = "sha256:625955884b862cc58748920f9e21efdfb8e0d4f98cca4ab0d3918576d5b606ad"}, - {file = "sentry_sdk-2.17.0.tar.gz", hash = "sha256:dd0a05352b78ffeacced73a94e86f38b32e2eae15fff5f30ca5abb568a72eacf"}, + {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"}, + {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"}, ] [package.dependencies] @@ -2353,14 +2353,16 @@ grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"] http2 = ["httpcore[http2] (==1.*)"] httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] -huggingface-hub = ["huggingface-hub (>=0.22)"] +huggingface-hub = ["huggingface_hub (>=0.22)"] langchain = ["langchain (>=0.0.210)"] +launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] litestar = ["litestar (>=2.0.0)"] loguru = ["loguru (>=0.5)"] openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] +openfeature = ["openfeature-sdk (>=0.7.1)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] From c199ede287a219c4d7e5ef2f1d75bfec386fed94 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Jan 2025 14:13:02 +0000 Subject: [PATCH 076/691] 1.122.0rc1 --- CHANGES.md | 56 +++++++++++++++++++++++++++++++++++++++ changelog.d/17846.misc | 1 - changelog.d/17849.feature | 1 - changelog.d/17916.feature | 1 - changelog.d/17930.bugfix | 1 - changelog.d/17948.feature | 3 --- changelog.d/17954.doc | 1 - changelog.d/17964.feature | 1 - changelog.d/17976.doc | 1 - changelog.d/17983.feature | 1 - changelog.d/17992.doc | 1 - changelog.d/17994.doc | 1 - changelog.d/17996.misc | 1 - changelog.d/17998.misc | 1 - changelog.d/17999.misc | 1 - changelog.d/18017.misc | 1 - changelog.d/18020.misc | 1 - changelog.d/18024.bugfix | 1 - changelog.d/18029.bugfix | 1 - changelog.d/18034.removal | 1 - changelog.d/18043.bugfix | 1 - changelog.d/18063.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- 24 files changed, 63 insertions(+), 24 deletions(-) delete mode 100644 changelog.d/17846.misc delete mode 100644 changelog.d/17849.feature delete mode 100644 changelog.d/17916.feature delete mode 100644 changelog.d/17930.bugfix delete mode 100644 changelog.d/17948.feature delete mode 100644 changelog.d/17954.doc delete mode 100644 changelog.d/17964.feature delete mode 100644 changelog.d/17976.doc delete mode 100644 changelog.d/17983.feature delete mode 100644 changelog.d/17992.doc delete mode 100644 changelog.d/17994.doc delete mode 100644 changelog.d/17996.misc delete mode 100644 changelog.d/17998.misc delete mode 100644 changelog.d/17999.misc delete mode 100644 changelog.d/18017.misc delete mode 100644 changelog.d/18020.misc delete mode 100644 changelog.d/18024.bugfix delete mode 100644 changelog.d/18029.bugfix delete mode 100644 changelog.d/18034.removal delete mode 100644 changelog.d/18043.bugfix delete mode 100644 changelog.d/18063.misc diff --git a/CHANGES.md b/CHANGES.md index 116f48d2e0..2bfe41f046 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,59 @@ +# Synapse 1.122.0rc1 (2025-01-07) + +### Features + +- Added the `email.tlsname` config option. This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to. ([\#17849](https://github.com/element-hq/synapse/issues/17849)) +- Module developers will have access to the user ID of requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. ([\#17916](https://github.com/element-hq/synapse/issues/17916)) +- Add endpoints to the Admin API to fetch the number of invites the provided user has sent after a given timestamp, + fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event + reports against a provided user (i.e. where the user was the sender of the reported event). ([\#17948](https://github.com/element-hq/synapse/issues/17948)) +- Support stable account suspension from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17964](https://github.com/element-hq/synapse/issues/17964)) +- Add `macaroon_secret_key_path` config option. ([\#17983](https://github.com/element-hq/synapse/issues/17983)) + +### Bugfixes + +- Fix bug when rejecting withdrew invite with a `third_party_rules` module, where the invite would be stuck for the client. ([\#17930](https://github.com/element-hq/synapse/issues/17930)) +- Properly purge state groups tables when purging a room with the Admin API. ([\#18024](https://github.com/element-hq/synapse/issues/18024)) +- Fix a bug preventing the admin redaction endpoint from working on messages from remote users. ([\#18029](https://github.com/element-hq/synapse/issues/18029), [\#18043](https://github.com/element-hq/synapse/issues/18043)) + +### Improved Documentation + +- Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL. ([\#17954](https://github.com/element-hq/synapse/issues/17954)) +- Add previously-undocumented `last_seen_ts` to query user Admin API. ([\#17976](https://github.com/element-hq/synapse/issues/17976)) +- Improve documentation for the `TaskScheduler` class. ([\#17992](https://github.com/element-hq/synapse/issues/17992)) +- Fix example in reverse proxy docs to include server port. ([\#17994](https://github.com/element-hq/synapse/issues/17994)) + +### Deprecations and Removals + +- Remove support for PostgreSQL 11 and 12. Contributed by @clokep. ([\#18034](https://github.com/element-hq/synapse/issues/18034)) + +### Internal Changes + +- Update Alpine Linux Synapse Package Maintainer within installation.md. ([\#17846](https://github.com/element-hq/synapse/issues/17846)) +- Add `RoomID` & `EventID` rust types. ([\#17996](https://github.com/element-hq/synapse/issues/17996)) +- Fix various type errors across the codebase. ([\#17998](https://github.com/element-hq/synapse/issues/17998)) +- Disable DB statement timeout when doing a room purge since it can be quite long. ([\#18017](https://github.com/element-hq/synapse/issues/18017)) +- Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson. ([\#18020](https://github.com/element-hq/synapse/issues/18020)) +- Refactor `get_profile` to no longer include fields with a value of `None`. ([\#18063](https://github.com/element-hq/synapse/issues/18063)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.93 to 1.0.95. ([\#18012](https://github.com/element-hq/synapse/issues/18012), [\#18045](https://github.com/element-hq/synapse/issues/18045)) +* Bump authlib from 1.3.2 to 1.4.0. ([\#18048](https://github.com/element-hq/synapse/issues/18048)) +* Bump dawidd6/action-download-artifact from 6 to 7. ([\#17981](https://github.com/element-hq/synapse/issues/17981)) +* Bump http from 1.1.0 to 1.2.0. ([\#18013](https://github.com/element-hq/synapse/issues/18013)) +- Bump mypy from 1.11.2 to 1.12.1. ([\#17999](https://github.com/element-hq/synapse/issues/17999)) +* Bump mypy-zope from 1.0.8 to 1.0.9. ([\#18047](https://github.com/element-hq/synapse/issues/18047)) +* Bump pillow from 10.4.0 to 11.0.0. ([\#18015](https://github.com/element-hq/synapse/issues/18015)) +* Bump pydantic from 2.9.2 to 2.10.3. ([\#18014](https://github.com/element-hq/synapse/issues/18014)) +* Bump pyicu from 2.13.1 to 2.14. ([\#18060](https://github.com/element-hq/synapse/issues/18060)) +* Bump pyo3 from 0.23.2 to 0.23.3. ([\#18001](https://github.com/element-hq/synapse/issues/18001)) +* Bump python-multipart from 0.0.16 to 0.0.18. ([\#17985](https://github.com/element-hq/synapse/issues/17985)) +* Bump sentry-sdk from 2.17.0 to 2.19.2. ([\#18061](https://github.com/element-hq/synapse/issues/18061)) +* Bump serde from 1.0.215 to 1.0.217. ([\#18031](https://github.com/element-hq/synapse/issues/18031), [\#18059](https://github.com/element-hq/synapse/issues/18059)) +* Bump serde_json from 1.0.133 to 1.0.134. ([\#18044](https://github.com/element-hq/synapse/issues/18044)) +* Bump twine from 5.1.1 to 6.0.1. ([\#18049](https://github.com/element-hq/synapse/issues/18049)) + # Synapse 1.121.1 (2024-12-11) This release contains a fix for our docker build CI. It is functionally identical to 1.121.0, whose changelog is below. diff --git a/changelog.d/17846.misc b/changelog.d/17846.misc deleted file mode 100644 index 6d1f930f10..0000000000 --- a/changelog.d/17846.misc +++ /dev/null @@ -1 +0,0 @@ -Update Alpine Linux Synapse Package Maintainer within installation.md. diff --git a/changelog.d/17849.feature b/changelog.d/17849.feature deleted file mode 100644 index 4de580f9ed..0000000000 --- a/changelog.d/17849.feature +++ /dev/null @@ -1 +0,0 @@ -Added the `email.tlsname` config option. This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to. diff --git a/changelog.d/17916.feature b/changelog.d/17916.feature deleted file mode 100644 index 118997c5e5..0000000000 --- a/changelog.d/17916.feature +++ /dev/null @@ -1 +0,0 @@ -Module developers will have access to user id of requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. \ No newline at end of file diff --git a/changelog.d/17930.bugfix b/changelog.d/17930.bugfix deleted file mode 100644 index 2e37686857..0000000000 --- a/changelog.d/17930.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug when rejecting withdrew invite with a third_party_rules module, where the invite would be stuck for the client. diff --git a/changelog.d/17948.feature b/changelog.d/17948.feature deleted file mode 100644 index d404996cd6..0000000000 --- a/changelog.d/17948.feature +++ /dev/null @@ -1,3 +0,0 @@ -Add endpoints to Admin API to fetch the number of invites the provided user has sent after a given timestamp, -fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event -reports against a provided user (ie where the user was the sender of the reported event). diff --git a/changelog.d/17954.doc b/changelog.d/17954.doc deleted file mode 100644 index 8986e41e58..0000000000 --- a/changelog.d/17954.doc +++ /dev/null @@ -1 +0,0 @@ -Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL. diff --git a/changelog.d/17964.feature b/changelog.d/17964.feature deleted file mode 100644 index e2ae566eb9..0000000000 --- a/changelog.d/17964.feature +++ /dev/null @@ -1 +0,0 @@ -Support stable account suspension from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). \ No newline at end of file diff --git a/changelog.d/17976.doc b/changelog.d/17976.doc deleted file mode 100644 index c45ead4cf9..0000000000 --- a/changelog.d/17976.doc +++ /dev/null @@ -1 +0,0 @@ -Add previously-undocumented `last_seen_ts` to query user admin API. \ No newline at end of file diff --git a/changelog.d/17983.feature b/changelog.d/17983.feature deleted file mode 100644 index 2c54c80c40..0000000000 --- a/changelog.d/17983.feature +++ /dev/null @@ -1 +0,0 @@ -Add `macaroon_secret_key_path` config option. \ No newline at end of file diff --git a/changelog.d/17992.doc b/changelog.d/17992.doc deleted file mode 100644 index 74afabe40f..0000000000 --- a/changelog.d/17992.doc +++ /dev/null @@ -1 +0,0 @@ -Improve documentation for the `TaskScheduler` class. \ No newline at end of file diff --git a/changelog.d/17994.doc b/changelog.d/17994.doc deleted file mode 100644 index 54b7cf1000..0000000000 --- a/changelog.d/17994.doc +++ /dev/null @@ -1 +0,0 @@ -Fix example in reverse proxy docs to include server port. diff --git a/changelog.d/17996.misc b/changelog.d/17996.misc deleted file mode 100644 index 8664ee45ae..0000000000 --- a/changelog.d/17996.misc +++ /dev/null @@ -1 +0,0 @@ -Add `RoomID` & `EventID` rust types. diff --git a/changelog.d/17998.misc b/changelog.d/17998.misc deleted file mode 100644 index 5ddd54cd08..0000000000 --- a/changelog.d/17998.misc +++ /dev/null @@ -1 +0,0 @@ -Fix various type errors across the codebase. \ No newline at end of file diff --git a/changelog.d/17999.misc b/changelog.d/17999.misc deleted file mode 100644 index 9cabc5b15d..0000000000 --- a/changelog.d/17999.misc +++ /dev/null @@ -1 +0,0 @@ -Bump mypy from 1.11.2 to 1.12.1. \ No newline at end of file diff --git a/changelog.d/18017.misc b/changelog.d/18017.misc deleted file mode 100644 index 6b943a5bed..0000000000 --- a/changelog.d/18017.misc +++ /dev/null @@ -1 +0,0 @@ -Disable DB statement timeout when doing a purge room since it can be quite long. diff --git a/changelog.d/18020.misc b/changelog.d/18020.misc deleted file mode 100644 index 8d2dd883b9..0000000000 --- a/changelog.d/18020.misc +++ /dev/null @@ -1 +0,0 @@ -Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson. diff --git a/changelog.d/18024.bugfix b/changelog.d/18024.bugfix deleted file mode 100644 index 956f43f036..0000000000 --- a/changelog.d/18024.bugfix +++ /dev/null @@ -1 +0,0 @@ -Properly purge state groups tables when purging a room with the admin API. diff --git a/changelog.d/18029.bugfix b/changelog.d/18029.bugfix deleted file mode 100644 index f7036fe9fc..0000000000 --- a/changelog.d/18029.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug preventing the admin redaction endpoint from working on messages from remote users. diff --git a/changelog.d/18034.removal b/changelog.d/18034.removal deleted file mode 100644 index 303b442fd4..0000000000 --- a/changelog.d/18034.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for PostgreSQL 11 and 12. Contributed by @clokep. diff --git a/changelog.d/18043.bugfix b/changelog.d/18043.bugfix deleted file mode 100644 index 05f82a4487..0000000000 --- a/changelog.d/18043.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug preventing the admin redaction endpoint from working on messages from remote users. \ No newline at end of file diff --git a/changelog.d/18063.misc b/changelog.d/18063.misc deleted file mode 100644 index 424bf25408..0000000000 --- a/changelog.d/18063.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `get_profile` to no longer include fields with a value of `None`. diff --git a/debian/changelog b/debian/changelog index fc34d8ac3f..e08a212309 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.122.0~rc1) stable; urgency=medium + + * New Synapse release 1.122.0rc1. + + -- Synapse Packaging team Tue, 07 Jan 2025 14:06:19 +0000 + matrix-synapse-py3 (1.121.1) stable; urgency=medium * New Synapse release 1.121.1. diff --git a/pyproject.toml b/pyproject.toml index 0430371586..af6f82bcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.121.1" +version = "1.122.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 1143e144792de0cf8c7bbc9882f88b3d8d8f7084 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Jan 2025 15:20:24 +0000 Subject: [PATCH 077/691] Tweak changelog --- CHANGES.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2bfe41f046..3ddc96a96a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,15 @@ # Synapse 1.122.0rc1 (2025-01-07) +Please note that this version of Synapse drops support for PostgresQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. + +### Deprecations and Removals + +- Remove support for PostgreSQL 11 and 12. Contributed by @clokep. ([\#18034](https://github.com/element-hq/synapse/issues/18034)) + ### Features - Added the `email.tlsname` config option. This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to. ([\#17849](https://github.com/element-hq/synapse/issues/17849)) -- Module developers will have access to the user ID of requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. ([\#17916](https://github.com/element-hq/synapse/issues/17916)) +- Module developers will have access to the user ID of the requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat. ([\#17916](https://github.com/element-hq/synapse/issues/17916)) - Add endpoints to the Admin API to fetch the number of invites the provided user has sent after a given timestamp, fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event reports against a provided user (i.e. where the user was the sender of the reported event). ([\#17948](https://github.com/element-hq/synapse/issues/17948)) @@ -19,17 +25,13 @@ ### Improved Documentation - Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL. ([\#17954](https://github.com/element-hq/synapse/issues/17954)) -- Add previously-undocumented `last_seen_ts` to query user Admin API. ([\#17976](https://github.com/element-hq/synapse/issues/17976)) +- Add documentation for the previously-undocumented `last_seen_ts` query parameter to the query user Admin API. ([\#17976](https://github.com/element-hq/synapse/issues/17976)) - Improve documentation for the `TaskScheduler` class. ([\#17992](https://github.com/element-hq/synapse/issues/17992)) - Fix example in reverse proxy docs to include server port. ([\#17994](https://github.com/element-hq/synapse/issues/17994)) - -### Deprecations and Removals - -- Remove support for PostgreSQL 11 and 12. Contributed by @clokep. ([\#18034](https://github.com/element-hq/synapse/issues/18034)) +- Update Alpine Linux Synapse Package Maintainer within the installation instructions. ([\#17846](https://github.com/element-hq/synapse/issues/17846)) ### Internal Changes -- Update Alpine Linux Synapse Package Maintainer within installation.md. ([\#17846](https://github.com/element-hq/synapse/issues/17846)) - Add `RoomID` & `EventID` rust types. ([\#17996](https://github.com/element-hq/synapse/issues/17996)) - Fix various type errors across the codebase. ([\#17998](https://github.com/element-hq/synapse/issues/17998)) - Disable DB statement timeout when doing a room purge since it can be quite long. ([\#18017](https://github.com/element-hq/synapse/issues/18017)) From 60f596b4d8f215e6bbbafdc0babdefab7c00490b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:50:50 +0000 Subject: [PATCH 078/691] Bump pyopenssl from 24.2.1 to 24.3.0 (#18062) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb02363e8a..f83124bbfc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1907,20 +1907,20 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" -version = "24.2.1" +version = "24.3.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" files = [ - {file = "pyOpenSSL-24.2.1-py3-none-any.whl", hash = "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"}, - {file = "pyopenssl-24.2.1.tar.gz", hash = "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95"}, + {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, + {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, ] [package.dependencies] -cryptography = ">=41.0.5,<44" +cryptography = ">=41.0.5,<45" [package.extras] -docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] [[package]] From fa320c4fcbac2a4f5500e27afdb64dcf77ee03a8 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Jan 2025 17:43:41 +0000 Subject: [PATCH 079/691] Fix typographical error in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3ddc96a96a..8ab44b5771 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Synapse 1.122.0rc1 (2025-01-07) -Please note that this version of Synapse drops support for PostgresQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. +Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. ### Deprecations and Removals From beea39f0006ad1e2e5557d229227d57a3f846e08 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Jan 2025 15:45:57 -0700 Subject: [PATCH 080/691] Drop unstable MSC4151 implementation (#18052) It's been rotated out of known clients, and should be safe for removal now. Fixes https://github.com/element-hq/synapse/issues/17373 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18052.removal | 1 + synapse/config/experimental.py | 3 --- synapse/rest/client/reporting.py | 12 ------------ synapse/rest/client/versions.py | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) create mode 100644 changelog.d/18052.removal diff --git a/changelog.d/18052.removal b/changelog.d/18052.removal new file mode 100644 index 0000000000..f173b9ba4d --- /dev/null +++ b/changelog.d/18052.removal @@ -0,0 +1 @@ +Remove the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) implementation. The stable support remains, per [Matrix 1.13](https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv3roomsroomidreport). \ No newline at end of file diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index eb8d967e70..90d19849ff 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -436,9 +436,6 @@ class ExperimentalConfig(Config): ("experimental", "msc4108_delegation_endpoint"), ) - # MSC4151: Report room API (Client-Server API) - self.msc4151_enabled: bool = experimental.get("msc4151_enabled", False) - # MSC4210: Remove legacy mentions self.msc4210_enabled: bool = experimental.get("msc4210_enabled", False) diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 949f077035..c5037be8b7 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -20,13 +20,11 @@ # import logging -import re from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse._pydantic_compat import StrictStr from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError -from synapse.api.urls import CLIENT_API_PREFIX from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -127,16 +125,6 @@ class ReportRoomRestServlet(RestServlet): self.clock = hs.get_clock() self.store = hs.get_datastores().main - # TODO: Remove the unstable variant after 2-3 releases - # https://github.com/element-hq/synapse/issues/17373 - if hs.config.experimental.msc4151_enabled: - self.PATTERNS.append( - re.compile( - f"^{CLIENT_API_PREFIX}/unstable/org.matrix.msc4151" - "/rooms/(?P[^/]*)/report$" - ) - ) - class PostBody(RequestBodyModel): reason: StrictStr diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index ba1141bbe5..a1d089ebac 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -170,8 +170,6 @@ class VersionsRestServlet(RestServlet): ), # MSC4140: Delayed events "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), - # MSC4151: Report room API (Client-Server API) - "org.matrix.msc4151": self.config.experimental.msc4151_enabled, # Simplified sliding sync "org.matrix.simplified_msc3575": msc3575_enabled, }, From e34fd1228df8f20ae56896c3cbf2b15efcfaa06a Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 8 Jan 2025 07:38:26 -0800 Subject: [PATCH 081/691] Add the ability to filter by state event type on admin room state endpoint (#18035) Adds a query param `type` to `/_synapse/admin/v1/rooms/{room_id}/state` that filters the state event query by state event type. --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18035.feature | 1 + docs/admin_api/rooms.md | 7 ++++++ synapse/rest/admin/rooms.py | 14 ++++++++++- tests/rest/admin/test_room.py | 46 +++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18035.feature diff --git a/changelog.d/18035.feature b/changelog.d/18035.feature new file mode 100644 index 0000000000..99b68a9e45 --- /dev/null +++ b/changelog.d/18035.feature @@ -0,0 +1 @@ +Add a unit test for the `type` parameter of the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api). \ No newline at end of file diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 8e3a367e90..bfc2cd4376 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -385,6 +385,13 @@ The API is: GET /_synapse/admin/v1/rooms//state ``` +**Parameters** + +The following query parameter is available: + +* `type` - The type of room state event to filter by, eg "m.room.create". If provided, only state events + of this type will be returned (regardless of their `state_key` value). + A response body like the following is returned: ```json diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 01f9de9ffa..3097cb1a9d 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -23,6 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple, cast import attr +from immutabledict import immutabledict from synapse.api.constants import Direction, EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError @@ -463,7 +464,18 @@ class RoomStateRestServlet(RestServlet): if not room: raise NotFoundError("Room not found") - event_ids = await self._storage_controllers.state.get_current_state_ids(room_id) + state_filter = None + type = parse_string(request, "type") + + if type: + state_filter = StateFilter( + types=immutabledict({type: None}), + include_others=False, + ) + + event_ids = await self._storage_controllers.state.get_current_state_ids( + room_id, state_filter + ) events = await self.store.get_events(event_ids.values()) now = self.clock.time_msec() room_state = await self._event_serializer.serialize_events(events.values(), now) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 99df591529..1817d67a00 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2035,6 +2035,52 @@ class RoomTestCase(unittest.HomeserverTestCase): # the create_room already does the right thing, so no need to verify that we got # the state events it created. + def test_room_state_param(self) -> None: + """Test that filtering by state event type works when requesting state""" + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=m.room.member", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + # only one member has joined so there should be one membership event + self.assertEqual(1, len(state)) + event = state[0] + self.assertEqual(event["type"], "m.room.member") + self.assertEqual(event["state_key"], self.admin_user) + + def test_room_state_param_empty(self) -> None: + """Test that passing an empty string as state filter param returns no state events""" + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + self.assertEqual(5, len(state)) + + def test_room_state_param_not_in_room(self) -> None: + """ + Test that passing a state filter param for a state event not in the room + returns no state events + """ + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/rooms/{room_id}/state?type=m.room.custom", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + state = channel.json_body["state"] + self.assertEqual(0, len(state)) + def _set_canonical_alias( self, room_id: str, test_alias: str, admin_user_tok: str ) -> None: From d0677dca3967bd68167495b25b11f2ed79b5a11d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:08:43 +0000 Subject: [PATCH 082/691] Bump jinja2 from 3.1.4 to 3.1.5 (#18067) --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index f83124bbfc..4655d295c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -842,13 +842,13 @@ trio = ["async_generator", "trio"] [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, ] [package.dependencies] @@ -1590,6 +1590,7 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, From aab3672037ee9ea71c54c89c86d9c0cc35ae184f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 Jan 2025 10:11:09 -0600 Subject: [PATCH 083/691] Bust `_membership_stream_cache` cache when current state changes (#17732) This is particularly a problem in a state reset scenario where the membership might change without a corresponding event. This PR is targeting a scenario where a state reset happens which causes room membership to change. Previously, the cache would just hold onto stale data and now we properly bust the cache in this scenario. We have a few tests for these scenarios which you can see are now fixed because we can remove the `FIXME` where we were previously manually busting the cache in the test itself. This is a general Synapse thing so by it's nature it helps out Sliding Sync. Fix https://github.com/element-hq/synapse/issues/17368 Prerequisite for https://github.com/element-hq/synapse/issues/17929 --- Match when are busting `_curr_state_delta_stream_cache` --- changelog.d/17732.bugfix | 1 + synapse/storage/_base.py | 4 ++- synapse/storage/databases/main/cache.py | 36 +++++++++++++++++++ synapse/storage/databases/main/events.py | 15 +++++++- synapse/util/caches/stream_change_cache.py | 9 +++++ .../client/sliding_sync/test_sliding_sync.py | 18 ---------- tests/storage/test_stream.py | 6 ---- tests/util/test_stream_change_cache.py | 25 +++++++++++++ 8 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 changelog.d/17732.bugfix diff --git a/changelog.d/17732.bugfix b/changelog.d/17732.bugfix new file mode 100644 index 0000000000..572c13fc57 --- /dev/null +++ b/changelog.d/17732.bugfix @@ -0,0 +1 @@ +Fix membership caches not updating in state reset scenarios. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e14d711c76..7251e72e3a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -86,7 +86,9 @@ class SQLBaseStore(metaclass=ABCMeta): """ def _invalidate_state_caches( - self, room_id: str, members_changed: Collection[str] + self, + room_id: str, + members_changed: Collection[str], ) -> None: """Invalidates caches that are based on the current state, but does not stream invalidations down replication. diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 707d18de78..f364464c23 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -219,6 +219,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): room_id = row.keys[0] members_changed = set(row.keys[1:]) self._invalidate_state_caches(room_id, members_changed) + self._curr_state_delta_stream_cache.entity_has_changed( # type: ignore[attr-defined] + room_id, token + ) + for user_id in members_changed: + self._membership_stream_cache.entity_has_changed(user_id, token) # type: ignore[attr-defined] elif row.cache_func == PURGE_HISTORY_CACHE_NAME: if row.keys is None: raise Exception( @@ -236,6 +241,35 @@ class CacheInvalidationWorkerStore(SQLBaseStore): room_id = row.keys[0] self._invalidate_caches_for_room_events(room_id) self._invalidate_caches_for_room(room_id) + self._curr_state_delta_stream_cache.entity_has_changed( # type: ignore[attr-defined] + room_id, token + ) + # Note: This code is commented out to improve cache performance. + # While uncommenting would provide complete correctness, our + # automatic forgotten room purge logic (see + # `forgotten_room_retention_period`) means this would frequently + # clear the entire cache (effectively) and probably have a noticable + # impact on the cache hit ratio. + # + # Not updating the cache here is safe because: + # + # 1. `_membership_stream_cache` is only used to indicate the + # *absence* of changes, i.e. "nothing has changed between tokens + # X and Y and so return early and don't query the database". + # 2. `_membership_stream_cache` is used when we query data from + # `current_state_delta_stream` and `room_memberships` but since + # nothing new is written to the database for those tables when + # purging/deleting a room (only deleting rows), there is nothing + # changed to care about. + # + # At worst, the cache might indicate a change at token X, at which + # point, we will query the database and discover nothing is there. + # + # Ideally, we would make it so that we could clear the cache on a + # more granular level but that's a bit complex and fiddly to do with + # room membership. + # + # self._membership_stream_cache.all_entities_changed(token) # type: ignore[attr-defined] else: self._attempt_to_invalidate_cache(row.cache_func, row.keys) @@ -275,6 +309,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "get_sliding_sync_rooms_for_user", None ) + self._membership_stream_cache.entity_has_changed(data.state_key, token) # type: ignore[attr-defined] elif data.type == EventTypes.RoomEncryption: self._attempt_to_invalidate_cache( "get_room_encryption", (data.room_id,) @@ -291,6 +326,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): # Similar to the above, but the entire caches are invalidated. This is # unfortunate for the membership caches, but should recover quickly. self._curr_state_delta_stream_cache.entity_has_changed(data.room_id, token) # type: ignore[attr-defined] + self._membership_stream_cache.all_entities_changed(token) # type: ignore[attr-defined] self._attempt_to_invalidate_cache("get_rooms_for_user", None) self._attempt_to_invalidate_cache("get_room_type", (data.room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (data.room_id,)) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index dd6ac909e9..a23aaf5096 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1605,7 +1605,13 @@ class PersistEventsStore: room_id delta_state: Deltas that are going to be used to update the `current_state_events` table. Changes to the current state of the room. - stream_id: TODO + stream_id: This is expected to be the minimum `stream_ordering` for the + batch of events that we are persisting; which means we do not end up in a + situation where workers see events before the `current_state_delta` updates. + FIXME: However, this function also gets called with next upcoming + `stream_ordering` when we re-sync the state of a partial stated room (see + `update_current_state(...)`) which may be "correct" but it would be good to + nail down what exactly is the expected value here. sliding_sync_table_changes: Changes to the `sliding_sync_membership_snapshots` and `sliding_sync_joined_rooms` tables derived from the given `delta_state` (see @@ -1908,6 +1914,13 @@ class PersistEventsStore: stream_id, ) + for user_id in members_to_cache_bust: + txn.call_after( + self.store._membership_stream_cache.entity_has_changed, + user_id, + stream_id, + ) + # Invalidate the various caches self.store._invalidate_state_caches_and_stream( txn, room_id, members_to_cache_bust diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 03503abe0f..5ac8643eef 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -314,6 +314,15 @@ class StreamChangeCache: self._entity_to_key[entity] = stream_pos self._evict() + def all_entities_changed(self, stream_pos: int) -> None: + """ + Mark all entities as changed. This is useful when the cache is invalidated and + there may be some potential change for all of the entities. + """ + self._cache.clear() + self._entity_to_key.clear() + self._earliest_known_stream_pos = stream_pos + def _evict(self) -> None: """ Ensure the cache has not exceeded the maximum size. diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index 578cb384cd..f3cf2111ec 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -1169,12 +1169,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) @@ -1322,12 +1316,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(space_room_id)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) @@ -1506,12 +1494,6 @@ class SlidingSyncTestCase(SlidingSyncBase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - # Ensure that the state reset worked and only user2 is in the room now users_in_room = self.get_success(self.store.get_users_in_room(space_room_id)) self.assertIncludes(set(users_in_room), {user2_id}, exact=True) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 38a56419f3..0f58dc8a0a 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -1209,12 +1209,6 @@ class GetCurrentStateDeltaMembershipChangesForUserTestCase(HomeserverTestCase): self.persistence.persist_event(join_rule_event, join_rule_context) ) - # FIXME: We're manually busting the cache since - # https://github.com/element-hq/synapse/issues/17368 is not solved yet - self.store._membership_stream_cache.entity_has_changed( - user1_id, join_rule_event_pos.stream - ) - after_reset_token = self.event_sources.get_current_token() membership_changes = self.get_success( diff --git a/tests/util/test_stream_change_cache.py b/tests/util/test_stream_change_cache.py index c41f5706af..9254bff79b 100644 --- a/tests/util/test_stream_change_cache.py +++ b/tests/util/test_stream_change_cache.py @@ -255,3 +255,28 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): # Unknown entities will return None self.assertEqual(cache.get_max_pos_of_last_change("not@here.website"), None) + + def test_all_entities_changed(self) -> None: + """ + `StreamChangeCache.all_entities_changed(...)` will mark all entites as changed. + """ + cache = StreamChangeCache("#test", 1, max_size=10) + + cache.entity_has_changed("user@foo.com", 2) + cache.entity_has_changed("bar@baz.net", 3) + cache.entity_has_changed("user@elsewhere.org", 4) + + cache.all_entities_changed(5) + + # Everything should be marked as changed before the stream position where the + # change occurred. + self.assertTrue(cache.has_entity_changed("user@foo.com", 4)) + self.assertTrue(cache.has_entity_changed("bar@baz.net", 4)) + self.assertTrue(cache.has_entity_changed("user@elsewhere.org", 4)) + + # Nothing should be marked as changed at/after the stream position where the + # change occurred. In other words, nothing has changed since the stream position + # 5. + self.assertFalse(cache.has_entity_changed("user@foo.com", 5)) + self.assertFalse(cache.has_entity_changed("bar@baz.net", 5)) + self.assertFalse(cache.has_entity_changed("user@elsewhere.org", 5)) From e2d757f62dcec83a482b17d62b0b7541f061f244 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 13 Jan 2025 15:01:00 -0600 Subject: [PATCH 084/691] Increase `rc_invites.per_issuer` for Complement (#18072) It's possible to run into `SynapseError: 429 - Too Many Requests (rc_invites.per_issuer)` `rc_invites.per_issuer` originally introduced in https://github.com/matrix-org/synapse/pull/13125 --- changelog.d/18072.misc | 1 + docker/complement/conf/workers-shared-extra.yaml.j2 | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/18072.misc diff --git a/changelog.d/18072.misc b/changelog.d/18072.misc new file mode 100644 index 0000000000..d701768727 --- /dev/null +++ b/changelog.d/18072.misc @@ -0,0 +1 @@ +Increase invite rate limits (`rc_invites.per_issuer`) for Complement. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index c5228af72d..4c8fa65e62 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -85,6 +85,9 @@ rc_invites: per_user: per_second: 1000 burst_count: 1000 + per_issuer: + per_second: 1000 + burst_count: 1000 federation_rr_transactions_per_room_per_second: 9999 From b4fd694ce3260eab771bc4646649edeab3b93901 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 14 Jan 2025 14:14:23 +0000 Subject: [PATCH 085/691] 1.122.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8ab44b5771..6e2e4ad7ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.122.0 (2025-01-14) + +No significant changes since 1.122.0rc1. + + + + # Synapse 1.122.0rc1 (2025-01-07) Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. diff --git a/debian/changelog b/debian/changelog index e08a212309..7e50f3ce94 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.122.0) stable; urgency=medium + + * New Synapse release 1.122.0. + + -- Synapse Packaging team Tue, 14 Jan 2025 14:14:14 +0000 + matrix-synapse-py3 (1.122.0~rc1) stable; urgency=medium * New Synapse release 1.122.0rc1. diff --git a/pyproject.toml b/pyproject.toml index af6f82bcf9..c947a89f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.122.0rc1" +version = "1.122.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 48334fbc40e6ad3f4895ed8b2e3cdf1045d0b534 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 14 Jan 2025 14:17:55 +0000 Subject: [PATCH 086/691] move postgres changelog to the top --- CHANGES.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6e2e4ad7ac..cd67b0b752 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,12 @@ # Synapse 1.122.0 (2025-01-14) +Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. + No significant changes since 1.122.0rc1. - - # Synapse 1.122.0rc1 (2025-01-07) -Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. - ### Deprecations and Removals - Remove support for PostgreSQL 11 and 12. Contributed by @clokep. ([\#18034](https://github.com/element-hq/synapse/issues/18034)) From e70e8d132c4f4b58a2eb3203b84dfbc3b1b1a953 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 14 Jan 2025 14:20:08 +0000 Subject: [PATCH 087/691] Move 2023/4 changelog entries under docs/changelogs --- CHANGES.md | 3780 ------------------------------- docs/changelogs/CHANGES-2023.md | 2202 ++++++++++++++++++ docs/changelogs/CHANGES-2024.md | 1576 +++++++++++++ 3 files changed, 3778 insertions(+), 3780 deletions(-) create mode 100644 docs/changelogs/CHANGES-2023.md create mode 100644 docs/changelogs/CHANGES-2024.md diff --git a/CHANGES.md b/CHANGES.md index cd67b0b752..9613aefe4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -72,3784 +72,4 @@ This release contains a fix for our docker build CI. It is functionally identica -# Synapse 1.121.0 (2024-12-11) - -### Internal Changes - -- Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) - - - -# Synapse 1.121.0rc1 (2024-12-04) - -### Features - -- Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. ([\#17705](https://github.com/element-hq/synapse/issues/17705)) -- Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. ([\#17947](https://github.com/element-hq/synapse/issues/17947)) -- Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). ([\#17965](https://github.com/element-hq/synapse/issues/17965)) -- [MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. ([\#17975](https://github.com/element-hq/synapse/issues/17975)) - -### Bugfixes - -- Fix long-standing bug where read receipts could get overly delayed being sent over federation. ([\#17933](https://github.com/element-hq/synapse/issues/17933)) - -### Improved Documentation - -- Add OIDC example configuration for Forgejo (fork of Gitea). ([\#17872](https://github.com/element-hq/synapse/issues/17872)) -- Link to element-docker-demo from contrib/docker*. ([\#17953](https://github.com/element-hq/synapse/issues/17953)) - -### Internal Changes - -- [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. ([\#17253](https://github.com/element-hq/synapse/issues/17253)) -- Fix incorrect comment in new schema delta. ([\#17936](https://github.com/element-hq/synapse/issues/17936)) -- Raise setuptools_rust version cap to 1.10.2. ([\#17944](https://github.com/element-hq/synapse/issues/17944)) -- Enable encrypted appservice related experimental features in the complement docker image. ([\#17945](https://github.com/element-hq/synapse/issues/17945)) -- Return whether the user is suspended when querying the user account in the Admin API. ([\#17952](https://github.com/element-hq/synapse/issues/17952)) -- Fix new scheduled tasks jumping the queue. ([\#17962](https://github.com/element-hq/synapse/issues/17962)) -- Bump pyo3 and dependencies to v0.23.2. ([\#17966](https://github.com/element-hq/synapse/issues/17966)) -- Update setuptools-rust and fix building abi3 wheels in latest version. ([\#17969](https://github.com/element-hq/synapse/issues/17969)) -- Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. ([\#17972](https://github.com/element-hq/synapse/issues/17972)) -- Fix Docker and Complement config to be able to use `public_baseurl`. ([\#17986](https://github.com/element-hq/synapse/issues/17986)) -- Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. ([\#17993](https://github.com/element-hq/synapse/issues/17993)) -- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970), [\#17995](https://github.com/element-hq/synapse/issues/17995)) - - -### Updates to locked dependencies - -* Bump bytes from 1.8.0 to 1.9.0. ([\#17982](https://github.com/element-hq/synapse/issues/17982)) -* Bump pysaml2 from 7.3.1 to 7.5.0. ([\#17978](https://github.com/element-hq/synapse/issues/17978)) -* Bump serde_json from 1.0.132 to 1.0.133. ([\#17939](https://github.com/element-hq/synapse/issues/17939)) -* Bump tomli from 2.0.2 to 2.1.0. ([\#17959](https://github.com/element-hq/synapse/issues/17959)) -* Bump tomli from 2.1.0 to 2.2.1. ([\#17979](https://github.com/element-hq/synapse/issues/17979)) -* Bump tornado from 6.4.1 to 6.4.2. ([\#17955](https://github.com/element-hq/synapse/issues/17955)) - -# Synapse 1.120.2 (2024-12-03) - -This version has building of wheels for macOS disabled. -It is functionally identical to 1.120.1, which contains multiple security fixes. -If you are already using 1.120.1, there is no need to upgrade to this version. - - - -# Synapse 1.120.1 (2024-12-03) - -This patch release fixes multiple security vulnerabilities, some affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. - -Administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. - -### Security advisory - -The following issues are fixed in 1.120.1. - -- [GHSA-rfq8-j7rh-8hf2](https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2) / [CVE-2024-52805](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52805): **Unsupported content types can lead to memory exhaustion** - - Synapse instances which have a high `max_upload_size` and which don't have a reverse proxy in front of them that would otherwise limit upload size are affected. - - Fixed by [4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf](https://github.com/element-hq/synapse/commit/4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf). - -- [GHSA-f3r3-h2mq-hx2h](https://github.com/element-hq/synapse/security/advisories/GHSA-f3r3-h2mq-hx2h) / [CVE-2024-52815](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52815): **Malicious invites via federation can break a user's sync** - - Fixed by [d82e1ed357b7ee21dff83d06cba7a67840cfd464](https://github.com/element-hq/synapse/commit/d82e1ed357b7ee21dff83d06cba7a67840cfd464). - -- [GHSA-vp6v-whfm-rv3g](https://github.com/element-hq/synapse/security/advisories/GHSA-vp6v-whfm-rv3g) / [CVE-2024-53863](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53863): **Synapse can be forced to thumbnail unexpected file formats, invoking potentially untrustworthy decoders** - - Synapse instances can disable dynamic thumbnailing by setting `dynamic_thumbnails` to `false` in the configuration file. - - Fixed by [b64a4e5fbbbf119b6c65aedf0d999b4237d55503](https://github.com/element-hq/synapse/commit/b64a4e5fbbbf119b6c65aedf0d999b4237d55503). - -- [GHSA-56w4-5538-8v8h](https://github.com/element-hq/synapse/security/advisories/GHSA-56w4-5538-8v8h) / [CVE-2024-53867](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53867): **The Sliding Sync feature on Synapse versions between 1.113.0rc1 and 1.120.0 can leak partial room state changes to users no longer in a room** - - Non-state events, like messages, are unaffected. - - Synapse instances can disable the Sliding Sync feature by setting `experimental_features.msc3575_enabled` to `false` in the configuration file. - - Fixed by [4daa533e82f345ce87b9495d31781af570ba3ead](https://github.com/element-hq/synapse/commit/4daa533e82f345ce87b9495d31781af570ba3ead). - -See the advisories for more details. If you have any questions, email [security at element.io](mailto:security@element.io). - -### Bugfixes - -- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970)) - - - -# Synapse 1.120.0 (2024-11-26) - -### Bugfixes - -- Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. ([\#17960](https://github.com/element-hq/synapse/issues/17960)) - - - - -# Synapse 1.120.0rc1 (2024-11-20) - -This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the -homeserver's media store. - -Most homeservers operating in the public federation will not be impacted by this change, given that -the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers -will already have updated as a result. - -Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients -and older federated homeservers. -See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information. - -### Features - -- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889)) -- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934)) - -### Improved Documentation - -- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913)) -- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931)) - -### Deprecations and Removals - -- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638)) - -### Internal Changes - -- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865)) -- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923)) -- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924)) -- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928)) -- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932)) -- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946)) - -### Updates to locked dependencies - -* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920)) -* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918)) -* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941)) -* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940)) -* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942)) -* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917)) -* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919)) -* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938)) - -# Synapse 1.119.0 (2024-11-13) - -No significant changes since 1.119.0rc2. - -### Python 3.8 support dropped - -Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9. - -If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse. - - -# Synapse 1.119.0rc2 (2024-11-11) - -Note that due to packaging issues there was no v1.119.0rc1. - - -### Features - -- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374)) -- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888)) - -### Bugfixes - -- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809)) -- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839)) -- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in - the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847)) -- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850)) -- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861)) -- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903)) -- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915)) - -### Internal Changes - -- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908)) -- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786)) -- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890)) -- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830)) -- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852)) -- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884)) -- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887)) -- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894)) -- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905)) -- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911)) -- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902)) -- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909)) -- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912)) -- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921)) - -### Updates to locked dependencies - -* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657)) -* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901)) -* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877)) -* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853)) -* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898)) -* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899)) -* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879)) -* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874)) -* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897)) -* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900)) -* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857)) -* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855)) -* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856)) - -# Synapse 1.118.0 (2024-10-29) - -No significant changes since 1.118.0rc1. - -### Python 3.8 support will be dropped in the next release - -Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. - -Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month. - -### Python 3.13 and PostgreSQL 17 support - -On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. - - -# Synapse 1.118.0rc1 (2024-10-22) - -### Features - -- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) -- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783)) - -### Bugfixes - -- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736)) -- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805)) -- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835)) - -### Improved Documentation - -- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627)) -- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802)) -- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825)) - -### Internal Changes - -- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718)) -- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752)) -- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803)) -- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824)) -- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826)) -- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) - - - -### Updates to locked dependencies - -* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) -* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827)) -* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797)) -* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843)) -* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794)) -* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795)) -* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829)) -* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844)) -* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798)) -* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796)) -* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841)) -* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828)) - -# Synapse 1.117.0 (2024-10-15) - -No significant changes since 1.117.0rc1. - - - - -# Synapse 1.117.0rc1 (2024-10-08) - -### Features - -- Add config option `redis.password_path`. ([\#17717](https://github.com/element-hq/synapse/issues/17717)) - -### Bugfixes - -- Fix a rare bug introduced in v1.29.0 where invalidating a user's access token from a worker could raise an error. ([\#17779](https://github.com/element-hq/synapse/issues/17779)) -- In the response to `GET /_matrix/client/versions`, set the `unstable_features` flag for [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) to `false` when server configuration disables support for delayed events. ([\#17780](https://github.com/element-hq/synapse/issues/17780)) -- Improve input validation and room membership checks in admin redaction API. ([\#17792](https://github.com/element-hq/synapse/issues/17792)) - -### Improved Documentation - -- Clarify the docstring of `test_forget_when_not_left`. ([\#17628](https://github.com/element-hq/synapse/issues/17628)) -- Add documentation note about PYTHONMALLOC for accurate jemalloc memory tracking. Contributed by @hensg. ([\#17709](https://github.com/element-hq/synapse/issues/17709)) -- Remove spurious "TODO UPDATE ALL THIS" note in the Debian installation docs. ([\#17749](https://github.com/element-hq/synapse/issues/17749)) -- Explain how load balancing works for `federation_sender_instances`. ([\#17776](https://github.com/element-hq/synapse/issues/17776)) - -### Internal Changes - -- Minor performance increase for large accounts using sliding sync. ([\#17751](https://github.com/element-hq/synapse/issues/17751)) -- Increase performance of the notifier when there are many syncing users. ([\#17765](https://github.com/element-hq/synapse/issues/17765), [\#17766](https://github.com/element-hq/synapse/issues/17766)) -- Fix performance of streams that don't change often. ([\#17767](https://github.com/element-hq/synapse/issues/17767)) -- Improve performance of sliding sync connections that do not ask for any rooms. ([\#17768](https://github.com/element-hq/synapse/issues/17768)) -- Reduce overhead of sliding sync E2EE loops. ([\#17771](https://github.com/element-hq/synapse/issues/17771)) -- Sliding sync minor performance speed up using new table. ([\#17787](https://github.com/element-hq/synapse/issues/17787)) -- Sliding sync minor performance improvement by omitting unchanged data from incremental responses. ([\#17788](https://github.com/element-hq/synapse/issues/17788)) -- Speed up sliding sync when there are many active subscriptions. ([\#17789](https://github.com/element-hq/synapse/issues/17789)) -- Add missing license headers on new source files. ([\#17799](https://github.com/element-hq/synapse/issues/17799)) - - - -### Updates to locked dependencies - -* Bump phonenumbers from 8.13.45 to 8.13.46. ([\#17773](https://github.com/element-hq/synapse/issues/17773)) -* Bump python-multipart from 0.0.10 to 0.0.12. ([\#17772](https://github.com/element-hq/synapse/issues/17772)) -* Bump regex from 1.10.6 to 1.11.0. ([\#17770](https://github.com/element-hq/synapse/issues/17770)) -* Bump ruff from 0.6.7 to 0.6.8. ([\#17774](https://github.com/element-hq/synapse/issues/17774)) - -# Synapse 1.116.0 (2024-10-01) - -No significant changes since 1.116.0rc2. - - - - -# Synapse 1.116.0rc2 (2024-09-26) - -### Features - -- Add implementation of restricting who can overwrite a state event as proposed by [MSC3757](https://github.com/matrix-org/matrix-spec-proposals/pull/3757). ([\#17513](https://github.com/element-hq/synapse/issues/17513)) - - - - -# Synapse 1.116.0rc1 (2024-09-25) - -### Features - -- Add initial implementation of delayed events as proposed by [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). ([\#17326](https://github.com/element-hq/synapse/issues/17326)) -- Add an asynchronous Admin API endpoint [to redact all a user's events](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#redact-all-the-events-of-a-user), - and [an endpoint to check on the status of that redaction task](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#check-the-status-of-a-redaction-process). ([\#17506](https://github.com/element-hq/synapse/issues/17506)) -- Add support for the `tags` and `not_tags` filters for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17662](https://github.com/element-hq/synapse/issues/17662)) -- Guests can use the new media endpoints to download media, as described by [MSC4189](https://github.com/matrix-org/matrix-spec-proposals/pull/4189). ([\#17675](https://github.com/element-hq/synapse/issues/17675)) -- Add config option `turn_shared_secret_path`. ([\#17690](https://github.com/element-hq/synapse/issues/17690)) -- Return room tags in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync account data extension. ([\#17707](https://github.com/element-hq/synapse/issues/17707)) - -### Bugfixes - -- Make sure we get up-to-date state information when using the new [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables to derive room membership. ([\#17692](https://github.com/element-hq/synapse/issues/17692)) -- Fix bug where room account data would not correctly be sent down [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync for old rooms. ([\#17695](https://github.com/element-hq/synapse/issues/17695)) -- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync which could prevent /sync from working for certain user accounts. ([\#17727](https://github.com/element-hq/synapse/issues/17727), [\#17733](https://github.com/element-hq/synapse/issues/17733)) -- Ignore invites from ignored users in Sliding Sync. ([\#17729](https://github.com/element-hq/synapse/issues/17729)) -- Fix bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync where the server would incorrectly return a negative bump stamp, which caused Element X apps to stop syncing. ([\#17748](https://github.com/element-hq/synapse/issues/17748)) - -### Internal Changes - -- Import pydantic objects from the `_pydantic_compat` module. - This allows `check_pydantic_models.py` to mock those pydantic objects - only in the synapse module, and not interfere with pydantic objects in - external dependencies. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) -- Use [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables as a bulk shortcut for getting the max `event_stream_ordering` of rooms. ([\#17693](https://github.com/element-hq/synapse/issues/17693)) -- Speed up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests a bit where there are many room changes. ([\#17696](https://github.com/element-hq/synapse/issues/17696)) -- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync filter unit tests so the sliding sync API has better test coverage. ([\#17703](https://github.com/element-hq/synapse/issues/17703)) -- Fetch `bump_stamp`s more efficiently in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17723](https://github.com/element-hq/synapse/issues/17723)) -- Shortcut for checking if certain background updates have completed (utilized in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync). ([\#17724](https://github.com/element-hq/synapse/issues/17724)) -- More efficiently fetch rooms for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17725](https://github.com/element-hq/synapse/issues/17725)) -- Fix `_bulk_get_max_event_pos` being inefficient. ([\#17728](https://github.com/element-hq/synapse/issues/17728)) -- Add cache to `get_tags_for_room(...)`. ([\#17730](https://github.com/element-hq/synapse/issues/17730)) -- Small performance improvement in speeding up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17731](https://github.com/element-hq/synapse/issues/17731)) -- Minor speed up of initial [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests. ([\#17734](https://github.com/element-hq/synapse/issues/17734)) -- Remove usage of the deprecated `cgi` module, deprecated in Python 3.11 and removed in Python 3.13. ([\#17741](https://github.com/element-hq/synapse/issues/17741)) -- Fix typing of a variable that is not `Unknown` anymore after updating `treq`. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.86 to 1.0.89. ([\#17685](https://github.com/element-hq/synapse/issues/17685), [\#17716](https://github.com/element-hq/synapse/issues/17716)) -* Bump bytes from 1.7.1 to 1.7.2. ([\#17743](https://github.com/element-hq/synapse/issues/17743)) -* Bump cryptography from 43.0.0 to 43.0.1. ([\#17689](https://github.com/element-hq/synapse/issues/17689)) -* Bump idna from 3.8 to 3.10. ([\#17758](https://github.com/element-hq/synapse/issues/17758)) -* Bump msgpack from 1.0.8 to 1.1.0. ([\#17759](https://github.com/element-hq/synapse/issues/17759)) -* Bump phonenumbers from 8.13.44 to 8.13.45. ([\#17762](https://github.com/element-hq/synapse/issues/17762)) -* Bump prometheus-client from 0.20.0 to 0.21.0. ([\#17746](https://github.com/element-hq/synapse/issues/17746)) -* Bump pyasn1 from 0.6.0 to 0.6.1. ([\#17714](https://github.com/element-hq/synapse/issues/17714)) -* Bump pyasn1-modules from 0.4.0 to 0.4.1. ([\#17747](https://github.com/element-hq/synapse/issues/17747)) -* Bump pydantic from 2.8.2 to 2.9.2. ([\#17756](https://github.com/element-hq/synapse/issues/17756)) -* Bump python-multipart from 0.0.9 to 0.0.10. ([\#17745](https://github.com/element-hq/synapse/issues/17745)) -* Bump ruff from 0.6.4 to 0.6.7. ([\#17715](https://github.com/element-hq/synapse/issues/17715), [\#17760](https://github.com/element-hq/synapse/issues/17760)) -* Bump sentry-sdk from 2.13.0 to 2.14.0. ([\#17712](https://github.com/element-hq/synapse/issues/17712)) -* Bump serde from 1.0.209 to 1.0.210. ([\#17686](https://github.com/element-hq/synapse/issues/17686)) -* Bump serde_json from 1.0.127 to 1.0.128. ([\#17687](https://github.com/element-hq/synapse/issues/17687)) -* Bump treq from 23.11.0 to 24.9.1. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) -* Bump types-pyyaml from 6.0.12.20240808 to 6.0.12.20240917. ([\#17755](https://github.com/element-hq/synapse/issues/17755)) -* Bump types-requests from 2.32.0.20240712 to 2.32.0.20240914. ([\#17713](https://github.com/element-hq/synapse/issues/17713)) -* Bump types-setuptools from 74.1.0.20240907 to 75.1.0.20240917. ([\#17757](https://github.com/element-hq/synapse/issues/17757)) - -# Synapse 1.115.0 (2024-09-17) - -No significant changes since 1.115.0rc2. - - - - -# Synapse 1.115.0rc2 (2024-09-12) - -### Internal Changes - -- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17652](https://github.com/element-hq/synapse/issues/17652)) -- Speed up sliding sync by reducing amount of data pulled out of the database for large rooms. ([\#17683](https://github.com/element-hq/synapse/issues/17683)) - - - - -# Synapse 1.115.0rc1 (2024-09-10) - -### Features - -- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) - -### Bugfixes - -- Return `400 M_BAD_JSON` upon attempting to complete various room actions with a non-local user ID and unknown room ID, rather than an internal server error. ([\#17607](https://github.com/element-hq/synapse/issues/17607)) -- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) -- Fix bug where we returned the wrong `bump_stamp` for invites in sliding sync response, causing incorrect ordering of invites in the room list. ([\#17674](https://github.com/element-hq/synapse/issues/17674)) - -### Improved Documentation - -- Clarify that the admin api resource is only loaded on the main process and not workers. ([\#17590](https://github.com/element-hq/synapse/issues/17590)) -- Fixed typo in `saml2_config` config [example](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#saml2_config). ([\#17594](https://github.com/element-hq/synapse/issues/17594)) - -### Deprecations and Removals - -- Stabilise [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156) by removing the `msc4156_enabled` config setting and defaulting it to `true`. ([\#17650](https://github.com/element-hq/synapse/issues/17650)) - -### Internal Changes - -- Update [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) -- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17512](https://github.com/element-hq/synapse/issues/17512), [\#17632](https://github.com/element-hq/synapse/issues/17632), [\#17633](https://github.com/element-hq/synapse/issues/17633), [\#17634](https://github.com/element-hq/synapse/issues/17634), [\#17635](https://github.com/element-hq/synapse/issues/17635), [\#17636](https://github.com/element-hq/synapse/issues/17636), [\#17641](https://github.com/element-hq/synapse/issues/17641), [\#17654](https://github.com/element-hq/synapse/issues/17654), [\#17673](https://github.com/element-hq/synapse/issues/17673)) -- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599), [\#17631](https://github.com/element-hq/synapse/issues/17631)) -- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) -- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620), [\#17643](https://github.com/element-hq/synapse/issues/17643)) -- Sliding Sync: Split up `get_room_membership_for_user_at_to_token`. ([\#17629](https://github.com/element-hq/synapse/issues/17629)) -- Use new database tables for sliding sync. ([\#17630](https://github.com/element-hq/synapse/issues/17630), [\#17649](https://github.com/element-hq/synapse/issues/17649)) -- Prevent duplicate tags being added to Sliding Sync traces. ([\#17655](https://github.com/element-hq/synapse/issues/17655)) -- Get `bump_stamp` from [new sliding sync tables](https://github.com/element-hq/synapse/pull/17512) which should be faster. ([\#17658](https://github.com/element-hq/synapse/issues/17658)) -- Speed up incremental Sliding Sync requests by avoiding extra work. ([\#17665](https://github.com/element-hq/synapse/issues/17665)) -- Small performance improvement in speeding up sliding sync. ([\#17666](https://github.com/element-hq/synapse/issues/17666), [\#17670](https://github.com/element-hq/synapse/issues/17670), [\#17672](https://github.com/element-hq/synapse/issues/17672)) -- Speed up sliding sync by reducing number of database calls. ([\#17684](https://github.com/element-hq/synapse/issues/17684)) -- Speed up sync by pulling out fewer events from the database. ([\#17688](https://github.com/element-hq/synapse/issues/17688)) - - - -### Updates to locked dependencies - -* Bump authlib from 1.3.1 to 1.3.2. ([\#17679](https://github.com/element-hq/synapse/issues/17679)) -* Bump idna from 3.7 to 3.8. ([\#17682](https://github.com/element-hq/synapse/issues/17682)) -* Bump ruff from 0.6.2 to 0.6.4. ([\#17680](https://github.com/element-hq/synapse/issues/17680)) -* Bump towncrier from 24.7.1 to 24.8.0. ([\#17645](https://github.com/element-hq/synapse/issues/17645)) -* Bump twisted from 24.7.0rc1 to 24.7.0. ([\#17647](https://github.com/element-hq/synapse/issues/17647)) -* Bump types-pillow from 10.2.0.20240520 to 10.2.0.20240822. ([\#17644](https://github.com/element-hq/synapse/issues/17644)) -* Bump types-psycopg2 from 2.9.21.20240417 to 2.9.21.20240819. ([\#17646](https://github.com/element-hq/synapse/issues/17646)) -* Bump types-setuptools from 71.1.0.20240818 to 74.1.0.20240907. ([\#17681](https://github.com/element-hq/synapse/issues/17681)) - -# Synapse 1.114.0 (2024-09-02) - -This release enables support for -[MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) — -Simplified Sliding Sync. This allows using the upcoming releases of the Element -X mobile apps without having to run a Sliding Sync Proxy. - - -### Features - -- Enable native sliding sync support ([MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) and [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186)) by default. ([\#17648](https://github.com/element-hq/synapse/issues/17648)) - - - - -# Synapse 1.114.0rc3 (2024-08-30) - -### Bugfixes - -- Fix regression in v1.114.0rc2 that caused workers to fail to start. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) - - - - -# Synapse 1.114.0rc2 (2024-08-30) - -### Features - -- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) -- Make `hash_password` script accept password input from stdin. ([\#17608](https://github.com/element-hq/synapse/issues/17608)) - -### Bugfixes - -- Fix hierarchy returning 403 when room is accessible through federation. Contributed by Krishan (@kfiven). ([\#17194](https://github.com/element-hq/synapse/issues/17194)) -- Fix content-length on federation `/thumbnail` responses. ([\#17532](https://github.com/element-hq/synapse/issues/17532)) -- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17543](https://github.com/element-hq/synapse/issues/17543)) - -### Internal Changes - -- MSC3861: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) -- Refactor sliding sync class into multiple files. ([\#17595](https://github.com/element-hq/synapse/issues/17595)) -- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599)) -- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) -- Add support to `@tag_args` for standalone functions. ([\#17604](https://github.com/element-hq/synapse/issues/17604)) -- Speed up incremental syncs in sliding sync by adding some more caching. ([\#17606](https://github.com/element-hq/synapse/issues/17606)) -- Always return the user's own read receipts in sliding sync. ([\#17617](https://github.com/element-hq/synapse/issues/17617)) -- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620)) -- Refactor sliding sync code to move room list logic out into a separate class. ([\#17622](https://github.com/element-hq/synapse/issues/17622)) - - - -### Updates to locked dependencies - -* Bump attrs from 23.2.0 to 24.2.0. ([\#17609](https://github.com/element-hq/synapse/issues/17609)) -* Bump cryptography from 42.0.8 to 43.0.0. ([\#17584](https://github.com/element-hq/synapse/issues/17584)) -* Bump phonenumbers from 8.13.43 to 8.13.44. ([\#17610](https://github.com/element-hq/synapse/issues/17610)) -* Bump pygithub from 2.3.0 to 2.4.0. ([\#17612](https://github.com/element-hq/synapse/issues/17612)) -* Bump pyyaml from 6.0.1 to 6.0.2. ([\#17611](https://github.com/element-hq/synapse/issues/17611)) -* Bump sentry-sdk from 2.12.0 to 2.13.0. ([\#17585](https://github.com/element-hq/synapse/issues/17585)) -* Bump serde from 1.0.206 to 1.0.208. ([\#17581](https://github.com/element-hq/synapse/issues/17581)) -* Bump serde from 1.0.208 to 1.0.209. ([\#17613](https://github.com/element-hq/synapse/issues/17613)) -* Bump serde_json from 1.0.124 to 1.0.125. ([\#17582](https://github.com/element-hq/synapse/issues/17582)) -* Bump serde_json from 1.0.125 to 1.0.127. ([\#17614](https://github.com/element-hq/synapse/issues/17614)) -* Bump types-jsonschema from 4.23.0.20240712 to 4.23.0.20240813. ([\#17583](https://github.com/element-hq/synapse/issues/17583)) -* Bump types-setuptools from 71.1.0.20240726 to 71.1.0.20240818. ([\#17586](https://github.com/element-hq/synapse/issues/17586)) - -# Synapse 1.114.0rc1 (2024-08-20) - -### Features - -- Add a flag to `/versions`, `org.matrix.simplified_msc3575`, to indicate whether experimental sliding sync support has been enabled. ([\#17571](https://github.com/element-hq/synapse/issues/17571)) -- Handle changes in `timeline_limit` in experimental sliding sync. ([\#17579](https://github.com/element-hq/synapse/issues/17579)) -- Correctly track read receipts that should be sent down in experimental sliding sync. ([\#17575](https://github.com/element-hq/synapse/issues/17575), [\#17589](https://github.com/element-hq/synapse/issues/17589), [\#17592](https://github.com/element-hq/synapse/issues/17592)) - -### Bugfixes - -- Start handlers for new media endpoints when media resource configured. ([\#17483](https://github.com/element-hq/synapse/issues/17483)) -- Fix timeline ordering (using `stream_ordering` instead of topological ordering) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17510](https://github.com/element-hq/synapse/issues/17510)) -- Fix experimental sliding sync implementation to remember any updates in rooms that were not sent down immediately. ([\#17535](https://github.com/element-hq/synapse/issues/17535)) -- Better exclude partially stated rooms if we must await full state in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17538](https://github.com/element-hq/synapse/issues/17538)) -- Handle lower-case http headers in `_Mulitpart_Parser_Protocol`. ([\#17545](https://github.com/element-hq/synapse/issues/17545)) -- Fix fetching federation signing keys from servers that omit `old_verify_keys`. Contributed by @tulir @ Beeper. ([\#17568](https://github.com/element-hq/synapse/issues/17568)) -- Fix bug where we would respond with an error when a remote server asked for media that had a length of 0, using the new multipart federation media endpoint. ([\#17570](https://github.com/element-hq/synapse/issues/17570)) - -### Improved Documentation - -- Clarify default behaviour of the - [`auto_accept_invites.worker_to_run_on`](https://element-hq.github.io/synapse/develop/usage/configuration/config_documentation.html#auto-accept-invites) - option. ([\#17515](https://github.com/element-hq/synapse/issues/17515)) -- Improve docstrings for profile methods. ([\#17559](https://github.com/element-hq/synapse/issues/17559)) - -### Internal Changes - -- Add more tracing to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17514](https://github.com/element-hq/synapse/issues/17514)) -- Fixup comment in sliding sync implementation. ([\#17531](https://github.com/element-hq/synapse/issues/17531)) -- Replace override of deprecated method `HTTPAdapter.get_connection` with `get_connection_with_tls_context`. ([\#17536](https://github.com/element-hq/synapse/issues/17536)) -- Fix performance of device lists in `/key/changes` and sliding sync. ([\#17537](https://github.com/element-hq/synapse/issues/17537), [\#17548](https://github.com/element-hq/synapse/issues/17548)) -- Bump setuptools from 67.6.0 to 72.1.0. ([\#17542](https://github.com/element-hq/synapse/issues/17542)) -- Add a utility function for generating random event IDs. ([\#17557](https://github.com/element-hq/synapse/issues/17557)) -- Speed up responding to media requests. ([\#17558](https://github.com/element-hq/synapse/issues/17558), [\#17561](https://github.com/element-hq/synapse/issues/17561), [\#17564](https://github.com/element-hq/synapse/issues/17564), [\#17566](https://github.com/element-hq/synapse/issues/17566), [\#17567](https://github.com/element-hq/synapse/issues/17567), [\#17569](https://github.com/element-hq/synapse/issues/17569)) -- Test github token before running release script steps. ([\#17562](https://github.com/element-hq/synapse/issues/17562)) -- Reduce log spam of multipart files. ([\#17563](https://github.com/element-hq/synapse/issues/17563)) -- Refactor per-connection state in experimental sliding sync handler. ([\#17574](https://github.com/element-hq/synapse/issues/17574)) -- Add histogram metrics for sliding sync processing time. ([\#17593](https://github.com/element-hq/synapse/issues/17593)) - - - -### Updates to locked dependencies - -* Bump bytes from 1.6.1 to 1.7.1. ([\#17526](https://github.com/element-hq/synapse/issues/17526)) -* Bump lxml from 5.2.2 to 5.3.0. ([\#17550](https://github.com/element-hq/synapse/issues/17550)) -* Bump phonenumbers from 8.13.42 to 8.13.43. ([\#17551](https://github.com/element-hq/synapse/issues/17551)) -* Bump regex from 1.10.5 to 1.10.6. ([\#17527](https://github.com/element-hq/synapse/issues/17527)) -* Bump sentry-sdk from 2.10.0 to 2.12.0. ([\#17553](https://github.com/element-hq/synapse/issues/17553)) -* Bump serde from 1.0.204 to 1.0.206. ([\#17556](https://github.com/element-hq/synapse/issues/17556)) -* Bump serde_json from 1.0.122 to 1.0.124. ([\#17555](https://github.com/element-hq/synapse/issues/17555)) -* Bump sigstore/cosign-installer from 3.5.0 to 3.6.0. ([\#17549](https://github.com/element-hq/synapse/issues/17549)) -* Bump types-pyyaml from 6.0.12.20240311 to 6.0.12.20240808. ([\#17552](https://github.com/element-hq/synapse/issues/17552)) -* Bump types-requests from 2.31.0.20240406 to 2.32.0.20240712. ([\#17524](https://github.com/element-hq/synapse/issues/17524)) - -# Synapse 1.113.0 (2024-08-13) - -No significant changes since 1.113.0rc1. - - - - -# Synapse 1.113.0rc1 (2024-08-06) - -### Features - -- Track which rooms have been sent to clients in the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17447](https://github.com/element-hq/synapse/issues/17447)) -- Add Account Data extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17477](https://github.com/element-hq/synapse/issues/17477)) -- Add receipts extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17489](https://github.com/element-hq/synapse/issues/17489)) -- Add typing notification extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17505](https://github.com/element-hq/synapse/issues/17505)) - -### Bugfixes - -- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to handle invite/knock rooms when filtering. ([\#17450](https://github.com/element-hq/synapse/issues/17450)) -- Fix a bug introduced in v1.110.0 which caused `/keys/query` to return incomplete results, leading to high network activity and CPU usage on Matrix clients. ([\#17499](https://github.com/element-hq/synapse/issues/17499)) - -### Improved Documentation - -- Update the [`allowed_local_3pids`](https://element-hq.github.io/synapse/v1.112/usage/configuration/config_documentation.html#allowed_local_3pids) config option's msisdn address to a working example. ([\#17476](https://github.com/element-hq/synapse/issues/17476)) - -### Internal Changes - -- Change sliding sync to use their own token format in preparation for storing per-connection state. ([\#17452](https://github.com/element-hq/synapse/issues/17452)) -- Ensure we don't send down negative `bump_stamp` in experimental sliding sync endpoint. ([\#17478](https://github.com/element-hq/synapse/issues/17478)) -- Do not send down empty room entries down experimental sliding sync endpoint. ([\#17479](https://github.com/element-hq/synapse/issues/17479)) -- Refactor Sliding Sync tests to better utilize the `SlidingSyncBase`. ([\#17481](https://github.com/element-hq/synapse/issues/17481), [\#17482](https://github.com/element-hq/synapse/issues/17482)) -- Add some opentracing tags and logging to the experimental sliding sync implementation. ([\#17501](https://github.com/element-hq/synapse/issues/17501)) -- Split and move Sliding Sync tests so we have some more sane test file sizes. ([\#17504](https://github.com/element-hq/synapse/issues/17504)) -- Update the `limited` field description in the Sliding Sync response to accurately describe what it actually represents. ([\#17507](https://github.com/element-hq/synapse/issues/17507)) -- Easier to understand `timeline` assertions in Sliding Sync tests. ([\#17511](https://github.com/element-hq/synapse/issues/17511)) -- Reset the sliding sync connection if we don't recognize the per-connection state position. ([\#17529](https://github.com/element-hq/synapse/issues/17529)) - - - -### Updates to locked dependencies - -* Bump bcrypt from 4.1.3 to 4.2.0. ([\#17495](https://github.com/element-hq/synapse/issues/17495)) -* Bump black from 24.4.2 to 24.8.0. ([\#17522](https://github.com/element-hq/synapse/issues/17522)) -* Bump phonenumbers from 8.13.39 to 8.13.42. ([\#17521](https://github.com/element-hq/synapse/issues/17521)) -* Bump ruff from 0.5.4 to 0.5.5. ([\#17494](https://github.com/element-hq/synapse/issues/17494)) -* Bump serde_json from 1.0.120 to 1.0.121. ([\#17493](https://github.com/element-hq/synapse/issues/17493)) -* Bump serde_json from 1.0.121 to 1.0.122. ([\#17525](https://github.com/element-hq/synapse/issues/17525)) -* Bump towncrier from 23.11.0 to 24.7.1. ([\#17523](https://github.com/element-hq/synapse/issues/17523)) -* Bump types-pyopenssl from 24.1.0.20240425 to 24.1.0.20240722. ([\#17496](https://github.com/element-hq/synapse/issues/17496)) -* Bump types-setuptools from 70.1.0.20240627 to 71.1.0.20240726. ([\#17497](https://github.com/element-hq/synapse/issues/17497)) - -# Synapse 1.112.0 (2024-07-30) - -This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). - -Note that this security fix is also available as **Synapse 1.111.1**, which does not include the rest of the changes in Synapse 1.112.0. - -This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. -If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. - -With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. -The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. - -Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. - -**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. - -### Internal Changes - -- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) - - -# Synapse 1.112.0rc1 (2024-07-23) - -Please note that this release candidate does not include the security dependency update -included in version 1.111.1 as this version was released before 1.111.1. -The same security fix can be found in the full release of 1.112.0. - -### Features - -- Add to-device extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17416](https://github.com/element-hq/synapse/issues/17416)) -- Populate `name`/`avatar` fields in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17418](https://github.com/element-hq/synapse/issues/17418)) -- Populate `heroes` and room summary fields (`joined_count`, `invited_count`) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17419](https://github.com/element-hq/synapse/issues/17419)) -- Populate `is_dm` room field in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17429](https://github.com/element-hq/synapse/issues/17429)) -- Add room subscriptions to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17432](https://github.com/element-hq/synapse/issues/17432)) -- Prepare for authenticated media freeze. ([\#17433](https://github.com/element-hq/synapse/issues/17433)) -- Add E2EE extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17454](https://github.com/element-hq/synapse/issues/17454)) - -### Bugfixes - -- Add configurable option to always include offline users in presence sync results. Contributed by @Michael-Hollister. ([\#17231](https://github.com/element-hq/synapse/issues/17231)) -- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using room type filters and the user has one or more remote invites. ([\#17434](https://github.com/element-hq/synapse/issues/17434)) -- Order `heroes` by `stream_ordering` as the Matrix specification states (applies to `/sync`). ([\#17435](https://github.com/element-hq/synapse/issues/17435)) -- Fix rare bug where `/sync` would break for a user when using workers with multiple stream writers. ([\#17438](https://github.com/element-hq/synapse/issues/17438)) - -### Improved Documentation - -- Update the readme image to have a white background, so that it is readable in dark mode. ([\#17387](https://github.com/element-hq/synapse/issues/17387)) -- Add Red Hat Enterprise Linux and Rocky Linux 8 and 9 installation instructions. ([\#17423](https://github.com/element-hq/synapse/issues/17423)) -- Improve documentation for the [`default_power_level_content_override`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#default_power_level_content_override) config option. ([\#17451](https://github.com/element-hq/synapse/issues/17451)) - -### Internal Changes - -- Make sure we always use the right logic for enabling the media repo. ([\#17424](https://github.com/element-hq/synapse/issues/17424)) -- Fix argument documentation for method `RateLimiter.record_action`. ([\#17426](https://github.com/element-hq/synapse/issues/17426)) -- Reduce volume of 'Waiting for current token' logs, which were introduced in v1.109.0. ([\#17428](https://github.com/element-hq/synapse/issues/17428)) -- Limit concurrent remote downloads to 6 per IP address, and decrement remote downloads without a content-length from the ratelimiter after the download is complete. ([\#17439](https://github.com/element-hq/synapse/issues/17439)) -- Remove unnecessary call to resume producing in fake channel. ([\#17449](https://github.com/element-hq/synapse/issues/17449)) -- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to bump room when it is created. ([\#17453](https://github.com/element-hq/synapse/issues/17453)) -- Speed up generating sliding sync responses. ([\#17458](https://github.com/element-hq/synapse/issues/17458)) -- Add cache to `get_rooms_for_local_user_where_membership_is` to speed up sliding sync. ([\#17460](https://github.com/element-hq/synapse/issues/17460)) -- Speed up fetching room keys from backup. ([\#17461](https://github.com/element-hq/synapse/issues/17461)) -- Speed up sorting of the room list in sliding sync. ([\#17468](https://github.com/element-hq/synapse/issues/17468)) -- Implement handling of `$ME` as a state key in sliding sync. ([\#17469](https://github.com/element-hq/synapse/issues/17469)) - - - -### Updates to locked dependencies - -* Bump bytes from 1.6.0 to 1.6.1. ([\#17441](https://github.com/element-hq/synapse/issues/17441)) -* Bump hiredis from 2.3.2 to 3.0.0. ([\#17464](https://github.com/element-hq/synapse/issues/17464)) -* Bump jsonschema from 4.22.0 to 4.23.0. ([\#17444](https://github.com/element-hq/synapse/issues/17444)) -* Bump matrix-org/done-action from 2 to 3. ([\#17440](https://github.com/element-hq/synapse/issues/17440)) -* Bump mypy from 1.9.0 to 1.10.1. ([\#17445](https://github.com/element-hq/synapse/issues/17445)) -* Bump pyopenssl from 24.1.0 to 24.2.1. ([\#17465](https://github.com/element-hq/synapse/issues/17465)) -* Bump ruff from 0.5.0 to 0.5.4. ([\#17466](https://github.com/element-hq/synapse/issues/17466)) -* Bump sentry-sdk from 2.6.0 to 2.8.0. ([\#17456](https://github.com/element-hq/synapse/issues/17456)) -* Bump sentry-sdk from 2.8.0 to 2.10.0. ([\#17467](https://github.com/element-hq/synapse/issues/17467)) -* Bump setuptools from 67.6.0 to 70.0.0. ([\#17448](https://github.com/element-hq/synapse/issues/17448)) -* Bump twine from 5.1.0 to 5.1.1. ([\#17443](https://github.com/element-hq/synapse/issues/17443)) -* Bump types-jsonschema from 4.22.0.20240610 to 4.23.0.20240712. ([\#17446](https://github.com/element-hq/synapse/issues/17446)) -* Bump ulid from 1.1.2 to 1.1.3. ([\#17442](https://github.com/element-hq/synapse/issues/17442)) -* Bump zipp from 3.15.0 to 3.19.1. ([\#17427](https://github.com/element-hq/synapse/issues/17427)) - - -# Synapse 1.111.1 (2024-07-30) - -This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). - -This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. -If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. - -With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. -The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. - -Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. - -**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. - - -### Internal Changes - -- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) - - -# Synapse 1.111.0 (2024-07-16) - -No significant changes since 1.111.0rc2. - - - - -# Synapse 1.111.0rc2 (2024-07-10) - -### Bugfixes - -- Fix bug where using `synapse.app.media_repository` worker configuration would break the new media endpoints. ([\#17420](https://github.com/element-hq/synapse/issues/17420)) - -### Improved Documentation - -- Document the new federation media worker endpoints in the [upgrade notes](https://element-hq.github.io/synapse/v1.111/upgrade.html) and [worker docs](https://element-hq.github.io/synapse/v1.111/workers.html). ([\#17421](https://github.com/element-hq/synapse/issues/17421)) - -### Internal Changes - -- Route authenticated federation media requests to media repository workers in Complement tests. ([\#17422](https://github.com/element-hq/synapse/issues/17422)) - - - - -# Synapse 1.111.0rc1 (2024-07-09) - -### Features - -- Add `rooms` data to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17320](https://github.com/element-hq/synapse/issues/17320)) -- Add `room_types`/`not_room_types` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17337](https://github.com/element-hq/synapse/issues/17337)) -- Return "required state" in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17342](https://github.com/element-hq/synapse/issues/17342)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding [`_matrix/client/v1/media/download`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediadownloadservernamemediaid) endpoint. ([\#17365](https://github.com/element-hq/synapse/issues/17365)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/rav/authentication-for-media/proposals/3916-authentication-for-media.md) - by adding [`_matrix/client/v1/media/thumbnail`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid), [`_matrix/federation/v1/media/thumbnail`](https://spec.matrix.org/v1.11/server-server-api/#get_matrixfederationv1mediathumbnailmediaid) endpoints and stabilizing the - remaining [`_matrix/client/v1/media`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediaconfig) endpoints. ([\#17388](https://github.com/element-hq/synapse/issues/17388)) -- Add `rooms.bump_stamp` for easier client-side sorting in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17395](https://github.com/element-hq/synapse/issues/17395)) -- Forget all of a user's rooms upon deactivation, preventing local room purges from being blocked on deactivated users. ([\#17400](https://github.com/element-hq/synapse/issues/17400)) -- Declare support for [Matrix 1.11](https://matrix.org/blog/2024/06/20/matrix-v1.11-release/). ([\#17403](https://github.com/element-hq/synapse/issues/17403)) -- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow overriding the introspection endpoint. ([\#17406](https://github.com/element-hq/synapse/issues/17406)) - -### Bugfixes - -- Fix rare race which caused no new to-device messages to be received from remote server. ([\#17362](https://github.com/element-hq/synapse/issues/17362)) -- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using an old database. ([\#17398](https://github.com/element-hq/synapse/issues/17398)) - -### Improved Documentation - -- Clarify that `url_preview_url_blacklist` is a usability feature. ([\#17356](https://github.com/element-hq/synapse/issues/17356)) -- Fix broken links in README. ([\#17379](https://github.com/element-hq/synapse/issues/17379)) -- Clarify that changelog content *and file extension* need to match in order for entries to merge. ([\#17399](https://github.com/element-hq/synapse/issues/17399)) - -### Internal Changes - -- Make the release script create a release branch for Complement as well. ([\#17318](https://github.com/element-hq/synapse/issues/17318)) -- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) -- Add CI check for the README. ([\#17367](https://github.com/element-hq/synapse/issues/17367)) -- Fix linting errors from new `ruff` version. ([\#17381](https://github.com/element-hq/synapse/issues/17381), [\#17411](https://github.com/element-hq/synapse/issues/17411)) -- Fix building debian packages on non-clean checkouts. ([\#17390](https://github.com/element-hq/synapse/issues/17390)) -- Finish up work to allow per-user feature flags. ([\#17392](https://github.com/element-hq/synapse/issues/17392), [\#17410](https://github.com/element-hq/synapse/issues/17410)) -- Allow enabling sliding sync per-user. ([\#17393](https://github.com/element-hq/synapse/issues/17393)) - - - -### Updates to locked dependencies - -* Bump certifi from 2023.7.22 to 2024.7.4. ([\#17404](https://github.com/element-hq/synapse/issues/17404)) -* Bump cryptography from 42.0.7 to 42.0.8. ([\#17382](https://github.com/element-hq/synapse/issues/17382)) -* Bump ijson from 3.2.3 to 3.3.0. ([\#17413](https://github.com/element-hq/synapse/issues/17413)) -* Bump log from 0.4.21 to 0.4.22. ([\#17384](https://github.com/element-hq/synapse/issues/17384)) -* Bump mypy-zope from 1.0.4 to 1.0.5. ([\#17414](https://github.com/element-hq/synapse/issues/17414)) -* Bump pillow from 10.3.0 to 10.4.0. ([\#17412](https://github.com/element-hq/synapse/issues/17412)) -* Bump pydantic from 2.7.1 to 2.8.2. ([\#17415](https://github.com/element-hq/synapse/issues/17415)) -* Bump ruff from 0.3.7 to 0.5.0. ([\#17381](https://github.com/element-hq/synapse/issues/17381)) -* Bump serde from 1.0.203 to 1.0.204. ([\#17409](https://github.com/element-hq/synapse/issues/17409)) -* Bump serde_json from 1.0.117 to 1.0.120. ([\#17385](https://github.com/element-hq/synapse/issues/17385), [\#17408](https://github.com/element-hq/synapse/issues/17408)) -* Bump types-setuptools from 69.5.0.20240423 to 70.1.0.20240627. ([\#17380](https://github.com/element-hq/synapse/issues/17380)) - -# Synapse 1.110.0 (2024-07-03) - -No significant changes since 1.110.0rc3. - - - - -# Synapse 1.110.0rc3 (2024-07-02) - -### Bugfixes - -- Fix bug where `/sync` requests could get blocked indefinitely after an upgrade from Synapse versions before v1.109.0. ([\#17386](https://github.com/element-hq/synapse/issues/17386), [\#17391](https://github.com/element-hq/synapse/issues/17391)) - -### Internal Changes - -- Limit size of presence EDUs to 50 entries. ([\#17371](https://github.com/element-hq/synapse/issues/17371)) -- Fix building debian package for debian sid. ([\#17389](https://github.com/element-hq/synapse/issues/17389)) - - - - -# Synapse 1.110.0rc2 (2024-06-26) - -### Internal Changes - -- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) - - - - -# Synapse 1.110.0rc1 (2024-06-26) - -### Features - -- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17187](https://github.com/element-hq/synapse/issues/17187)) -- Add experimental support for [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823) - Account suspension. ([\#17255](https://github.com/element-hq/synapse/issues/17255)) -- Improve ratelimiting in Synapse. ([\#17256](https://github.com/element-hq/synapse/issues/17256)) -- Add support for the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) report room API. ([\#17270](https://github.com/element-hq/synapse/issues/17270), [\#17296](https://github.com/element-hq/synapse/issues/17296)) -- Filter for public and empty rooms added to Admin-API [List Room API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#list-room-api). ([\#17276](https://github.com/element-hq/synapse/issues/17276)) -- Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17277](https://github.com/element-hq/synapse/issues/17277)) -- Add `is_encrypted` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17281](https://github.com/element-hq/synapse/issues/17281)) -- Include user membership in events served to clients, per [MSC4115](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17282](https://github.com/element-hq/synapse/issues/17282)) -- Do not require user-interactive authentication for uploading cross-signing keys for the first time, per [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967). ([\#17284](https://github.com/element-hq/synapse/issues/17284)) -- Add `stream_ordering` sort to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17293](https://github.com/element-hq/synapse/issues/17293)) -- `register_new_matrix_user` now supports a --password-file flag, which - is useful for scripting. ([\#17294](https://github.com/element-hq/synapse/issues/17294)) -- `register_new_matrix_user` now supports a --exists-ok flag to allow registration of users that already exist in the database. - This is useful for scripts that bootstrap user accounts with initial passwords. ([\#17304](https://github.com/element-hq/synapse/issues/17304)) -- Add support for via query parameter from [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156). ([\#17322](https://github.com/element-hq/synapse/issues/17322)) -- Add `is_invite` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17335](https://github.com/element-hq/synapse/issues/17335)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding a federation /download endpoint. ([\#17350](https://github.com/element-hq/synapse/issues/17350)) - -### Bugfixes - -- Fix searching for users with their exact localpart whose ID includes a hyphen. ([\#17254](https://github.com/element-hq/synapse/issues/17254)) -- Fix wrong retention policy being used when filtering events. ([\#17272](https://github.com/element-hq/synapse/issues/17272)) -- Fix bug where OTKs were not always included in `/sync` response when using workers. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) -- Fix a long-standing bug where an invalid 'from' parameter to [`/notifications`](https://spec.matrix.org/v1.10/client-server-api/#get_matrixclientv3notifications) would result in an Internal Server Error. ([\#17283](https://github.com/element-hq/synapse/issues/17283)) -- Fix edge case in `/sync` returning the wrong the state when using sharded event persisters. ([\#17295](https://github.com/element-hq/synapse/issues/17295)) -- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17301](https://github.com/element-hq/synapse/issues/17301)) -- Fix email notification subject when invited to a space. ([\#17336](https://github.com/element-hq/synapse/issues/17336)) - -### Improved Documentation - -- Add missing quotes for example for `exclude_rooms_from_sync`. ([\#17308](https://github.com/element-hq/synapse/issues/17308)) -- Update header in the README to visually fix the the auto-generated table of contents. ([\#17329](https://github.com/element-hq/synapse/issues/17329)) -- Fix stale references to the Foundation's Security Disclosure Policy. ([\#17341](https://github.com/element-hq/synapse/issues/17341)) -- Add default values for `rc_invites.per_issuer` to docs. ([\#17347](https://github.com/element-hq/synapse/issues/17347)) -- Fix an error in the docs for `search_all_users` parameter under `user_directory`. ([\#17348](https://github.com/element-hq/synapse/issues/17348)) - -### Internal Changes - -- Remove unused `expire_access_token` option in the Synapse Docker config file. Contributed by @AaronDewes. ([\#17198](https://github.com/element-hq/synapse/issues/17198)) -- Use fully-qualified `PersistedEventPosition` when returning `RoomsForUser` to facilitate proper comparisons and `RoomStreamToken` generation. ([\#17265](https://github.com/element-hq/synapse/issues/17265)) -- Add debug logging for when room keys are uploaded, including whether they are replacing other room keys. ([\#17266](https://github.com/element-hq/synapse/issues/17266)) -- Handle OTK uploads off master. ([\#17271](https://github.com/element-hq/synapse/issues/17271)) -- Don't try and resync devices for remote users whose servers are marked as down. ([\#17273](https://github.com/element-hq/synapse/issues/17273)) -- Re-organize Pydantic models and types used in handlers. ([\#17279](https://github.com/element-hq/synapse/issues/17279)) -- Expose the worker instance that persisted the event on `event.internal_metadata.instance_name`. ([\#17300](https://github.com/element-hq/synapse/issues/17300)) -- Update the README with Element branding, improve headers and fix the #synapse:matrix.org support room link rendering. ([\#17324](https://github.com/element-hq/synapse/issues/17324)) -- Change path of the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync implementation to `/org.matrix.simplified_msc3575/sync` since our simplified API is slightly incompatible with what's in the current MSC. ([\#17331](https://github.com/element-hq/synapse/issues/17331)) -- Handle device lists notifications for large accounts more efficiently in worker mode. ([\#17333](https://github.com/element-hq/synapse/issues/17333), [\#17358](https://github.com/element-hq/synapse/issues/17358)) -- Do not block event sending/receiving while calculating large event auth chains. ([\#17338](https://github.com/element-hq/synapse/issues/17338)) -- Tidy up `parse_integer` docs and call sites to reflect the fact that they require non-negative integers by default, and bring `parse_integer_from_args` default in alignment. Contributed by Denis Kasak (@dkasak). ([\#17339](https://github.com/element-hq/synapse/issues/17339)) - - - -### Updates to locked dependencies - -* Bump authlib from 1.3.0 to 1.3.1. ([\#17343](https://github.com/element-hq/synapse/issues/17343)) -* Bump dawidd6/action-download-artifact from 3.1.4 to 5. ([\#17289](https://github.com/element-hq/synapse/issues/17289)) -* Bump dawidd6/action-download-artifact from 5 to 6. ([\#17313](https://github.com/element-hq/synapse/issues/17313)) -* Bump docker/build-push-action from 5 to 6. ([\#17312](https://github.com/element-hq/synapse/issues/17312)) -* Bump jinja2 from 3.1.3 to 3.1.4. ([\#17287](https://github.com/element-hq/synapse/issues/17287)) -* Bump lazy_static from 1.4.0 to 1.5.0. ([\#17355](https://github.com/element-hq/synapse/issues/17355)) -* Bump msgpack from 1.0.7 to 1.0.8. ([\#17317](https://github.com/element-hq/synapse/issues/17317)) -* Bump netaddr from 1.2.1 to 1.3.0. ([\#17353](https://github.com/element-hq/synapse/issues/17353)) -* Bump packaging from 24.0 to 24.1. ([\#17352](https://github.com/element-hq/synapse/issues/17352)) -* Bump phonenumbers from 8.13.37 to 8.13.39. ([\#17315](https://github.com/element-hq/synapse/issues/17315)) -* Bump regex from 1.10.4 to 1.10.5. ([\#17290](https://github.com/element-hq/synapse/issues/17290)) -* Bump requests from 2.31.0 to 2.32.2. ([\#17345](https://github.com/element-hq/synapse/issues/17345)) -* Bump sentry-sdk from 2.1.1 to 2.3.1. ([\#17263](https://github.com/element-hq/synapse/issues/17263)) -* Bump sentry-sdk from 2.3.1 to 2.6.0. ([\#17351](https://github.com/element-hq/synapse/issues/17351)) -* Bump tornado from 6.4 to 6.4.1. ([\#17344](https://github.com/element-hq/synapse/issues/17344)) -* Bump mypy from 1.8.0 to 1.9.0. ([\#17297](https://github.com/element-hq/synapse/issues/17297)) -* Bump types-jsonschema from 4.21.0.20240311 to 4.22.0.20240610. ([\#17288](https://github.com/element-hq/synapse/issues/17288)) -* Bump types-netaddr from 1.2.0.20240219 to 1.3.0.20240530. ([\#17314](https://github.com/element-hq/synapse/issues/17314)) -* Bump types-pillow from 10.2.0.20240423 to 10.2.0.20240520. ([\#17285](https://github.com/element-hq/synapse/issues/17285)) -* Bump types-pyyaml from 6.0.12.12 to 6.0.12.20240311. ([\#17316](https://github.com/element-hq/synapse/issues/17316)) -* Bump typing-extensions from 4.11.0 to 4.12.2. ([\#17354](https://github.com/element-hq/synapse/issues/17354)) -* Bump urllib3 from 2.0.7 to 2.2.2. ([\#17346](https://github.com/element-hq/synapse/issues/17346)) - -# Synapse 1.109.0 (2024-06-18) - -### Internal Changes - -- Fix the building of binary wheels for macOS by switching to macOS 12 CI runners. ([\#17319](https://github.com/element-hq/synapse/issues/17319)) - - - - -# Synapse 1.109.0rc3 (2024-06-17) - -### Bugfixes - -- When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL. ([\#17305](https://github.com/element-hq/synapse/issues/17305), [\#17309](https://github.com/element-hq/synapse/issues/17309)) - -### Internal Changes - -- Use the release branch for sytest in release-branch PRs. ([\#17306](https://github.com/element-hq/synapse/issues/17306)) - - - - -# Synapse 1.109.0rc2 (2024-06-11) - -### Bugfixes - -- Fix bug where one-time-keys were not always included in `/sync` response when using workers. Introduced in v1.109.0rc1. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) -- Fix bug where `/sync` could get stuck due to edge case in device lists handling. Introduced in v1.109.0rc1. ([\#17292](https://github.com/element-hq/synapse/issues/17292)) - - - - -# Synapse 1.109.0rc1 (2024-06-04) - -### Features - -- Add the ability to auto-accept invites on the behalf of users. See the [`auto_accept_invites`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#auto-accept-invites) config option for details. ([\#17147](https://github.com/element-hq/synapse/issues/17147)) -- Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for to-device messages and device encryption info. ([\#17167](https://github.com/element-hq/synapse/issues/17167)) -- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/issues/3916) by adding unstable media endpoints to `/_matrix/client`. ([\#17213](https://github.com/element-hq/synapse/issues/17213)) -- Add logging to tasks managed by the task scheduler, showing CPU and database usage. ([\#17219](https://github.com/element-hq/synapse/issues/17219)) - -### Bugfixes - -- Fix deduplicating of membership events to not create unused state groups. ([\#17164](https://github.com/element-hq/synapse/issues/17164)) -- Fix bug where duplicate events could be sent down sync when using workers that are overloaded. ([\#17215](https://github.com/element-hq/synapse/issues/17215)) -- Ignore attempts to send to-device messages to bad users, to avoid log spam when we try to connect to the bad server. ([\#17240](https://github.com/element-hq/synapse/issues/17240)) -- Fix handling of duplicate concurrent uploading of device one-time-keys. ([\#17241](https://github.com/element-hq/synapse/issues/17241)) -- Fix reporting of default tags to Sentry, such as worker name. Broke in v1.108.0. ([\#17251](https://github.com/element-hq/synapse/issues/17251)) -- Fix bug where typing updates would not be sent when using workers after a restart. ([\#17252](https://github.com/element-hq/synapse/issues/17252)) - -### Improved Documentation - -- Update the LemonLDAP documentation to say that claims should be explicitly included in the returned `id_token`, as Synapse won't request them. ([\#17204](https://github.com/element-hq/synapse/issues/17204)) - -### Internal Changes - -- Improve DB usage when fetching related events. ([\#17083](https://github.com/element-hq/synapse/issues/17083)) -- Log exceptions when failing to auto-join new user according to the `auto_join_rooms` option. ([\#17176](https://github.com/element-hq/synapse/issues/17176)) -- Reduce work of calculating outbound device lists updates. ([\#17211](https://github.com/element-hq/synapse/issues/17211)) -- Improve performance of calculating device lists changes in `/sync`. ([\#17216](https://github.com/element-hq/synapse/issues/17216)) -- Move towards using `MultiWriterIdGenerator` everywhere. ([\#17226](https://github.com/element-hq/synapse/issues/17226)) -- Replaces all usages of `StreamIdGenerator` with `MultiWriterIdGenerator`. ([\#17229](https://github.com/element-hq/synapse/issues/17229)) -- Change the `allow_unsafe_locale` config option to also apply when setting up new databases. ([\#17238](https://github.com/element-hq/synapse/issues/17238)) -- Fix errors in logs about closing incorrect logging contexts when media gets rejected by a module. ([\#17239](https://github.com/element-hq/synapse/issues/17239), [\#17246](https://github.com/element-hq/synapse/issues/17246)) -- Clean out invalid destinations from `device_federation_outbox` table. ([\#17242](https://github.com/element-hq/synapse/issues/17242)) -- Stop logging errors when receiving invalid User IDs in key querys requests. ([\#17250](https://github.com/element-hq/synapse/issues/17250)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.83 to 1.0.86. ([\#17220](https://github.com/element-hq/synapse/issues/17220)) -* Bump bcrypt from 4.1.2 to 4.1.3. ([\#17224](https://github.com/element-hq/synapse/issues/17224)) -* Bump lxml from 5.2.1 to 5.2.2. ([\#17261](https://github.com/element-hq/synapse/issues/17261)) -* Bump mypy-zope from 1.0.3 to 1.0.4. ([\#17262](https://github.com/element-hq/synapse/issues/17262)) -* Bump phonenumbers from 8.13.35 to 8.13.37. ([\#17235](https://github.com/element-hq/synapse/issues/17235)) -* Bump prometheus-client from 0.19.0 to 0.20.0. ([\#17233](https://github.com/element-hq/synapse/issues/17233)) -* Bump pyasn1 from 0.5.1 to 0.6.0. ([\#17223](https://github.com/element-hq/synapse/issues/17223)) -* Bump pyicu from 2.13 to 2.13.1. ([\#17236](https://github.com/element-hq/synapse/issues/17236)) -* Bump pyopenssl from 24.0.0 to 24.1.0. ([\#17234](https://github.com/element-hq/synapse/issues/17234)) -* Bump serde from 1.0.201 to 1.0.202. ([\#17221](https://github.com/element-hq/synapse/issues/17221)) -* Bump serde from 1.0.202 to 1.0.203. ([\#17232](https://github.com/element-hq/synapse/issues/17232)) -* Bump twine from 5.0.0 to 5.1.0. ([\#17225](https://github.com/element-hq/synapse/issues/17225)) -* Bump types-psycopg2 from 2.9.21.20240311 to 2.9.21.20240417. ([\#17222](https://github.com/element-hq/synapse/issues/17222)) -* Bump types-pyopenssl from 24.0.0.20240311 to 24.1.0.20240425. ([\#17260](https://github.com/element-hq/synapse/issues/17260)) - -# Synapse 1.108.0 (2024-05-28) - -No significant changes since 1.108.0rc1. - - - - -# Synapse 1.108.0rc1 (2024-05-21) - -### Features - -- Add a feature that allows clients to query the configured federation whitelist. Disabled by default. ([\#16848](https://github.com/element-hq/synapse/issues/16848), [\#17199](https://github.com/element-hq/synapse/issues/17199)) -- Add the ability to allow numeric user IDs with a specific prefix when in the CAS flow. Contributed by Aurélien Grimpard. ([\#17098](https://github.com/element-hq/synapse/issues/17098)) - -### Bugfixes - -- Fix bug where push rules would be empty in `/sync` for some accounts. Introduced in v1.93.0. ([\#17142](https://github.com/element-hq/synapse/issues/17142)) -- Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas. ([\#17145](https://github.com/element-hq/synapse/issues/17145)) -- Fix bug where disabling room publication prevented public rooms being created on workers. ([\#17177](https://github.com/element-hq/synapse/issues/17177), [\#17184](https://github.com/element-hq/synapse/issues/17184)) - -### Improved Documentation - -- Document [`/v1/make_knock`](https://spec.matrix.org/v1.10/server-server-api/#get_matrixfederationv1make_knockroomiduserid) and [`/v1/send_knock/`](https://spec.matrix.org/v1.10/server-server-api/#put_matrixfederationv1send_knockroomideventid) federation endpoints as worker-compatible. ([\#17058](https://github.com/element-hq/synapse/issues/17058)) -- Update User Admin API with note about prefixing OIDC external_id providers. ([\#17139](https://github.com/element-hq/synapse/issues/17139)) -- Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option. ([\#17150](https://github.com/element-hq/synapse/issues/17150)) -- Update the Admin FAQ with the current libjemalloc version for latest Debian stable. Additionally update the name of the "push_rules" stream in the Workers documentation. ([\#17171](https://github.com/element-hq/synapse/issues/17171)) - -### Internal Changes - -- Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will remain supported for some time. ([\#17151](https://github.com/element-hq/synapse/issues/17151)) -- Update dependency PyO3 to 0.21. ([\#17162](https://github.com/element-hq/synapse/issues/17162)) -- Fixes linter errors found in PR #17147. ([\#17166](https://github.com/element-hq/synapse/issues/17166)) -- Bump black from 24.2.0 to 24.4.2. ([\#17170](https://github.com/element-hq/synapse/issues/17170)) -- Cache literal sync filter validation for performance. ([\#17186](https://github.com/element-hq/synapse/issues/17186)) -- Improve performance by fixing a reactor pause. ([\#17192](https://github.com/element-hq/synapse/issues/17192)) -- Route `/make_knock` and `/send_knock` federation APIs to the federation reader worker in Complement test runs. ([\#17195](https://github.com/element-hq/synapse/issues/17195)) -- Prepare sync handler to be able to return different sync responses (`SyncVersion`). ([\#17200](https://github.com/element-hq/synapse/issues/17200)) -- Organize the sync cache key parameter outside of the sync config (separate concerns). ([\#17201](https://github.com/element-hq/synapse/issues/17201)) -- Refactor `SyncResultBuilder` assembly to its own function. ([\#17202](https://github.com/element-hq/synapse/issues/17202)) -- Rename to be obvious: `joined_rooms` -> `joined_room_ids`. ([\#17203](https://github.com/element-hq/synapse/issues/17203), [\#17208](https://github.com/element-hq/synapse/issues/17208)) -- Add a short pause when rate-limiting a request. ([\#17210](https://github.com/element-hq/synapse/issues/17210)) - - - -### Updates to locked dependencies - -* Bump cryptography from 42.0.5 to 42.0.7. ([\#17180](https://github.com/element-hq/synapse/issues/17180)) -* Bump gitpython from 3.1.41 to 3.1.43. ([\#17181](https://github.com/element-hq/synapse/issues/17181)) -* Bump immutabledict from 4.1.0 to 4.2.0. ([\#17179](https://github.com/element-hq/synapse/issues/17179)) -* Bump sentry-sdk from 1.40.3 to 2.1.1. ([\#17178](https://github.com/element-hq/synapse/issues/17178)) -* Bump serde from 1.0.200 to 1.0.201. ([\#17183](https://github.com/element-hq/synapse/issues/17183)) -* Bump serde_json from 1.0.116 to 1.0.117. ([\#17182](https://github.com/element-hq/synapse/issues/17182)) - -Synapse 1.107.0 (2024-05-14) -============================ - -No significant changes since 1.107.0rc1. - - -# Synapse 1.107.0rc1 (2024-05-07) - -### Features - -- Add preliminary support for [MSC3823: Account Suspension](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17051](https://github.com/element-hq/synapse/issues/17051)) -- Declare support for [Matrix v1.10](https://matrix.org/blog/2024/03/22/matrix-v1.10-release/). Contributed by @clokep. ([\#17082](https://github.com/element-hq/synapse/issues/17082)) -- Add support for [MSC4115: membership metadata on events](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17104](https://github.com/element-hq/synapse/issues/17104), [\#17137](https://github.com/element-hq/synapse/issues/17137)) - -### Bugfixes - -- Fixed search feature of Element Android on homesevers using SQLite by returning search terms as search highlights. ([\#17000](https://github.com/element-hq/synapse/issues/17000)) -- Fixes a bug introduced in v1.52.0 where the `destination` query parameter for the [Destination Rooms Admin API](https://element-hq.github.io/synapse/v1.105/usage/administration/admin_api/federation.html#destination-rooms) failed to actually filter returned rooms. ([\#17077](https://github.com/element-hq/synapse/issues/17077)) -- For MSC3266 room summaries, support queries at the recommended endpoint of `/_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}`. The existing endpoint of `/_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` is deprecated. ([\#17078](https://github.com/element-hq/synapse/issues/17078)) -- Apply user email & picture during OIDC registration if present & selected. ([\#17120](https://github.com/element-hq/synapse/issues/17120)) -- Improve error message for cross signing reset with [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) enabled. ([\#17121](https://github.com/element-hq/synapse/issues/17121)) -- Fix a bug which meant that to-device messages received over federation could be dropped when the server was under load or networking problems caused problems between Synapse processes or the database. ([\#17127](https://github.com/element-hq/synapse/issues/17127)) -- Fix bug where `StreamChangeCache` would not respect configured cache factors. ([\#17152](https://github.com/element-hq/synapse/issues/17152)) - -### Updates to the Docker image - -- Correct licensing metadata on Docker image. ([\#17141](https://github.com/element-hq/synapse/issues/17141)) - -### Improved Documentation - -- Update the `event_cache_size` and `global_factor` configuration options' documentation. ([\#17071](https://github.com/element-hq/synapse/issues/17071)) -- Remove broken sphinx docs. ([\#17073](https://github.com/element-hq/synapse/issues/17073), [\#17148](https://github.com/element-hq/synapse/issues/17148)) -- Add RuntimeDirectory to example matrix-synapse.service systemd unit. ([\#17084](https://github.com/element-hq/synapse/issues/17084)) -- Fix various small typos throughout the docs. ([\#17114](https://github.com/element-hq/synapse/issues/17114)) -- Update enable_notifs configuration documentation. ([\#17116](https://github.com/element-hq/synapse/issues/17116)) -- Update the Upgrade Notes with the latest minimum supported Rust version of 1.66.0. Contributed by @jahway603. ([\#17140](https://github.com/element-hq/synapse/issues/17140)) - -### Internal Changes - -- Enable [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) by default in the Synapse Complement image. ([\#17105](https://github.com/element-hq/synapse/issues/17105)) -- Add optimisation to `StreamChangeCache.get_entities_changed(..)`. ([\#17130](https://github.com/element-hq/synapse/issues/17130)) - - - -### Updates to locked dependencies - -* Bump furo from 2024.1.29 to 2024.4.27. ([\#17133](https://github.com/element-hq/synapse/issues/17133)) -* Bump idna from 3.6 to 3.7. ([\#17136](https://github.com/element-hq/synapse/issues/17136)) -* Bump jsonschema from 4.21.1 to 4.22.0. ([\#17157](https://github.com/element-hq/synapse/issues/17157)) -* Bump lxml from 5.1.0 to 5.2.1. ([\#17158](https://github.com/element-hq/synapse/issues/17158)) -* Bump phonenumbers from 8.13.29 to 8.13.35. ([\#17106](https://github.com/element-hq/synapse/issues/17106)) -- Bump pillow from 10.2.0 to 10.3.0. ([\#17146](https://github.com/element-hq/synapse/issues/17146)) -* Bump pydantic from 2.6.4 to 2.7.0. ([\#17107](https://github.com/element-hq/synapse/issues/17107)) -* Bump pydantic from 2.7.0 to 2.7.1. ([\#17160](https://github.com/element-hq/synapse/issues/17160)) -* Bump pyicu from 2.12 to 2.13. ([\#17109](https://github.com/element-hq/synapse/issues/17109)) -* Bump serde from 1.0.197 to 1.0.198. ([\#17111](https://github.com/element-hq/synapse/issues/17111)) -* Bump serde from 1.0.198 to 1.0.199. ([\#17132](https://github.com/element-hq/synapse/issues/17132)) -* Bump serde from 1.0.199 to 1.0.200. ([\#17161](https://github.com/element-hq/synapse/issues/17161)) -* Bump serde_json from 1.0.115 to 1.0.116. ([\#17112](https://github.com/element-hq/synapse/issues/17112)) -- Update `tornado` Python dependency from 6.2 to 6.4. ([\#17131](https://github.com/element-hq/synapse/issues/17131)) -* Bump twisted from 23.10.0 to 24.3.0. ([\#17135](https://github.com/element-hq/synapse/issues/17135)) -* Bump types-bleach from 6.1.0.1 to 6.1.0.20240331. ([\#17110](https://github.com/element-hq/synapse/issues/17110)) -* Bump types-pillow from 10.2.0.20240415 to 10.2.0.20240423. ([\#17159](https://github.com/element-hq/synapse/issues/17159)) -* Bump types-setuptools from 69.0.0.20240125 to 69.5.0.20240423. ([\#17134](https://github.com/element-hq/synapse/issues/17134)) - -# Synapse 1.106.0 (2024-04-30) - -No significant changes since 1.106.0rc1. - - - - -# Synapse 1.106.0rc1 (2024-04-25) - -### Features - -- Send an email if the address is already bound to an user account. ([\#16819](https://github.com/element-hq/synapse/issues/16819)) -- Implement the rendezvous mechanism described by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108). ([\#17056](https://github.com/element-hq/synapse/issues/17056)) -- Support delegating the rendezvous mechanism described [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108) to an external implementation. ([\#17086](https://github.com/element-hq/synapse/issues/17086)) - -### Bugfixes - -- Add validation to ensure that the `limit` parameter on `/publicRooms` is non-negative. ([\#16920](https://github.com/element-hq/synapse/issues/16920)) -- Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error. ([\#16923](https://github.com/element-hq/synapse/issues/16923)) -- Make the CSAPI endpoint `/keys/device_signing/upload` idempotent. ([\#16943](https://github.com/element-hq/synapse/issues/16943)) -- Redact membership events if the user requested erasure upon deactivating. ([\#17076](https://github.com/element-hq/synapse/issues/17076)) - -### Improved Documentation - -- Add a prompt in the contributing guide to manually configure icu4c. ([\#17069](https://github.com/element-hq/synapse/issues/17069)) -- Clarify what part of message retention is still experimental. ([\#17099](https://github.com/element-hq/synapse/issues/17099)) - -### Internal Changes - -- Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar). ([\#17032](https://github.com/element-hq/synapse/issues/17032), [\#17096](https://github.com/element-hq/synapse/issues/17096)) -- Fix mypy with latest Twisted release. ([\#17036](https://github.com/element-hq/synapse/issues/17036)) -- Bump minimum supported Rust version to 1.66.0. ([\#17079](https://github.com/element-hq/synapse/issues/17079)) -- Add helpers to transform Twisted requests to Rust http Requests/Responses. ([\#17081](https://github.com/element-hq/synapse/issues/17081)) -- Fix type annotation for `visited_chains` after `mypy` upgrade. ([\#17125](https://github.com/element-hq/synapse/issues/17125)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.81 to 1.0.82. ([\#17095](https://github.com/element-hq/synapse/issues/17095)) -* Bump peaceiris/actions-gh-pages from 3.9.3 to 4.0.0. ([\#17087](https://github.com/element-hq/synapse/issues/17087)) -* Bump peaceiris/actions-mdbook from 1.2.0 to 2.0.0. ([\#17089](https://github.com/element-hq/synapse/issues/17089)) -* Bump pyasn1-modules from 0.3.0 to 0.4.0. ([\#17093](https://github.com/element-hq/synapse/issues/17093)) -* Bump pygithub from 2.2.0 to 2.3.0. ([\#17092](https://github.com/element-hq/synapse/issues/17092)) -* Bump ruff from 0.3.5 to 0.3.7. ([\#17094](https://github.com/element-hq/synapse/issues/17094)) -* Bump sigstore/cosign-installer from 3.4.0 to 3.5.0. ([\#17088](https://github.com/element-hq/synapse/issues/17088)) -* Bump twine from 4.0.2 to 5.0.0. ([\#17091](https://github.com/element-hq/synapse/issues/17091)) -* Bump types-pillow from 10.2.0.20240406 to 10.2.0.20240415. ([\#17090](https://github.com/element-hq/synapse/issues/17090)) - -# Synapse 1.105.1 (2024-04-23) - -## Security advisory - -The following issues are fixed in 1.105.1. - -- [GHSA-3h7q-rfh9-xm4v](https://github.com/element-hq/synapse/security/advisories/GHSA-3h7q-rfh9-xm4v) / [CVE-2024-31208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-31208) — High Severity - - Weakness in auth chain indexing allows DoS from remote room members through disk fill and high CPU usage. - -See the advisories for more details. If you have any questions, email security@element.io. - - - -# Synapse 1.105.0 (2024-04-16) - -No significant changes since 1.105.0rc1. - - - - -# Synapse 1.105.0rc1 (2024-04-11) - -### Features - -- Stabilize support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which clarifies the interaction of push rules and account data. Contributed by @clokep. ([\#17022](https://github.com/element-hq/synapse/issues/17022)) -- Stabilize support for [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981): `/relations` recursion. Contributed by @clokep. ([\#17023](https://github.com/element-hq/synapse/issues/17023)) -- Add support for moving `/pushrules` off of main process. ([\#17037](https://github.com/element-hq/synapse/issues/17037), [\#17038](https://github.com/element-hq/synapse/issues/17038)) - -### Bugfixes - -- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16930](https://github.com/element-hq/synapse/issues/16930), [\#16932](https://github.com/element-hq/synapse/issues/16932), [\#16942](https://github.com/element-hq/synapse/issues/16942), [\#17064](https://github.com/element-hq/synapse/issues/17064), [\#17065](https://github.com/element-hq/synapse/issues/17065), [\#17066](https://github.com/element-hq/synapse/issues/17066)) -- Fix server notice rooms not always being created as unencrypted rooms, even when `encryption_enabled_by_default_for_room_type` is in use (server notices are always unencrypted). ([\#17033](https://github.com/element-hq/synapse/issues/17033)) -- Fix the `.m.rule.encrypted_room_one_to_one` and `.m.rule.room_one_to_one` default underride push rules being in the wrong order. Contributed by @Sumpy1. ([\#17043](https://github.com/element-hq/synapse/issues/17043)) - -### Internal Changes - -- Refactor auth chain fetching to reduce duplication. ([\#17044](https://github.com/element-hq/synapse/issues/17044)) -- Improve database performance by adding a missing index to `access_tokens.refresh_token_id`. ([\#17045](https://github.com/element-hq/synapse/issues/17045), [\#17054](https://github.com/element-hq/synapse/issues/17054)) -- Improve database performance by reducing number of receipts fetched when sending push notifications. ([\#17049](https://github.com/element-hq/synapse/issues/17049)) - - - -### Updates to locked dependencies - -* Bump packaging from 23.2 to 24.0. ([\#17027](https://github.com/element-hq/synapse/issues/17027)) -* Bump regex from 1.10.3 to 1.10.4. ([\#17028](https://github.com/element-hq/synapse/issues/17028)) -* Bump ruff from 0.3.2 to 0.3.5. ([\#17060](https://github.com/element-hq/synapse/issues/17060)) -* Bump serde_json from 1.0.114 to 1.0.115. ([\#17041](https://github.com/element-hq/synapse/issues/17041)) -* Bump types-pillow from 10.2.0.20240125 to 10.2.0.20240406. ([\#17061](https://github.com/element-hq/synapse/issues/17061)) -* Bump types-requests from 2.31.0.20240125 to 2.31.0.20240406. ([\#17063](https://github.com/element-hq/synapse/issues/17063)) -* Bump typing-extensions from 4.9.0 to 4.11.0. ([\#17062](https://github.com/element-hq/synapse/issues/17062)) - -# Synapse 1.104.0 (2024-04-02) - -### Bugfixes - -- Fix regression when using OIDC provider. Introduced in v1.104.0rc1. ([\#17031](https://github.com/element-hq/synapse/issues/17031)) - - -# Synapse 1.104.0rc1 (2024-03-26) - -### Features - -- Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) -- Add support for OIDC provider returning JWT. ([\#16972](https://github.com/element-hq/synapse/issues/16972), [\#17031](https://github.com/element-hq/synapse/issues/17031)) - -### Bugfixes - -- Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. ([\#16925](https://github.com/element-hq/synapse/issues/16925)) -- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16949](https://github.com/element-hq/synapse/issues/16949)) -- Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. ([\#16990](https://github.com/element-hq/synapse/issues/16990)) -- Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. ([\#17010](https://github.com/element-hq/synapse/issues/17010)) - -### Updates to the Docker image - -- Updated `start.py` to generate config using the correct user ID when running as root (fixes [\#16824](https://github.com/element-hq/synapse/issues/16824), [\#15202](https://github.com/element-hq/synapse/issues/15202)). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) - -### Improved Documentation - -- Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. ([\#16892](https://github.com/element-hq/synapse/issues/16892)) -- Minor grammatical corrections to the upgrade documentation. ([\#16965](https://github.com/element-hq/synapse/issues/16965)) -- Fix the sort order for the documentation version picker, so that newer releases appear above older ones. ([\#16966](https://github.com/element-hq/synapse/issues/16966)) -- Remove recommendation for a specific poetry version from contributing guide. ([\#17002](https://github.com/element-hq/synapse/issues/17002)) - -### Internal Changes - -- Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) -- Update power level default for public rooms. ([\#16907](https://github.com/element-hq/synapse/issues/16907)) -- Improve event validation. ([\#16908](https://github.com/element-hq/synapse/issues/16908)) -- Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) -- Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) -- Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) -- Specify IP subnets in canonical form. ([\#16953](https://github.com/element-hq/synapse/issues/16953)) -- As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. ([\#16974](https://github.com/element-hq/synapse/issues/16974)) -- Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. ([\#16985](https://github.com/element-hq/synapse/issues/16985)) -- Raise poetry-core version cap to 1.9.0. ([\#16986](https://github.com/element-hq/synapse/issues/16986)) -- Patch the db conn pool sooner in tests. ([\#17017](https://github.com/element-hq/synapse/issues/17017)) - - - -### Updates to locked dependencies - -* Bump anyhow from 1.0.80 to 1.0.81. ([\#17009](https://github.com/element-hq/synapse/issues/17009)) -* Bump black from 23.10.1 to 24.2.0. ([\#16936](https://github.com/element-hq/synapse/issues/16936)) -* Bump cryptography from 41.0.7 to 42.0.5. ([\#16958](https://github.com/element-hq/synapse/issues/16958)) -* Bump dawidd6/action-download-artifact from 3.1.1 to 3.1.2. ([\#16960](https://github.com/element-hq/synapse/issues/16960)) -* Bump dawidd6/action-download-artifact from 3.1.2 to 3.1.4. ([\#17008](https://github.com/element-hq/synapse/issues/17008)) -* Bump jinja2 from 3.1.2 to 3.1.3. ([\#17005](https://github.com/element-hq/synapse/issues/17005)) -* Bump log from 0.4.20 to 0.4.21. ([\#16977](https://github.com/element-hq/synapse/issues/16977)) -* Bump mypy from 1.5.1 to 1.8.0. ([\#16901](https://github.com/element-hq/synapse/issues/16901)) -* Bump netaddr from 0.9.0 to 1.2.1. ([\#17006](https://github.com/element-hq/synapse/issues/17006)) -* Bump pydantic from 2.6.0 to 2.6.4. ([\#17004](https://github.com/element-hq/synapse/issues/17004)) -* Bump pyo3 from 0.20.2 to 0.20.3. ([\#16962](https://github.com/element-hq/synapse/issues/16962)) -* Bump ruff from 0.1.14 to 0.3.2. ([\#16994](https://github.com/element-hq/synapse/issues/16994)) -* Bump serde from 1.0.196 to 1.0.197. ([\#16963](https://github.com/element-hq/synapse/issues/16963)) -* Bump serde_json from 1.0.113 to 1.0.114. ([\#16961](https://github.com/element-hq/synapse/issues/16961)) -* Bump types-jsonschema from 4.21.0.20240118 to 4.21.0.20240311. ([\#17007](https://github.com/element-hq/synapse/issues/17007)) -* Bump types-psycopg2 from 2.9.21.16 to 2.9.21.20240311. ([\#16995](https://github.com/element-hq/synapse/issues/16995)) -* Bump types-pyopenssl from 23.3.0.0 to 24.0.0.20240311. ([\#17003](https://github.com/element-hq/synapse/issues/17003)) - -# Synapse 1.103.0 (2024-03-19) - -No significant changes since 1.103.0rc1. - - - - -# Synapse 1.103.0rc1 (2024-03-12) - -### Features - -- Add a new [List Accounts v3](https://element-hq.github.io/synapse/v1.103/admin_api/user_admin_api.html#list-accounts-v3) Admin API with improved deactivated user filtering capabilities. ([\#16874](https://github.com/element-hq/synapse/issues/16874)) -- Include `Retry-After` header by default per [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041). Contributed by @clokep. ([\#16947](https://github.com/element-hq/synapse/issues/16947)) - -### Bugfixes - -- Fix joining remote rooms when a module uses the `on_new_event` callback. This callback may now pass partial state events instead of the full state for remote rooms. Introduced in v1.76.0. ([\#16973](https://github.com/element-hq/synapse/issues/16973)) -- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. Contributed by @ggogel. ([\#16968](https://github.com/element-hq/synapse/issues/16968)) - -### Improved Documentation - -- Add HAProxy example for single port operation to reverse proxy documentation. Contributed by Georg Pfuetzenreuter (@tacerus). ([\#16768](https://github.com/element-hq/synapse/issues/16768)) -- Improve the documentation around running Complement tests with new configuration parameters. ([\#16946](https://github.com/element-hq/synapse/issues/16946)) -- Add docs on upgrading from a very old version. ([\#16951](https://github.com/element-hq/synapse/issues/16951)) - - -### Updates to locked dependencies - -* Bump JasonEtco/create-an-issue from 2.9.1 to 2.9.2. ([\#16934](https://github.com/element-hq/synapse/issues/16934)) -* Bump anyhow from 1.0.79 to 1.0.80. ([\#16935](https://github.com/element-hq/synapse/issues/16935)) -* Bump dawidd6/action-download-artifact from 3.0.0 to 3.1.1. ([\#16933](https://github.com/element-hq/synapse/issues/16933)) -* Bump furo from 2023.9.10 to 2024.1.29. ([\#16939](https://github.com/element-hq/synapse/issues/16939)) -* Bump pyopenssl from 23.3.0 to 24.0.0. ([\#16937](https://github.com/element-hq/synapse/issues/16937)) -* Bump types-netaddr from 0.10.0.20240106 to 1.2.0.20240219. ([\#16938](https://github.com/element-hq/synapse/issues/16938)) - - -# Synapse 1.102.0 (2024-03-05) - -### Bugfixes - -- Revert https://github.com/element-hq/synapse/pull/16756, which caused incorrect notification counts on mobile clients since v1.100.0. ([\#16979](https://github.com/element-hq/synapse/issues/16979)) - - -# Synapse 1.102.0rc1 (2024-02-20) - -### Features - -- A metric was added for emails sent by Synapse, broken down by type: `synapse_emails_sent_total`. Contributed by Remi Rampin. ([\#16881](https://github.com/element-hq/synapse/issues/16881)) - -### Bugfixes - -- Do not send multiple concurrent requests for keys for the same server. ([\#16894](https://github.com/element-hq/synapse/issues/16894)) -- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. ([\#16903](https://github.com/element-hq/synapse/issues/16903)) -- Always prefer unthreaded receipt when >1 exist ([MSC4102](https://github.com/matrix-org/matrix-spec-proposals/pull/4102)). ([\#16927](https://github.com/element-hq/synapse/issues/16927)) - -### Improved Documentation - -- Fix a small typo in the Rooms section of the Admin API documentation. Contributed by @RainerZufall187. ([\#16857](https://github.com/element-hq/synapse/issues/16857)) - -### Internal Changes - -- Don't invalidate the entire event cache when we purge history. ([\#16905](https://github.com/element-hq/synapse/issues/16905)) -- Add experimental config option to not send device list updates for specific users. ([\#16909](https://github.com/element-hq/synapse/issues/16909)) -- Fix incorrect docker hub link in release script. ([\#16910](https://github.com/element-hq/synapse/issues/16910)) - - - -### Updates to locked dependencies - -* Bump attrs from 23.1.0 to 23.2.0. ([\#16899](https://github.com/element-hq/synapse/issues/16899)) -* Bump bcrypt from 4.0.1 to 4.1.2. ([\#16900](https://github.com/element-hq/synapse/issues/16900)) -* Bump pygithub from 2.1.1 to 2.2.0. ([\#16902](https://github.com/element-hq/synapse/issues/16902)) -* Bump sentry-sdk from 1.40.0 to 1.40.3. ([\#16898](https://github.com/element-hq/synapse/issues/16898)) - -# Synapse 1.101.0 (2024-02-13) - -### Bugfixes - -- Fix performance regression when fetching auth chains from the DB. Introduced in v1.100.0. ([\#16893](https://github.com/element-hq/synapse/issues/16893)) - - - - -# Synapse 1.101.0rc1 (2024-02-06) - -### Improved Documentation - -- Fix broken links in the documentation. ([\#16853](https://github.com/element-hq/synapse/issues/16853)) -- Update MacOS installation instructions to mention that libicu is optional. ([\#16854](https://github.com/element-hq/synapse/issues/16854)) -- The version picker now correctly lists versions after `v1.98.0`. ([\#16880](https://github.com/element-hq/synapse/issues/16880)) - -### Internal Changes - -- Add support for stabilised [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981) that adds a `recurse` parameter on the `/relations` API. ([\#16842](https://github.com/element-hq/synapse/issues/16842)) - - - -### Updates to locked dependencies - -* Bump dorny/paths-filter from 2 to 3. ([\#16869](https://github.com/element-hq/synapse/issues/16869)) -* Bump gitpython from 3.1.40 to 3.1.41. ([\#16850](https://github.com/element-hq/synapse/issues/16850)) -* Bump hiredis from 2.2.3 to 2.3.2. ([\#16862](https://github.com/element-hq/synapse/issues/16862)) -* Bump jsonschema from 4.20.0 to 4.21.1. ([\#16887](https://github.com/element-hq/synapse/issues/16887)) -* Bump lxml-stubs from 0.4.0 to 0.5.1. ([\#16885](https://github.com/element-hq/synapse/issues/16885)) -* Bump mypy-zope from 1.0.1 to 1.0.3. ([\#16865](https://github.com/element-hq/synapse/issues/16865)) -* Bump phonenumbers from 8.13.26 to 8.13.29. ([\#16868](https://github.com/element-hq/synapse/issues/16868)) -* Bump pydantic from 2.5.3 to 2.6.0. ([\#16888](https://github.com/element-hq/synapse/issues/16888)) -* Bump sentry-sdk from 1.39.1 to 1.40.0. ([\#16889](https://github.com/element-hq/synapse/issues/16889)) -* Bump serde from 1.0.195 to 1.0.196. ([\#16867](https://github.com/element-hq/synapse/issues/16867)) -* Bump serde_json from 1.0.111 to 1.0.113. ([\#16866](https://github.com/element-hq/synapse/issues/16866)) -* Bump sigstore/cosign-installer from 3.3.0 to 3.4.0. ([\#16890](https://github.com/element-hq/synapse/issues/16890)) -* Bump types-pillow from 10.1.0.2 to 10.2.0.20240125. ([\#16864](https://github.com/element-hq/synapse/issues/16864)) -* Bump types-requests from 2.31.0.10 to 2.31.0.20240125. ([\#16886](https://github.com/element-hq/synapse/issues/16886)) -* Bump types-setuptools from 69.0.0.0 to 69.0.0.20240125. ([\#16863](https://github.com/element-hq/synapse/issues/16863)) - -# Synapse 1.100.0 (2024-01-30) - -No significant changes since 1.100.0rc3. - - - - -# Synapse 1.100.0rc3 (2024-01-24) - -### Bugfixes - -- Fix database performance regression due to changing Postgres table statistics. Introduced in v1.100.0rc1. ([\#16849](https://github.com/element-hq/synapse/issues/16849)) - - - - -# Synapse 1.100.0rc2 (2024-01-24) - -This version is the same as 1.100.0rc1 but with fixes to the release process. - -### Internal Changes - -- Downgrade the `download-artifact` and `upload-artifact` actions to v3 due to breaking changes. ([\#16847](https://github.com/element-hq/synapse/issues/16847)) - - -# Synapse 1.100.0rc1 (2024-01-23) - -*This version was never released to PyPI or the Debian repository due to failures in the automatic part of the release process.* - -### Features - -- Advertise experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) through `/_matrix/clients/versions` if enabled. Contributed by @hanadi92. ([\#16787](https://github.com/element-hq/synapse/issues/16787)) - -### Bugfixes - -- Handle wildcard type filters properly for room messages endpoint. Contributed by Mo Balaa. ([\#14984](https://github.com/element-hq/synapse/issues/14984)) - -### Improved Documentation - -- Add a link to the "Request log format" explainer on the "Logging sample config" documentation page. ([\#16778](https://github.com/element-hq/synapse/issues/16778)) -- Fix broken links in issue templates and documentation. ([\#16810](https://github.com/element-hq/synapse/issues/16810)) -- NGINX listen http2 deprecation in documentation template for reverse proxy. ([\#16831](https://github.com/element-hq/synapse/issues/16831)) - -### Internal Changes - -- Faster partial join to room with complex auth graph. ([\#7](https://github.com/element-hq/synapse/issues/7)) -- Improve DB performance of calculating badge counts for push. ([\#16756](https://github.com/element-hq/synapse/issues/16756)) -- Split up deleting devices into batches. ([\#16766](https://github.com/element-hq/synapse/issues/16766)) -- Remove CI check for sign-off as we require a CLA signature instead. ([\#16776](https://github.com/element-hq/synapse/issues/16776)) -- Ensure CI fails when linting fails to make sure auto-merge does the correct thing. ([\#16781](https://github.com/element-hq/synapse/issues/16781)) -- Faster load recents for sync by reducing amount of state pulled out. ([\#16783](https://github.com/element-hq/synapse/issues/16783)) -- Reduce amount of state pulled out when querying federation hierachy. ([\#16785](https://github.com/element-hq/synapse/issues/16785)) -- Pull less state out of the DB when we retry fetching old events during backfill. ([\#16788](https://github.com/element-hq/synapse/issues/16788)) -- Optimize query for fetching to-device messages in `/sync`. ([\#16805](https://github.com/element-hq/synapse/issues/16805)) -- Reject OIDC config when `client_secret` isn't specified, but the auth method requires one. ([\#16806](https://github.com/element-hq/synapse/issues/16806)) -- Allow room creation but not publishing to continue if room publication rules are violated when creating - a new room. ([\#16811](https://github.com/element-hq/synapse/issues/16811)) -- Bump minimum supported Rust version to 1.65.0. ([\#16818](https://github.com/element-hq/synapse/issues/16818)) -- Fixup copyright lines in file headers after the licensing change. ([\#16820](https://github.com/element-hq/synapse/issues/16820)) -- Add a `--generate-only` option to the internal configuration/launch script for Complement. ([\#16828](https://github.com/element-hq/synapse/issues/16828)) -- Preparatory work for tweaking performance of auth chain lookups. ([\#16833](https://github.com/element-hq/synapse/issues/16833)) -- Speed up e2e device keys queries for bot accounts. ([\#16841](https://github.com/element-hq/synapse/issues/16841)) - -### Updates to locked dependencies - -* Bump actions/cache from 3 to 4. ([\#16832](https://github.com/element-hq/synapse/issues/16832)) -* Bump actions/download-artifact from 3 to 4. ([\#16795](https://github.com/element-hq/synapse/issues/16795)) -* Bump actions/upload-artifact from 3 to 4. ([\#16796](https://github.com/element-hq/synapse/issues/16796)) -* Bump anyhow from 1.0.75 to 1.0.79. ([\#16789](https://github.com/element-hq/synapse/issues/16789)) -* Bump authlib from 1.2.1 to 1.3.0. ([\#16801](https://github.com/element-hq/synapse/issues/16801)) -* Bump dawidd6/action-download-artifact from 2.28.0 to 3.0.0. ([\#16794](https://github.com/element-hq/synapse/issues/16794)) -* Bump immutabledict from 4.0.0 to 4.1.0. ([\#16812](https://github.com/element-hq/synapse/issues/16812)) -* Bump isort from 5.13.1 to 5.13.2. ([\#16835](https://github.com/element-hq/synapse/issues/16835)) -* Bump lxml from 4.9.3 to 5.1.0. ([\#16813](https://github.com/element-hq/synapse/issues/16813)) -* Bump pillow from 10.1.0 to 10.2.0. ([\#16802](https://github.com/element-hq/synapse/issues/16802)) -* Bump pydantic from 2.5.2 to 2.5.3. ([\#16836](https://github.com/element-hq/synapse/issues/16836)) -* Bump pyo3 from 0.20.0 to 0.20.2. ([\#16791](https://github.com/element-hq/synapse/issues/16791)) -* Bump regex from 1.9.6 to 1.10.3. ([\#16837](https://github.com/element-hq/synapse/issues/16837)) -* Bump ruff from 0.1.13 to 0.1.14. ([\#16838](https://github.com/element-hq/synapse/issues/16838)) -* Bump ruff from 0.1.7 to 0.1.13. ([\#16814](https://github.com/element-hq/synapse/issues/16814)) -* Bump sentry-sdk from 1.35.0 to 1.39.1. ([\#16799](https://github.com/element-hq/synapse/issues/16799)) -* Bump serde_json from 1.0.108 to 1.0.111. ([\#16792](https://github.com/element-hq/synapse/issues/16792)) -* Bump service-identity from 23.1.0 to 24.1.0. ([\#16816](https://github.com/element-hq/synapse/issues/16816)) -* Bump types-commonmark from 0.9.2.4 to 0.9.2.20240106. ([\#16797](https://github.com/element-hq/synapse/issues/16797)) -* Bump types-jsonschema from 4.20.0.0 to 4.20.0.20240105. ([\#16800](https://github.com/element-hq/synapse/issues/16800)) -* Bump types-jsonschema from 4.20.0.20240105 to 4.21.0.20240118. ([\#16834](https://github.com/element-hq/synapse/issues/16834)) -* Bump types-netaddr from 0.9.0.1 to 0.10.0.20240106. ([\#16839](https://github.com/element-hq/synapse/issues/16839)) -* Bump typing-extensions from 4.8.0 to 4.9.0. ([\#16815](https://github.com/element-hq/synapse/issues/16815)) - - -# Synapse 1.99.0 (2024-01-16) - -Synapse 1.99.0 is the first Synapse release under an AGPLv3.0 licence (with CLA to enable Element to sell AGPL -exceptions). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - - https://element.io/blog/synapse-now-lives-at-github-com-element-hq-synapse/ - -No significant changes since 1.99.0rc1. - - -# Synapse 1.99.0rc1 (2024-01-09) - -### Features - -- Add [config options](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#server_notices) to set the avatar and the topic of the server notices room, as well as the avatar of the server notices user. ([\#16679](https://github.com/matrix-org/synapse/issues/16679)) -- Add config option [`email.notif_delay_before_mail`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#email) to tweak the delay before an email is sent following a notification. ([\#16696](https://github.com/matrix-org/synapse/issues/16696)) -- Add new configuration option [`sentry.environment`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#sentry) for improved system monitoring. Contributed by @zeeshanrafiqrana. ([\#16738](https://github.com/matrix-org/synapse/issues/16738)) -- Filter out rooms from the room directory being served to other homeservers when those rooms block that homeserver by their Access Control Lists. ([\#16759](https://github.com/element-hq/synapse/issues/16759)) - -### Bugfixes - -- Fix a long-standing bug where the signing keys generated by Synapse were world-readable. Contributed by Fabian Klemp. ([\#16740](https://github.com/matrix-org/synapse/issues/16740)) -- Fix email verification redirection. Contributed by Fadhlan Ridhwanallah. ([\#16761](https://github.com/element-hq/synapse/issues/16761)) -- Fixed a bug that prevented users from being queried by display name if it contains non-ASCII characters. ([\#16767](https://github.com/element-hq/synapse/issues/16767)) -- Allow reactivate user without password with Admin API in some edge cases. ([\#16770](https://github.com/element-hq/synapse/issues/16770)) -- Adds the `recursion_depth` parameter to the response of the /relations endpoint if MSC3981 recursion is being performed. ([\#16775](https://github.com/element-hq/synapse/issues/16775)) - -### Improved Documentation - -- Added version picker for Synapse documentation. Contributed by @Dmytro27Ind. ([\#16533](https://github.com/matrix-org/synapse/issues/16533)) -- Clarify that `password_config.enabled: "only_for_reauth"` does not allow new logins to be created using password auth. ([\#16737](https://github.com/matrix-org/synapse/issues/16737)) -- Remove value from header in configuration documentation for `refresh_token_lifetime`. ([\#16763](https://github.com/element-hq/synapse/issues/16763)) -- Add another custom statistics collection server to the documentation. Contributed by @loelkes. ([\#16769](https://github.com/element-hq/synapse/issues/16769)) - -### Internal Changes - -- Remove run-once workflow after adding the version picker to the documentation. ([\#9453](https://github.com/element-hq/synapse/issues/9453)) -- Update the implementation of [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965) (OIDC Provider discovery). ([\#16726](https://github.com/matrix-org/synapse/issues/16726)) -- Move the rust stubs inline for better IDE integration. ([\#16757](https://github.com/element-hq/synapse/issues/16757)) -- Fix sample config doc CI. ([\#16758](https://github.com/element-hq/synapse/issues/16758)) -- Simplify event internal metadata class. ([\#16762](https://github.com/element-hq/synapse/issues/16762), [\#16780](https://github.com/element-hq/synapse/issues/16780)) -- Sign the published docker image using [cosign](https://docs.sigstore.dev/). ([\#16774](https://github.com/element-hq/synapse/issues/16774)) -- Port `EventInternalMetadata` class to Rust. ([\#16782](https://github.com/element-hq/synapse/issues/16782)) - - - -### Updates to locked dependencies - -* Bump actions/setup-go from 4 to 5. ([\#16749](https://github.com/matrix-org/synapse/issues/16749)) -* Bump actions/setup-python from 4 to 5. ([\#16748](https://github.com/matrix-org/synapse/issues/16748)) -* Bump immutabledict from 3.0.0 to 4.0.0. ([\#16743](https://github.com/matrix-org/synapse/issues/16743)) -* Bump isort from 5.12.0 to 5.13.0. ([\#16745](https://github.com/matrix-org/synapse/issues/16745)) -* Bump isort from 5.13.0 to 5.13.1. ([\#16752](https://github.com/matrix-org/synapse/issues/16752)) -* Bump pydantic from 2.5.1 to 2.5.2. ([\#16747](https://github.com/matrix-org/synapse/issues/16747)) -* Bump ruff from 0.1.6 to 0.1.7. ([\#16746](https://github.com/matrix-org/synapse/issues/16746)) -* Bump types-setuptools from 68.2.0.2 to 69.0.0.0. ([\#16744](https://github.com/matrix-org/synapse/issues/16744)) - -# Synapse 1.98.0 (2023-12-12) - -Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual announcements channels, -but we are striving to make this as seamless as possible. - - -No significant changes since 1.98.0rc1. - - - -# Synapse 1.98.0rc1 (2023-12-05) - -### Features - -- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) -- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback for when a user logs in. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) -- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) -- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) -- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) -- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) - -### Bugfixes - -- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) -- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) -- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) -- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) - -### Improved Documentation - -- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) -- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) -- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) -- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) - -### Internal Changes - -- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) -- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) -- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) -- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) -- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) -- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) -- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) -- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) -- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) -- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) - - -### Updates to locked dependencies - -* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) -* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) -* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) -* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) -* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) -* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) -* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) -* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) -* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) -* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) -* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) -* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) -* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) -* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) -* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) -* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) -* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) -* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) -* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) - -# Synapse 1.97.0 (2023-11-28) - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - - - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ - - https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual announcements channels, -but we are striving to make this as seamless as possible. - - -No significant changes since 1.97.0rc1. - - -# Synapse 1.97.0rc1 (2023-11-21) - -### Features - -- Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) -- Improve the performance of some operations in multi-worker deployments. ([\#16613](https://github.com/matrix-org/synapse/issues/16613), [\#16616](https://github.com/matrix-org/synapse/issues/16616)) - -### Bugfixes - -- Fix a long-standing bug where some queries updated the same row twice. Introduced in Synapse 1.57.0. ([\#16609](https://github.com/matrix-org/synapse/issues/16609)) -- Fix a long-standing bug where Synapse would not unbind third-party identifiers for Application Service users when deactivated and would not emit a compliant response. ([\#16617](https://github.com/matrix-org/synapse/issues/16617)) -- Fix sending out of order `POSITION` over replication, causing additional database load. ([\#16639](https://github.com/matrix-org/synapse/issues/16639)) - -### Improved Documentation - -- Note that the option [`outbound_federation_restricted_to`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#outbound_federation_restricted_to) was added in Synapse 1.89.0, and fix a nearby formatting error. ([\#16628](https://github.com/matrix-org/synapse/issues/16628)) -- Update parameter information for the `/timestamp_to_event` admin API. ([\#16631](https://github.com/matrix-org/synapse/issues/16631)) -- Provide an example for a common encrypted media response from the admin user media API and mention possible null values. ([\#16654](https://github.com/matrix-org/synapse/issues/16654)) - -### Internal Changes - -- Remove whole table locks on push rule modifications. Contributed by Nick @ Beeper (@fizzadar). ([\#16051](https://github.com/matrix-org/synapse/issues/16051)) -- Support reactor tick timings on more types of event loops. ([\#16532](https://github.com/matrix-org/synapse/issues/16532)) -- Improve type hints. ([\#16564](https://github.com/matrix-org/synapse/issues/16564), [\#16611](https://github.com/matrix-org/synapse/issues/16611), [\#16612](https://github.com/matrix-org/synapse/issues/16612)) -- Avoid executing no-op queries. ([\#16583](https://github.com/matrix-org/synapse/issues/16583)) -- Simplify persistence code to be per-room. ([\#16584](https://github.com/matrix-org/synapse/issues/16584)) -- Use standard SQL helpers in persistence code. ([\#16585](https://github.com/matrix-org/synapse/issues/16585)) -- Avoid updating the stream cache unnecessarily. ([\#16586](https://github.com/matrix-org/synapse/issues/16586)) -- Improve performance when using opentracing. ([\#16589](https://github.com/matrix-org/synapse/issues/16589)) -- Run push rule evaluator setup in parallel. ([\#16590](https://github.com/matrix-org/synapse/issues/16590)) -- Improve tests of the SQL generator. ([\#16596](https://github.com/matrix-org/synapse/issues/16596)) -- Use more generic database methods. ([\#16615](https://github.com/matrix-org/synapse/issues/16615)) -- Use `dbname` instead of the deprecated `database` connection parameter for psycopg2. ([\#16618](https://github.com/matrix-org/synapse/issues/16618)) -- Add an internal [Admin API endpoint](https://matrix-org.github.io/synapse/v1.97/usage/configuration/config_documentation.html#allow-replacing-master-cross-signing-key-without-user-interactive-auth) to temporarily grant the ability to update an existing cross-signing key without UIA. ([\#16634](https://github.com/matrix-org/synapse/issues/16634)) -- Improve references to GitHub issues. ([\#16637](https://github.com/matrix-org/synapse/issues/16637), [\#16638](https://github.com/matrix-org/synapse/issues/16638)) -- More efficiently handle no-op `POSITION` over replication. ([\#16640](https://github.com/matrix-org/synapse/issues/16640), [\#16655](https://github.com/matrix-org/synapse/issues/16655)) -- Speed up deleting of device messages when deleting a device. ([\#16643](https://github.com/matrix-org/synapse/issues/16643)) -- Speed up persisting large number of outliers. ([\#16649](https://github.com/matrix-org/synapse/issues/16649)) -- Reduce max concurrency of background tasks, reducing potential max DB load. ([\#16656](https://github.com/matrix-org/synapse/issues/16656), [\#16660](https://github.com/matrix-org/synapse/issues/16660)) -- Speed up purge room by adding an index to `event_push_summary`. ([\#16657](https://github.com/matrix-org/synapse/issues/16657)) - - - -### Updates to locked dependencies - -* Bump prometheus-client from 0.17.1 to 0.18.0. ([\#16626](https://github.com/matrix-org/synapse/issues/16626)) -* Bump pyicu from 2.11 to 2.12. ([\#16603](https://github.com/matrix-org/synapse/issues/16603)) -* Bump requests-toolbelt from 0.10.1 to 1.0.0. ([\#16659](https://github.com/matrix-org/synapse/issues/16659)) -* Bump ruff from 0.0.292 to 0.1.4. ([\#16600](https://github.com/matrix-org/synapse/issues/16600)) -* Bump serde from 1.0.190 to 1.0.192. ([\#16627](https://github.com/matrix-org/synapse/issues/16627)) -* Bump serde_json from 1.0.107 to 1.0.108. ([\#16604](https://github.com/matrix-org/synapse/issues/16604)) -* Bump setuptools-rust from 1.8.0 to 1.8.1. ([\#16601](https://github.com/matrix-org/synapse/issues/16601)) -* Bump towncrier from 23.6.0 to 23.11.0. ([\#16622](https://github.com/matrix-org/synapse/issues/16622)) -* Bump treq from 22.2.0 to 23.11.0. ([\#16623](https://github.com/matrix-org/synapse/issues/16623)) -* Bump twisted from 23.8.0 to 23.10.0. ([\#16588](https://github.com/matrix-org/synapse/issues/16588)) -* Bump types-bleach from 6.1.0.0 to 6.1.0.1. ([\#16624](https://github.com/matrix-org/synapse/issues/16624)) -* Bump types-jsonschema from 4.19.0.3 to 4.19.0.4. ([\#16599](https://github.com/matrix-org/synapse/issues/16599)) -* Bump types-pyopenssl from 23.2.0.2 to 23.3.0.0. ([\#16625](https://github.com/matrix-org/synapse/issues/16625)) -* Bump types-pyyaml from 6.0.12.11 to 6.0.12.12. ([\#16602](https://github.com/matrix-org/synapse/issues/16602)) - -# Synapse 1.96.1 (2023-11-17) - -Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for -proprietary dual licensing). You can read more about this here: - -* https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ -* https://element.io/blog/element-to-adopt-agplv3/ - -The Matrix.org Foundation copy of the project will be archived. Any changes needed -by server administrators will be communicated via our usual -[announcements channels](https://matrix.to/#/#homeowners:matrix.org), but we are -striving to make this as seamless as possible. - -This minor release was needed only because of CI-related trouble on [v1.96.0](https://github.com/matrix-org/synapse/releases/tag/v1.96.0), which was never released. - -### Internal Changes - -- Fix building of wheels in CI. ([\#16653](https://github.com/matrix-org/synapse/issues/16653)) - -# Synapse 1.96.0 (2023-11-16) - -### Bugfixes - -- Fix "'int' object is not iterable" error in `set_device_id_for_pushers` background update introduced in Synapse 1.95.0. ([\#16594](https://github.com/matrix-org/synapse/issues/16594)) - -# Synapse 1.96.0rc1 (2023-10-31) - -### Features - -- Add experimental support to allow multiple workers to write to receipts stream. ([\#16432](https://github.com/matrix-org/synapse/issues/16432)) -- Add a new module API for controller presence. ([\#16544](https://github.com/matrix-org/synapse/issues/16544)) -- Add a new module API callback that allows adding extra fields to events' unsigned section when sent down to clients. ([\#16549](https://github.com/matrix-org/synapse/issues/16549)) -- Improve the performance of claiming encryption keys. ([\#16565](https://github.com/matrix-org/synapse/issues/16565), [\#16570](https://github.com/matrix-org/synapse/issues/16570)) - -### Bugfixes - -- Fixed a bug in the example Grafana dashboard that prevents it from finding the correct datasource. Contributed by @MichaelSasser. ([\#16471](https://github.com/matrix-org/synapse/issues/16471)) -- Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. ([\#16473](https://github.com/matrix-org/synapse/issues/16473), [\#16557](https://github.com/matrix-org/synapse/issues/16557), [\#16561](https://github.com/matrix-org/synapse/issues/16561), [\#16578](https://github.com/matrix-org/synapse/issues/16578), [\#16580](https://github.com/matrix-org/synapse/issues/16580)) -- Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events. ([\#16485](https://github.com/matrix-org/synapse/issues/16485)) -- Fix a bug introduced in Synapse 1.41 where HTTP(S) forward proxy authorization would fail when using basic HTTP authentication with a long `username:password` string. ([\#16504](https://github.com/matrix-org/synapse/issues/16504)) -- Force TLS certificate verification in user registration script. ([\#16530](https://github.com/matrix-org/synapse/issues/16530)) -- Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. ([\#16540](https://github.com/matrix-org/synapse/issues/16540)) -- Fix ratelimiting of message sending when using workers, where the ratelimit would only be applied after most of the work has been done. ([\#16558](https://github.com/matrix-org/synapse/issues/16558)) -- Fix a long-standing bug where invited/knocking users would not leave during a room purge. ([\#16559](https://github.com/matrix-org/synapse/issues/16559)) - -### Improved Documentation - -- Improve documentation of presence router. ([\#16529](https://github.com/matrix-org/synapse/issues/16529)) -- Add a sentence to the [opentracing docs](https://matrix-org.github.io/synapse/latest/opentracing.html) on how you can have jaeger in a different place than synapse. ([\#16531](https://github.com/matrix-org/synapse/issues/16531)) -- Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. ([\#16541](https://github.com/matrix-org/synapse/issues/16541)) -- Pin the recommended poetry version in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16550](https://github.com/matrix-org/synapse/issues/16550)) -- Fix a broken link to the [client breakdown](https://matrix.org/ecosystem/clients/) in the README. ([\#16569](https://github.com/matrix-org/synapse/issues/16569)) - -### Internal Changes - -- Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479). ([\#16492](https://github.com/matrix-org/synapse/issues/16492)) -- Reduce memory allocations. ([\#16505](https://github.com/matrix-org/synapse/issues/16505)) -- Improve replication performance when purging rooms. ([\#16510](https://github.com/matrix-org/synapse/issues/16510)) -- Run tests against Python 3.12. ([\#16511](https://github.com/matrix-org/synapse/issues/16511)) -- Run trial & integration tests in continuous integration when `.ci` directory is modified. ([\#16512](https://github.com/matrix-org/synapse/issues/16512)) -- Remove duplicate call to mark remote server 'awake' when using a federation sending worker. ([\#16515](https://github.com/matrix-org/synapse/issues/16515)) -- Enable dirty runs on Complement CI, which is significantly faster. ([\#16520](https://github.com/matrix-org/synapse/issues/16520)) -- Stop deleting from an unused table. ([\#16521](https://github.com/matrix-org/synapse/issues/16521)) -- Improve type hints. ([\#16526](https://github.com/matrix-org/synapse/issues/16526), [\#16551](https://github.com/matrix-org/synapse/issues/16551)) -- Fix running unit tests on Twisted trunk. ([\#16528](https://github.com/matrix-org/synapse/issues/16528)) -- Reduce some spurious logging in worker mode. ([\#16555](https://github.com/matrix-org/synapse/issues/16555)) -- Stop porting a table in port db that we're going to nuke and rebuild anyway. ([\#16563](https://github.com/matrix-org/synapse/issues/16563)) -- Deal with warnings from running complement in CI. ([\#16567](https://github.com/matrix-org/synapse/issues/16567)) -- Allow building with `setuptools_rust` 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) - -### Updates to locked dependencies - -* Bump black from 23.10.0 to 23.10.1. ([\#16575](https://github.com/matrix-org/synapse/issues/16575)) -* Bump black from 23.9.1 to 23.10.0. ([\#16538](https://github.com/matrix-org/synapse/issues/16538)) -* Bump cryptography from 41.0.4 to 41.0.5. ([\#16572](https://github.com/matrix-org/synapse/issues/16572)) -* Bump gitpython from 3.1.37 to 3.1.40. ([\#16534](https://github.com/matrix-org/synapse/issues/16534)) -* Bump phonenumbers from 8.13.22 to 8.13.23. ([\#16576](https://github.com/matrix-org/synapse/issues/16576)) -* Bump pygithub from 1.59.1 to 2.1.1. ([\#16535](https://github.com/matrix-org/synapse/issues/16535)) -- Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. ([\#16539](https://github.com/matrix-org/synapse/issues/16539)) -* Bump serde from 1.0.189 to 1.0.190. ([\#16577](https://github.com/matrix-org/synapse/issues/16577)) -* Bump setuptools-rust from 1.7.0 to 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) -* Bump types-pillow from 10.0.0.3 to 10.1.0.0. ([\#16536](https://github.com/matrix-org/synapse/issues/16536)) -* Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15. ([\#16573](https://github.com/matrix-org/synapse/issues/16573)) -* Bump types-requests from 2.31.0.2 to 2.31.0.10. ([\#16537](https://github.com/matrix-org/synapse/issues/16537)) -* Bump urllib3 from 1.26.17 to 1.26.18. ([\#16516](https://github.com/matrix-org/synapse/issues/16516)) - -# Synapse 1.95.1 (2023-10-31) - -## Security advisory - -The following issue is fixed in 1.95.1. - -- [GHSA-mp92-3jfm-3575](https://github.com/matrix-org/synapse/security/advisories/GHSA-mp92-3jfm-3575) / [CVE-2023-43796](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-43796) — Moderate Severity - - Cached device information of remote users can be queried from Synapse. This can be used to enumerate the remote users known to a homeserver. - -See the advisory for more details. If you have any questions, email security@matrix.org. - - - -# Synapse 1.95.0 (2023-10-24) - -### Internal Changes - -- Build Debian packages for [Ubuntu 23.10 Mantic Minotaur](https://canonical.com/blog/canonical-releases-ubuntu-23-10-mantic-minotaur). ([\#16524](https://github.com/matrix-org/synapse/issues/16524)) - - -# Synapse 1.95.0rc1 (2023-10-17) - -### Bugfixes - -- Remove legacy unspecced `knock_state_events` field returned in some responses. ([\#16403](https://github.com/matrix-org/synapse/issues/16403)) -- Fix a bug introduced in Synapse 1.81.0 where an `AttributeError` would be raised when `_matrix/client/v3/account/whoami` is called over a unix socket. Contributed by @Sir-Photch. ([\#16404](https://github.com/matrix-org/synapse/issues/16404)) -- Properly return inline media when content types have parameters. ([\#16440](https://github.com/matrix-org/synapse/issues/16440)) -- Prevent the purging of large rooms from timing out when Postgres is in use. The timeout which causes this issue was introduced in Synapse 1.88.0. ([\#16455](https://github.com/matrix-org/synapse/issues/16455)) -- Improve the performance of purging rooms, particularly encrypted rooms. ([\#16457](https://github.com/matrix-org/synapse/issues/16457)) -- Fix a bug introduced in Synapse 1.59.0 where servers could be incorrectly marked as available after an error response was received. ([\#16506](https://github.com/matrix-org/synapse/issues/16506)) - -### Improved Documentation - -- Document internal background update mechanism. ([\#16420](https://github.com/matrix-org/synapse/issues/16420)) -- Fix a typo in the sql for [useful SQL for admins document](https://matrix-org.github.io/synapse/latest/usage/administration/useful_sql_for_admins.html). ([\#16477](https://github.com/matrix-org/synapse/issues/16477)) - -### Internal Changes - -- Bump pyo3 from 0.17.1 to 0.19.2. ([\#16162](https://github.com/matrix-org/synapse/issues/16162)) -- Update registration of media repository URLs. ([\#16419](https://github.com/matrix-org/synapse/issues/16419)) -- Improve type hints. ([\#16421](https://github.com/matrix-org/synapse/issues/16421), [\#16468](https://github.com/matrix-org/synapse/issues/16468), [\#16469](https://github.com/matrix-org/synapse/issues/16469), [\#16507](https://github.com/matrix-org/synapse/issues/16507)) -- Refactor some code to simplify and better type receipts stream adjacent code. ([\#16426](https://github.com/matrix-org/synapse/issues/16426)) -- Factor out `MultiWriter` token from `RoomStreamToken`. ([\#16427](https://github.com/matrix-org/synapse/issues/16427)) -- Improve code comments. ([\#16428](https://github.com/matrix-org/synapse/issues/16428)) -- Reduce memory allocations. ([\#16429](https://github.com/matrix-org/synapse/issues/16429), [\#16431](https://github.com/matrix-org/synapse/issues/16431), [\#16433](https://github.com/matrix-org/synapse/issues/16433), [\#16434](https://github.com/matrix-org/synapse/issues/16434), [\#16438](https://github.com/matrix-org/synapse/issues/16438), [\#16444](https://github.com/matrix-org/synapse/issues/16444)) -- Remove unused method. ([\#16435](https://github.com/matrix-org/synapse/issues/16435)) -- Improve rate limiting logic. ([\#16441](https://github.com/matrix-org/synapse/issues/16441)) -- Do not block running of CI behind the check for sign-off on PRs. ([\#16454](https://github.com/matrix-org/synapse/issues/16454)) -- Update the release script to remind releaser to check for special release notes. ([\#16461](https://github.com/matrix-org/synapse/issues/16461)) -- Update complement.sh to match new public API shape. ([\#16466](https://github.com/matrix-org/synapse/issues/16466)) -- Clean up logging on event persister endpoints. ([\#16488](https://github.com/matrix-org/synapse/issues/16488)) -- Remove useless async job to delete device messages on sync, since we only deliver (and hence delete) up to 100 device messages at a time. ([\#16491](https://github.com/matrix-org/synapse/issues/16491)) - -### Updates to locked dependencies - -* Bump bleach from 6.0.0 to 6.1.0. ([\#16451](https://github.com/matrix-org/synapse/issues/16451)) -* Bump jsonschema from 4.19.0 to 4.19.1. ([\#16500](https://github.com/matrix-org/synapse/issues/16500)) -* Bump netaddr from 0.8.0 to 0.9.0. ([\#16453](https://github.com/matrix-org/synapse/issues/16453)) -* Bump packaging from 23.1 to 23.2. ([\#16497](https://github.com/matrix-org/synapse/issues/16497)) -* Bump pillow from 10.0.1 to 10.1.0. ([\#16498](https://github.com/matrix-org/synapse/issues/16498)) -* Bump psycopg2 from 2.9.8 to 2.9.9. ([\#16452](https://github.com/matrix-org/synapse/issues/16452)) -* Bump pyo3-log from 0.8.3 to 0.8.4. ([\#16495](https://github.com/matrix-org/synapse/issues/16495)) -* Bump ruff from 0.0.290 to 0.0.292. ([\#16449](https://github.com/matrix-org/synapse/issues/16449)) -* Bump sentry-sdk from 1.31.0 to 1.32.0. ([\#16496](https://github.com/matrix-org/synapse/issues/16496)) -* Bump serde from 1.0.188 to 1.0.189. ([\#16494](https://github.com/matrix-org/synapse/issues/16494)) -* Bump types-bleach from 6.0.0.4 to 6.1.0.0. ([\#16450](https://github.com/matrix-org/synapse/issues/16450)) -* Bump types-jsonschema from 4.17.0.10 to 4.19.0.3. ([\#16499](https://github.com/matrix-org/synapse/issues/16499)) - -# Synapse 1.94.0 (2023-10-10) - -No significant changes since 1.94.0rc1. -However, please take note of the security advisory that follows. - -## Security advisory - -The following issue is fixed in 1.94.0 (and RC). - -- [GHSA-5chr-wjw5-3gq4](https://github.com/matrix-org/synapse/security/advisories/GHSA-5chr-wjw5-3gq4) / [CVE-2023-45129](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-45129) — Moderate Severity - - A malicious server ACL event can impact performance temporarily or permanently leading to a persistent denial of service. - - Homeservers running on a closed federation (which presumably do not need to use server ACLs) are not affected. - -See the advisory for more details. If you have any questions, email security@matrix.org. - - -# Synapse 1.94.0rc1 (2023-10-03) - -### Features - -- Render plain, CSS, CSV, JSON and common image formats in the browser (inline) when requested through the /download endpoint. ([\#15988](https://github.com/matrix-org/synapse/issues/15988)) -- Add experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) to push all encrypted events to clients. ([\#16361](https://github.com/matrix-org/synapse/issues/16361)) -- Minor performance improvement when sending presence to federated servers. ([\#16385](https://github.com/matrix-org/synapse/issues/16385)) -- Minor performance improvement by caching server ACL checking. ([\#16360](https://github.com/matrix-org/synapse/issues/16360)) - -### Improved Documentation - -- Add developer documentation concerning gradual schema migrations with column alterations. ([\#15691](https://github.com/matrix-org/synapse/issues/15691)) -- Improve documentation of the user directory search algorithm. ([\#16320](https://github.com/matrix-org/synapse/issues/16320)) -- Fix rendering of user admin API documentation around deactivation. This was broken in Synapse 1.91.0. ([\#16355](https://github.com/matrix-org/synapse/issues/16355)) -- Update documentation around message retention policies. ([\#16382](https://github.com/matrix-org/synapse/issues/16382)) -- Add note to `federation_domain_whitelist` config option to clarify its usage. ([\#16416](https://github.com/matrix-org/synapse/issues/16416)) -- Improve legacy release notes. ([\#16418](https://github.com/matrix-org/synapse/issues/16418)) - -### Deprecations and Removals - -- Remove Python version from `/_synapse/admin/v1/server_version`. ([\#16380](https://github.com/matrix-org/synapse/issues/16380)) - -### Internal Changes - -- Avoid running CI steps when the files they check have not been changed. ([\#14745](https://github.com/matrix-org/synapse/issues/14745), [\#16387](https://github.com/matrix-org/synapse/issues/16387)) -- Improve type hints. ([\#14911](https://github.com/matrix-org/synapse/issues/14911), [\#16350](https://github.com/matrix-org/synapse/issues/16350), [\#16356](https://github.com/matrix-org/synapse/issues/16356), [\#16395](https://github.com/matrix-org/synapse/issues/16395)) -- Added support for pydantic v2 in addition to pydantic v1. Contributed by Maxwell G (@gotmax23). ([\#16332](https://github.com/matrix-org/synapse/issues/16332)) -- Get CI to check PRs have been signed-off. ([\#16348](https://github.com/matrix-org/synapse/issues/16348)) -- Add missing licence header. ([\#16359](https://github.com/matrix-org/synapse/issues/16359)) -- Improve type hints, and bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) -- Improve comments in `StateGroupBackgroundUpdateStore`. ([\#16383](https://github.com/matrix-org/synapse/issues/16383)) -- Update maturin configuration. ([\#16394](https://github.com/matrix-org/synapse/issues/16394)) -- Downgrade replication stream time out error log lines to warning. ([\#16401](https://github.com/matrix-org/synapse/issues/16401)) - -### Updates to locked dependencies - -* Bump actions/checkout from 3 to 4. ([\#16250](https://github.com/matrix-org/synapse/issues/16250)) -* Bump cryptography from 41.0.3 to 41.0.4. ([\#16362](https://github.com/matrix-org/synapse/issues/16362)) -* Bump dawidd6/action-download-artifact from 2.27.0 to 2.28.0. ([\#16374](https://github.com/matrix-org/synapse/issues/16374)) -* Bump docker/setup-buildx-action from 2 to 3. ([\#16375](https://github.com/matrix-org/synapse/issues/16375)) -* Bump gitpython from 3.1.35 to 3.1.37. ([\#16376](https://github.com/matrix-org/synapse/issues/16376)) -* Bump msgpack from 1.0.5 to 1.0.6. ([\#16377](https://github.com/matrix-org/synapse/issues/16377)) -* Bump msgpack from 1.0.6 to 1.0.7. ([\#16412](https://github.com/matrix-org/synapse/issues/16412)) -* Bump phonenumbers from 8.13.19 to 8.13.22. ([\#16413](https://github.com/matrix-org/synapse/issues/16413)) -* Bump psycopg2 from 2.9.7 to 2.9.8. ([\#16409](https://github.com/matrix-org/synapse/issues/16409)) -* Bump pydantic from 2.3.0 to 2.4.2. ([\#16410](https://github.com/matrix-org/synapse/issues/16410)) -* Bump regex from 1.9.5 to 1.9.6. ([\#16408](https://github.com/matrix-org/synapse/issues/16408)) -* Bump sentry-sdk from 1.30.0 to 1.31.0. ([\#16378](https://github.com/matrix-org/synapse/issues/16378)) -* Bump types-netaddr from 0.8.0.9 to 0.9.0.1. ([\#16411](https://github.com/matrix-org/synapse/issues/16411)) -* Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) -* Bump urllib3 from 1.26.15 to 1.26.17. ([\#16422](https://github.com/matrix-org/synapse/issues/16422)) - -# Synapse 1.93.0 (2023-09-26) - -No significant changes since 1.93.0rc1. - - -## Security advisory - -The following issues are fixed in 1.93.0 (and RCs). - -- [GHSA-4f74-84v3-j9q5](https://github.com/matrix-org/synapse/security/advisories/GHSA-4f74-84v3-j9q5) / [CVE-2023-41335](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-41335) — Low Severity - - Temporary storage of plaintext passwords during password changes. - -- [GHSA-7565-cq32-vx2x](https://github.com/matrix-org/synapse/security/advisories/GHSA-7565-cq32-vx2x) / [CVE-2023-42453](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-42453) — Low Severity - - Improper validation of receipts allows forged read receipts. - -See the advisories for more details. If you have any questions, email security@matrix.org. - - -# Synapse 1.93.0rc1 (2023-09-19) - -### Features - -- Add automatic purge after all users have forgotten a room. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) -- Restore room purge/shutdown after a Synapse restart. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) -- Support resolving homeservers using `matrix-fed` DNS SRV records from [MSC4040](https://github.com/matrix-org/matrix-spec-proposals/pull/4040). ([\#16137](https://github.com/matrix-org/synapse/issues/16137)) -- Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes. ([\#16219](https://github.com/matrix-org/synapse/issues/16219)) -- Add span information to requests sent to appservices. Contributed by MTRNord. ([\#16227](https://github.com/matrix-org/synapse/issues/16227)) -- Add the ability to enable/disable registrations when using CAS. Contributed by Aurélien Grimpard. ([\#16262](https://github.com/matrix-org/synapse/issues/16262)) -- Allow the `/notifications` endpoint to be routed to workers. ([\#16265](https://github.com/matrix-org/synapse/issues/16265)) -- Enable users to easily unsubscribe to notifications emails via the `List-Unsubscribe` header. ([\#16274](https://github.com/matrix-org/synapse/issues/16274)) -- Report whether a user is `locked` in the [List Accounts admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#list-accounts), and exclude locked users by default. ([\#16328](https://github.com/matrix-org/synapse/issues/16328)) - -### Bugfixes - -- Fix a long-standing bug where multi-device accounts could cause high load due to presence. ([\#16066](https://github.com/matrix-org/synapse/issues/16066), [\#16170](https://github.com/matrix-org/synapse/issues/16170), [\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172), [\#16174](https://github.com/matrix-org/synapse/issues/16174)) -- Fix a long-standing bug where appservices using [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) to receive `to_device` messages would only get messages for one user. ([\#16251](https://github.com/matrix-org/synapse/issues/16251)) -- Fix bug when using workers where Synapse could end up re-requesting the same remote device repeatedly. ([\#16252](https://github.com/matrix-org/synapse/issues/16252)) -- Fix long-standing bug where we kept re-requesting a remote server's key repeatedly, potentially causing delays in receiving events over federation. ([\#16257](https://github.com/matrix-org/synapse/issues/16257)) -- Avoid temporary storage of sensitive information. ([\#16272](https://github.com/matrix-org/synapse/issues/16272)) -- Fix bug introduced in Synapse 1.49.0 when using dehydrated devices ([MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697)) and refresh tokens. Contributed by Hanadi. ([\#16288](https://github.com/matrix-org/synapse/issues/16288)) -- Fix a long-standing bug where invalid receipts would be accepted. ([\#16327](https://github.com/matrix-org/synapse/issues/16327)) -- Use standard name for UTF-8 charset in emails. ([\#16329](https://github.com/matrix-org/synapse/issues/16329)) -- Don't try refetching device lists for users on remote hosts that are marked as "down". ([\#16298](https://github.com/matrix-org/synapse/issues/16298)) - -### Improved Documentation - -- Fix typos in the documentation. ([\#16282](https://github.com/matrix-org/synapse/issues/16282)) -- Link to the Alpine Linux community package for Synapse. ([\#16304](https://github.com/matrix-org/synapse/issues/16304)) -- Use string for `federation_client_minimum_tls_version` documentation examples. Contributed by @jcgruenhage. ([\#16353](https://github.com/matrix-org/synapse/issues/16353)) - -### Internal Changes - -- Allow modules to delete rooms. ([\#15997](https://github.com/matrix-org/synapse/issues/15997)) -- Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled. ([\#16090](https://github.com/matrix-org/synapse/issues/16090), [\#16263](https://github.com/matrix-org/synapse/issues/16263)) -- Fix type checking when using the new version of Twisted. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) -- Delete device messages asynchronously and in staged batches using the task scheduler. ([\#16240](https://github.com/matrix-org/synapse/issues/16240), [\#16311](https://github.com/matrix-org/synapse/issues/16311), [\#16312](https://github.com/matrix-org/synapse/issues/16312), [\#16313](https://github.com/matrix-org/synapse/issues/16313)) -- Bump minimum supported Rust version to 1.61.0. ([\#16248](https://github.com/matrix-org/synapse/issues/16248)) -- Update rust to version 1.71.1 in the nix development environment. ([\#16260](https://github.com/matrix-org/synapse/issues/16260)) -- Simplify server key storage. ([\#16261](https://github.com/matrix-org/synapse/issues/16261)) -- Reduce CPU overhead of change password endpoint. ([\#16264](https://github.com/matrix-org/synapse/issues/16264)) -- Stop purging from tables slated for removal. ([\#16273](https://github.com/matrix-org/synapse/issues/16273)) -- Improve type hints. ([\#16276](https://github.com/matrix-org/synapse/issues/16276), [\#16301](https://github.com/matrix-org/synapse/issues/16301), [\#16325](https://github.com/matrix-org/synapse/issues/16325), [\#16326](https://github.com/matrix-org/synapse/issues/16326)) -- Raise `setuptools_rust` version cap to 1.7.0. ([\#16277](https://github.com/matrix-org/synapse/issues/16277)) -- Fix using the new task scheduler causing lots of CPU to be used. ([\#16278](https://github.com/matrix-org/synapse/issues/16278)) -- Upgrade CI run of Python 3.12 from rc1 to rc2. ([\#16280](https://github.com/matrix-org/synapse/issues/16280)) -- Include values in SQL debug when using `execute_values` with Postgres. ([\#16281](https://github.com/matrix-org/synapse/issues/16281)) -- Enable additional linting checks. ([\#16283](https://github.com/matrix-org/synapse/issues/16283)) -- Refactor `receipts_graph` Postgres transactions to stop error messages. ([\#16299](https://github.com/matrix-org/synapse/issues/16299)) -- Small improvements to logging in replication code. ([\#16309](https://github.com/matrix-org/synapse/issues/16309)) -- Remove a reference cycle in background processes. ([\#16314](https://github.com/matrix-org/synapse/issues/16314)) -- Only use literal strings for background process names. ([\#16315](https://github.com/matrix-org/synapse/issues/16315)) -- Refactor `get_user_by_id`. ([\#16316](https://github.com/matrix-org/synapse/issues/16316)) -- Speed up task to delete to-device messages. ([\#16318](https://github.com/matrix-org/synapse/issues/16318)) -- Avoid patching code in tests. ([\#16349](https://github.com/matrix-org/synapse/issues/16349)) -- Test against PostgreSQL 16. ([\#16351](https://github.com/matrix-org/synapse/issues/16351)) - -### Updates to locked dependencies - -* Bump mypy from 1.4.1 to 1.5.1. ([\#16300](https://github.com/matrix-org/synapse/issues/16300)) -* Bump black from 23.7.0 to 23.9.1. ([\#16295](https://github.com/matrix-org/synapse/issues/16295)) -* Bump docker/build-push-action from 4 to 5. ([\#16336](https://github.com/matrix-org/synapse/issues/16336)) -* Bump docker/login-action from 2 to 3. ([\#16339](https://github.com/matrix-org/synapse/issues/16339)) -* Bump docker/metadata-action from 4 to 5. ([\#16337](https://github.com/matrix-org/synapse/issues/16337)) -* Bump docker/setup-qemu-action from 2 to 3. ([\#16338](https://github.com/matrix-org/synapse/issues/16338)) -* Bump furo from 2023.8.19 to 2023.9.10. ([\#16340](https://github.com/matrix-org/synapse/issues/16340)) -* Bump gitpython from 3.1.32 to 3.1.35. ([\#16267](https://github.com/matrix-org/synapse/issues/16267), [\#16279](https://github.com/matrix-org/synapse/issues/16279)) -* Bump mypy-zope from 1.0.0 to 1.0.1. ([\#16291](https://github.com/matrix-org/synapse/issues/16291)) -* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) -* Bump regex from 1.9.4 to 1.9.5. ([\#16233](https://github.com/matrix-org/synapse/issues/16233)) -* Bump ruff from 0.0.286 to 0.0.290. ([\#16342](https://github.com/matrix-org/synapse/issues/16342)) -* Bump serde_json from 1.0.105 to 1.0.107. ([\#16296](https://github.com/matrix-org/synapse/issues/16296), [\#16345](https://github.com/matrix-org/synapse/issues/16345)) -* Bump twisted from 22.10.0 to 23.8.0. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) -* Bump types-pillow from 10.0.0.2 to 10.0.0.3. ([\#16293](https://github.com/matrix-org/synapse/issues/16293)) -* Bump types-setuptools from 68.0.0.3 to 68.2.0.0. ([\#16292](https://github.com/matrix-org/synapse/issues/16292)) -* Bump typing-extensions from 4.7.1 to 4.8.0. ([\#16341](https://github.com/matrix-org/synapse/issues/16341)) - -# Synapse 1.92.3 (2023-09-18) - -This is again a security update targeted at mitigating [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863). -It turns out that libwebp is bundled statically in Pillow wheels so we need to update this dependency instead of -libwebp package at the OS level. - -Unlike what was advertised in 1.92.2 changelog this release also impacts PyPI wheels and Debian packages from matrix.org. - -We encourage admins to upgrade as soon as possible. - - -### Internal Changes - -- Pillow 10.0.1 is now mandatory because of libwebp CVE-2023-4863, since Pillow provides libwebp in the wheels. ([\#16347](https://github.com/matrix-org/synapse/issues/16347)) - -### Updates to locked dependencies - -* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) - -# Synapse 1.92.2 (2023-09-15) - -This is a Docker-only update to mitigate [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863), a critical vulnerability in `libwebp`. Server admins not using Docker should ensure that their `libwebp` is up to date (if installed). We encourage admins to upgrade as soon as possible. - - -### Updates to the Docker image - -- Update docker image to use Debian bookworm as the base. ([\#16324](https://github.com/matrix-org/synapse/issues/16324)) - - -# Synapse 1.92.1 (2023-09-12) - -This minor release was needed only because of CI-related trouble on [v1.92.0](https://github.com/matrix-org/synapse/releases/tag/v1.92.0), which was never released. - -### Internal Changes - -- Stop building Ubuntu Kinetic since it is EOL and repos seem to be dead. - - -# Synapse 1.92.0 (2023-09-12) - -This release includes the same [bugfix](https://github.com/matrix-org/synapse/issues/16258) as Synapse 1.91.2. - -This version was never released following a CI build failure, cf [v1.92.1 changelog](https://github.com/matrix-org/synapse/releases/tag/v1.92.1). - -### Bugfixes - -- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) - -### Internal Changes - -- Fix incorrect docstring for `Ratelimiter`. ([\#16255](https://github.com/matrix-org/synapse/issues/16255)) -- Update the release script to work on macOS. ([\#16266](https://github.com/matrix-org/synapse/issues/16266)) - - -# Synapse 1.91.2 (2023-09-06) - -### Bugfixes - -- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) - - -# Synapse 1.92.0rc1 (2023-09-05) - -### Features - -- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816)) -- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113)) -- Experimental support for [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041): return a `Retry-After` header with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136)) -- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218)) -- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223)) - -### Bugfixes - -- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155)) -- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185)) -- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205)) -- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210)) -- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211)) -- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221)) - -### Improved Documentation - -- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008)) -- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168)) -- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178)) - -### Internal Changes - -- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099)) -- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213)) -- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135)) -- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170)) -- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172)) -- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175)) -- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180)) -- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183)) -- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184)) -- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201)) -- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187)) -- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212)) -- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241)) - -### Updates to locked dependencies - -* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141)) -* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238)) -* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237)) -* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196)) -* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195)) -* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198)) -* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236)) -* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194)) -* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140)) -* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200)) -* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199)) - - -# Synapse 1.91.1 (2023-09-04) - -### Bugfixes - -- Fix a performance regression introduced in Synapse 1.91.0 where event persistence would cause an excessive linear growth in CPU usage. ([\#16220](https://github.com/matrix-org/synapse/issues/16220)) - - -# Synapse 1.91.0 (2023-08-30) - -No significant changes since 1.91.0rc1. - - -# Synapse 1.91.0rc1 (2023-08-23) - -### Features - -- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) -- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) -- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) -- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) -- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) - -### Bugfixes - -- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) -- Fix a long-standing bu in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) -- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) -- Filter out user agent references to the sliding sync proxy and rust-sdk from the user_daily_visits table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) -- User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) -- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) -- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) -- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) - -### Improved Documentation - -- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) - -### Internal Changes - -- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) -- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) -- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) -- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) -- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) -- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) -- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) -- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) -- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) -- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) -- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) -- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) -- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) -- MSC3861: allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) -- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) -- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) -- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) -- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) -- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) -- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) -- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) - -### Updates to locked dependencies - -* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) -* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) -* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) -* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) -* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) -* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) -* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) -* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) -* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) -* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) -* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) -* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) - -# Synapse 1.91.0rc1 (2023-08-23) - -### Features - -- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) -- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) -- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) -- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) - -### Bugfixes - -- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) -- Fix a long-standing bug in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) -- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) -- Filter out user agent references to the sliding sync proxy and rust-sdk from the `user_daily_visits` table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) -- User constent and third-party ID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) -- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) -- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) -- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) - -### Improved Documentation - -- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) - -### Internal Changes - -- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) -- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) -- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) -- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) -- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) -- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) -- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) -- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) -- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) -- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) -- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) -- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) -- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) -- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) -- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) -- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) -- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) -- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) -- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) -- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) -- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) -- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) - -### Updates to locked dependencies - -* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) -* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) -* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) -* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) -* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) -* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) -* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) -* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) -* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) -* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) -* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) -* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) - -# Synapse 1.90.0 (2023-08-15) - -No significant changes since 1.90.0rc1. - - -# Synapse 1.90.0rc1 (2023-08-08) - -### Features - -- Scope transaction IDs to devices (implement [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970)). ([\#15629](https://github.com/matrix-org/synapse/issues/15629)) -- Remove old rows from the `cache_invalidation_stream_by_instance` table automatically (this table is unused in SQLite). ([\#15868](https://github.com/matrix-org/synapse/issues/15868)) - -### Bugfixes - -- Fix a long-standing bug where purging history and paginating simultaneously could lead to database corruption when using workers. ([\#15791](https://github.com/matrix-org/synapse/issues/15791)) -- Fix a long-standing bug where profile endpoint returned a 404 when the user's display name was empty. ([\#16012](https://github.com/matrix-org/synapse/issues/16012)) -- Fix a long-standing bug where the `synapse_port_db` failed to configure sequences for application services and partial stated rooms. ([\#16043](https://github.com/matrix-org/synapse/issues/16043)) -- Fix long-standing bug with deletion in dehydrated devices v2. ([\#16046](https://github.com/matrix-org/synapse/issues/16046)) - -### Updates to the Docker image - -- Add `org.opencontainers.image.version` labels to Docker containers [published by Matrix.org](https://hub.docker.com/r/matrixdotorg/synapse). Contributed by Mo Balaa. ([\#15972](https://github.com/matrix-org/synapse/issues/15972), [\#16009](https://github.com/matrix-org/synapse/issues/16009)) - -### Improved Documentation - -- Add a internal documentation page describing the ["streams" used within Synapse](https://matrix-org.github.io/synapse/v1.90/development/synapse_architecture/streams.html). ([\#16015](https://github.com/matrix-org/synapse/issues/16015)) -- Clarify comment on the keys/upload over replication enpoint. ([\#16016](https://github.com/matrix-org/synapse/issues/16016)) -- Do not expose Admin API in caddy reverse proxy example. Contributed by @NilsIrl. ([\#16027](https://github.com/matrix-org/synapse/issues/16027)) - -### Deprecations and Removals - -- Remove support for legacy application service paths. ([\#15964](https://github.com/matrix-org/synapse/issues/15964)) -- Move support for application service query parameter authorization behind a configuration option. ([\#16017](https://github.com/matrix-org/synapse/issues/16017)) - -### Internal Changes - -- Update SQL queries to inline boolean parameters as supported in SQLite 3.27. ([\#15525](https://github.com/matrix-org/synapse/issues/15525)) -- Allow for the configuration of the backoff algorithm for federation destinations. ([\#15754](https://github.com/matrix-org/synapse/issues/15754)) -- Allow modules to check whether the current worker is configured to run background tasks. ([\#15991](https://github.com/matrix-org/synapse/issues/15991)) -- Update support for [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958) to match the latest revision of the MSC. ([\#15992](https://github.com/matrix-org/synapse/issues/15992)) -- Allow modules to schedule delayed background calls. ([\#15993](https://github.com/matrix-org/synapse/issues/15993)) -- Properly overwrite the `redacts` content-property for forwards-compatibility with room versions 1 through 10. ([\#16013](https://github.com/matrix-org/synapse/issues/16013)) -- Fix building the nix development environment on MacOS systems. ([\#16019](https://github.com/matrix-org/synapse/issues/16019)) -- Remove leading and trailing spaces when setting a display name. ([\#16031](https://github.com/matrix-org/synapse/issues/16031)) -- Combine duplicated code. ([\#16023](https://github.com/matrix-org/synapse/issues/16023)) -- Collect additional metrics from `ResponseCache` for eviction. ([\#16028](https://github.com/matrix-org/synapse/issues/16028)) -- Fix endpoint improperly declaring support for MSC3814. ([\#16068](https://github.com/matrix-org/synapse/issues/16068)) -- Drop backwards compat hack for event serialization. ([\#16069](https://github.com/matrix-org/synapse/issues/16069)) - -### Updates to locked dependencies - -* Update PyYAML to 6.0.1. ([\#16011](https://github.com/matrix-org/synapse/issues/16011)) -* Bump cryptography from 41.0.2 to 41.0.3. ([\#16048](https://github.com/matrix-org/synapse/issues/16048)) -* Bump furo from 2023.5.20 to 2023.7.26. ([\#16077](https://github.com/matrix-org/synapse/issues/16077)) -* Bump immutabledict from 2.2.4 to 3.0.0. ([\#16034](https://github.com/matrix-org/synapse/issues/16034)) -* Update certifi to 2023.7.22 and pygments to 2.15.1. ([\#16044](https://github.com/matrix-org/synapse/issues/16044)) -* Bump jsonschema from 4.18.3 to 4.19.0. ([\#16081](https://github.com/matrix-org/synapse/issues/16081)) -* Bump phonenumbers from 8.13.14 to 8.13.18. ([\#16076](https://github.com/matrix-org/synapse/issues/16076)) -* Bump regex from 1.9.1 to 1.9.3. ([\#16073](https://github.com/matrix-org/synapse/issues/16073)) -* Bump serde from 1.0.171 to 1.0.175. ([\#15982](https://github.com/matrix-org/synapse/issues/15982)) -* Bump serde from 1.0.175 to 1.0.179. ([\#16033](https://github.com/matrix-org/synapse/issues/16033)) -* Bump serde from 1.0.179 to 1.0.183. ([\#16074](https://github.com/matrix-org/synapse/issues/16074)) -* Bump serde_json from 1.0.103 to 1.0.104. ([\#16032](https://github.com/matrix-org/synapse/issues/16032)) -* Bump service-identity from 21.1.0 to 23.1.0. ([\#16038](https://github.com/matrix-org/synapse/issues/16038)) -* Bump types-commonmark from 0.9.2.3 to 0.9.2.4. ([\#16037](https://github.com/matrix-org/synapse/issues/16037)) -* Bump types-jsonschema from 4.17.0.8 to 4.17.0.10. ([\#16036](https://github.com/matrix-org/synapse/issues/16036)) -* Bump types-netaddr from 0.8.0.8 to 0.8.0.9. ([\#16035](https://github.com/matrix-org/synapse/issues/16035)) -* Bump types-opentracing from 2.4.10.5 to 2.4.10.6. ([\#16078](https://github.com/matrix-org/synapse/issues/16078)) -* Bump types-setuptools from 68.0.0.0 to 68.0.0.3. ([\#16079](https://github.com/matrix-org/synapse/issues/16079)) - -# Synapse 1.89.0 (2023-08-01) - -No significant changes since 1.89.0rc1. - - -# Synapse 1.89.0rc1 (2023-07-25) - -### Features - -- Add Unix Socket support for HTTP Replication Listeners. [Document and provide usage instructions](https://matrix-org.github.io/synapse/v1.89/usage/configuration/config_documentation.html#listeners) for utilizing Unix sockets in Synapse. Contributed by Jason Little. ([\#15708](https://github.com/matrix-org/synapse/issues/15708), [\#15924](https://github.com/matrix-org/synapse/issues/15924)) -- Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009). ([\#15911](https://github.com/matrix-org/synapse/issues/15911)) -- Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820). ([\#15912](https://github.com/matrix-org/synapse/issues/15912)) -- Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`. ([\#15913](https://github.com/matrix-org/synapse/issues/15913), [\#15969](https://github.com/matrix-org/synapse/issues/15969)) -- Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly, H-Shay and poljar. ([\#15929](https://github.com/matrix-org/synapse/issues/15929)) - -### Bugfixes - -- Fix a long-standing bug where remote invites weren't correctly pushed. ([\#15820](https://github.com/matrix-org/synapse/issues/15820)) -- Fix background schema updates failing over a large upgrade gap. ([\#15887](https://github.com/matrix-org/synapse/issues/15887)) -- Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting. ([\#15925](https://github.com/matrix-org/synapse/issues/15925)) -- Fixed deploy annotations in the provided Grafana dashboard config, so that it shows for any homeserver and not just matrix.org. Contributed by @wrjlewis. ([\#15957](https://github.com/matrix-org/synapse/issues/15957)) -- Ensure a long state res does not starve CPU by occasionally yielding to the reactor. ([\#15960](https://github.com/matrix-org/synapse/issues/15960)) -- Properly handle redactions of creation events. ([\#15973](https://github.com/matrix-org/synapse/issues/15973)) -- Fix a bug where resyncing stale device lists could block responding to federation transactions, and thus delay receiving new data from the remote server. ([\#15975](https://github.com/matrix-org/synapse/issues/15975)) - -### Improved Documentation - -- Better clarify how to run a worker instance (pass both configs). ([\#15921](https://github.com/matrix-org/synapse/issues/15921)) -- Improve [the documentation](https://matrix-org.github.io/synapse/v1.89/admin_api/user_admin_api.html#login-as-a-user) for the login as a user admin API. ([\#15938](https://github.com/matrix-org/synapse/issues/15938)) -- Fix broken Arch Linux package link. Contributed by @SnipeXandrej. ([\#15981](https://github.com/matrix-org/synapse/issues/15981)) - -### Deprecations and Removals - -- Remove support for calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15928](https://github.com/matrix-org/synapse/issues/15928)) - -### Internal Changes - -- Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it. ([\#15884](https://github.com/matrix-org/synapse/issues/15884)) -- Document which Python version runs on a given Linux distribution so we can more easily clean up later. ([\#15909](https://github.com/matrix-org/synapse/issues/15909)) -- Add details to warning in log when we fail to fetch an alias. ([\#15922](https://github.com/matrix-org/synapse/issues/15922)) -- Remove unneeded `__init__`. ([\#15926](https://github.com/matrix-org/synapse/issues/15926)) -- Fix bug with read/write lock implementation. This is currently unused so has no observable effects. ([\#15933](https://github.com/matrix-org/synapse/issues/15933), [\#15958](https://github.com/matrix-org/synapse/issues/15958)) -- Unbreak the nix development environment by pinning the Rust version to 1.70.0. ([\#15940](https://github.com/matrix-org/synapse/issues/15940)) -- Update presence metrics to differentiate remote vs local users. ([\#15952](https://github.com/matrix-org/synapse/issues/15952)) -- Stop reading from column `user_id` of table `profiles`. ([\#15955](https://github.com/matrix-org/synapse/issues/15955)) -- Build packages for Debian Trixie. ([\#15961](https://github.com/matrix-org/synapse/issues/15961)) -- Reduce the amount of state we pull out. ([\#15968](https://github.com/matrix-org/synapse/issues/15968)) -- Speed up updating state in large rooms. ([\#15971](https://github.com/matrix-org/synapse/issues/15971)) - -### Updates to locked dependencies - -* Bump anyhow from 1.0.71 to 1.0.72. ([\#15949](https://github.com/matrix-org/synapse/issues/15949)) -* Bump click from 8.1.3 to 8.1.6. ([\#15984](https://github.com/matrix-org/synapse/issues/15984)) -* Bump cryptography from 41.0.1 to 41.0.2. ([\#15943](https://github.com/matrix-org/synapse/issues/15943)) -* Bump jsonschema from 4.17.3 to 4.18.3. ([\#15948](https://github.com/matrix-org/synapse/issues/15948)) -* Bump pillow from 9.4.0 to 10.0.0. ([\#15986](https://github.com/matrix-org/synapse/issues/15986)) -* Bump prometheus-client from 0.17.0 to 0.17.1. ([\#15945](https://github.com/matrix-org/synapse/issues/15945)) -* Bump pydantic from 1.10.10 to 1.10.11. ([\#15946](https://github.com/matrix-org/synapse/issues/15946)) -* Bump pygithub from 1.58.2 to 1.59.0. ([\#15834](https://github.com/matrix-org/synapse/issues/15834)) -* Bump pyo3-log from 0.8.2 to 0.8.3. ([\#15951](https://github.com/matrix-org/synapse/issues/15951)) -* Bump sentry-sdk from 1.26.0 to 1.28.1. ([\#15985](https://github.com/matrix-org/synapse/issues/15985)) -* Bump serde_json from 1.0.100 to 1.0.103. ([\#15950](https://github.com/matrix-org/synapse/issues/15950)) -* Bump types-pillow from 9.5.0.4 to 10.0.0.1. ([\#15932](https://github.com/matrix-org/synapse/issues/15932)) -* Bump types-requests from 2.31.0.1 to 2.31.0.2. ([\#15983](https://github.com/matrix-org/synapse/issues/15983)) -* Bump typing-extensions from 4.5.0 to 4.7.1. ([\#15947](https://github.com/matrix-org/synapse/issues/15947)) - -# Synapse 1.88.0 (2023-07-18) - -This release - - raises the minimum supported version of Python to 3.8, as Python 3.7 is now [end-of-life](https://devguide.python.org/versions/), and - - removes deprecated config options related to worker deployment. - -See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#upgrading-to-v1880) for more information. - - -### Bugfixes - -- Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters`", which was introduced in Synapse 1.88.0rc1. ([\#15953](https://github.com/matrix-org/synapse/issues/15953)) - - -# Synapse 1.88.0rc1 (2023-07-11) - -### Features - -- Add `not_user_type` param to the [list accounts admin API](https://matrix-org.github.io/synapse/v1.88/admin_api/user_admin_api.html#list-accounts). ([\#15844](https://github.com/matrix-org/synapse/issues/15844)) - -### Bugfixes - -- Pin `pydantic` to `^=1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. - Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) -- Correctly resize thumbnails with pillow version >=10. ([\#15876](https://github.com/matrix-org/synapse/issues/15876)) - -### Improved Documentation - -- Fixed header levels on the [Admin API "Users"](https://matrix-org.github.io/synapse/v1.87/admin_api/user_admin_api.html) documentation page. Contributed by @sumnerevans at @beeper. ([\#15852](https://github.com/matrix-org/synapse/issues/15852)) -- Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options. ([\#15872](https://github.com/matrix-org/synapse/issues/15872)) - -### Deprecations and Removals - -- **Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options.** See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#removal-of-worker_replication_-settings) for more details. ([\#15860](https://github.com/matrix-org/synapse/issues/15860)) -- Remove support for Python 3.7 and hence for Debian Buster. ([\#15851](https://github.com/matrix-org/synapse/issues/15851), [\#15892](https://github.com/matrix-org/synapse/issues/15892), [\#15893](https://github.com/matrix-org/synapse/issues/15893), [\#15917](https://github.com/matrix-org/synapse/pull/15917)) - -### Internal Changes - -- Add foreign key constraint to `event_forward_extremities`. ([\#15751](https://github.com/matrix-org/synapse/issues/15751), [\#15907](https://github.com/matrix-org/synapse/issues/15907)) -- Add read/write style cross-worker locks. ([\#15782](https://github.com/matrix-org/synapse/issues/15782)) -- Stop writing to column `user_id` of tables `profiles` and `user_filters`. ([\#15787](https://github.com/matrix-org/synapse/issues/15787)) -- Use lower isolation level when cleaning old presence stream data to avoid serialization errors. ([\#15826](https://github.com/matrix-org/synapse/issues/15826)) -- Add tracing to media `/upload` code paths. ([\#15850](https://github.com/matrix-org/synapse/issues/15850), [\#15888](https://github.com/matrix-org/synapse/issues/15888)) -- Add a timeout that aborts any Postgres statement taking more than 1 hour. ([\#15853](https://github.com/matrix-org/synapse/issues/15853)) -- Fix the `devenv up` configuration which was ignoring the config overrides. ([\#15854](https://github.com/matrix-org/synapse/issues/15854)) -- Optimised cleanup of old entries in `device_lists_stream`. ([\#15861](https://github.com/matrix-org/synapse/issues/15861)) -- Update the Matrix clients link in the _It works! Synapse is running_ landing page. ([\#15874](https://github.com/matrix-org/synapse/issues/15874)) -- Fix building Synapse with the nightly Rust compiler. ([\#15906](https://github.com/matrix-org/synapse/issues/15906)) -- Add `Server` to Access-Control-Expose-Headers header. ([\#15908](https://github.com/matrix-org/synapse/issues/15908)) - -### Updates to locked dependencies - -* Bump authlib from 1.2.0 to 1.2.1. ([\#15864](https://github.com/matrix-org/synapse/issues/15864)) -* Bump importlib-metadata from 6.6.0 to 6.7.0. ([\#15865](https://github.com/matrix-org/synapse/issues/15865)) -* Bump lxml from 4.9.2 to 4.9.3. ([\#15897](https://github.com/matrix-org/synapse/issues/15897)) -* Bump regex from 1.8.4 to 1.9.1. ([\#15902](https://github.com/matrix-org/synapse/issues/15902)) -* Bump ruff from 0.0.275 to 0.0.277. ([\#15900](https://github.com/matrix-org/synapse/issues/15900)) -* Bump sentry-sdk from 1.25.1 to 1.26.0. ([\#15867](https://github.com/matrix-org/synapse/issues/15867)) -* Bump serde_json from 1.0.99 to 1.0.100. ([\#15901](https://github.com/matrix-org/synapse/issues/15901)) -* Bump types-pyopenssl from 23.2.0.0 to 23.2.0.1. ([\#15866](https://github.com/matrix-org/synapse/issues/15866)) - -# Synapse 1.87.0 (2023-07-04) - -Please note that this will be the last release of Synapse that is compatible with -Python 3.7 and earlier. -This is due to Python 3.7 now having reached End of Life; see our [deprecation policy](https://matrix-org.github.io/synapse/v1.87/deprecation_policy.html) -for more details. - -### Bugfixes - -- Pin `pydantic` to `^1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. - Resolves https://github.com/matrix-org/synapse/issues/15858. - Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) - -### Internal Changes - -- Split out 2022 changes from the changelog so the rendered version in GitHub doesn't timeout as much. ([\#15846](https://github.com/matrix-org/synapse/issues/15846)) - - -# Synapse 1.87.0rc1 (2023-06-27) - -### Features - -- Improve `/messages` response time by avoiding backfill when we already have messages to return. ([\#15737](https://github.com/matrix-org/synapse/issues/15737)) -- Add spam checker module API for logins. ([\#15838](https://github.com/matrix-org/synapse/issues/15838)) - -### Bugfixes - -- Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou. ([\#15680](https://github.com/matrix-org/synapse/issues/15680)) -- Avoid invalidating a cache that was just prefilled. ([\#15758](https://github.com/matrix-org/synapse/issues/15758)) -- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15770](https://github.com/matrix-org/synapse/issues/15770)) -- Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper. ([\#15776](https://github.com/matrix-org/synapse/issues/15776)) -- Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`. ([\#15781](https://github.com/matrix-org/synapse/issues/15781)) -- Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend. ([\#15788](https://github.com/matrix-org/synapse/issues/15788)) -- Fix Sytest environmental variable evaluation in CI. ([\#15804](https://github.com/matrix-org/synapse/issues/15804)) -- Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly. ([\#15815](https://github.com/matrix-org/synapse/issues/15815)) -- Fix sqlite `user_filters` upgrade introduced in v1.86.0. ([\#15817](https://github.com/matrix-org/synapse/issues/15817)) - -### Improved Documentation - -- Document `looping_call()` functionality that will wait for the given function to finish before scheduling another. ([\#15772](https://github.com/matrix-org/synapse/issues/15772)) -- Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html). ([\#15805](https://github.com/matrix-org/synapse/issues/15805)) -- Fix typo in MSC number in faster remote room join architecture doc. ([\#15812](https://github.com/matrix-org/synapse/issues/15812)) - -### Deprecations and Removals - -- Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms. ([\#15748](https://github.com/matrix-org/synapse/issues/15748)) - -### Internal Changes - -- Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`. ([\#15233](https://github.com/matrix-org/synapse/issues/15233)) -- Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying. ([\#15743](https://github.com/matrix-org/synapse/issues/15743)) -- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15755](https://github.com/matrix-org/synapse/issues/15755)) -- Allow for the configuration of max request retries and min/max retry delays in the matrix federation client. ([\#15783](https://github.com/matrix-org/synapse/issues/15783)) -- Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic. ([\#15806](https://github.com/matrix-org/synapse/issues/15806)) -- Fix harmless exceptions being printed when running the port DB script. ([\#15814](https://github.com/matrix-org/synapse/issues/15814)) - -### Updates to locked dependencies - -* Bump attrs from 22.2.0 to 23.1.0. ([\#15801](https://github.com/matrix-org/synapse/issues/15801)) -* Bump cryptography from 40.0.2 to 41.0.1. ([\#15800](https://github.com/matrix-org/synapse/issues/15800)) -* Bump ijson from 3.2.0.post0 to 3.2.1. ([\#15802](https://github.com/matrix-org/synapse/issues/15802)) -* Bump phonenumbers from 8.13.13 to 8.13.14. ([\#15798](https://github.com/matrix-org/synapse/issues/15798)) -* Bump ruff from 0.0.265 to 0.0.272. ([\#15799](https://github.com/matrix-org/synapse/issues/15799)) -* Bump ruff from 0.0.272 to 0.0.275. ([\#15833](https://github.com/matrix-org/synapse/issues/15833)) -* Bump serde_json from 1.0.96 to 1.0.97. ([\#15797](https://github.com/matrix-org/synapse/issues/15797)) -* Bump serde_json from 1.0.97 to 1.0.99. ([\#15832](https://github.com/matrix-org/synapse/issues/15832)) -* Bump towncrier from 22.12.0 to 23.6.0. ([\#15831](https://github.com/matrix-org/synapse/issues/15831)) -* Bump types-opentracing from 2.4.10.4 to 2.4.10.5. ([\#15830](https://github.com/matrix-org/synapse/issues/15830)) -* Bump types-setuptools from 67.8.0.0 to 68.0.0.0. ([\#15835](https://github.com/matrix-org/synapse/issues/15835)) - -Synapse 1.86.0 (2023-06-20) -=========================== - -No significant changes since 1.86.0rc2. - - -Synapse 1.86.0rc2 (2023-06-14) -============================== - -Bugfixes --------- - -- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774)) - - -Synapse 1.86.0rc1 (2023-06-13) -============================== - -This version was tagged but never released. - -Features --------- - -- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388)) -- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450)) -- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520)) -- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582)) -- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674)) -- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705)) -- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710)) -- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740)) - - -Bugfixes --------- - -- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609)) -- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695)) - - -Improved Documentation ----------------------- - -- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732)) - - -Internal Changes ----------------- - -- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213)) -- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649)) -- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650)) -- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675)) -- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689)) -- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690)) -- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697)) -- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709)) -- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721)) -- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723)) -- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725)) -- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726)) -- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729)) -- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731)) -- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733)) -- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752)) -- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768)) - -### Updates to locked dependencies - -* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711)) -* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712)) -* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761)) -* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763)) -* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713)) -* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762)) -* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759)) -* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765)) -* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769)) -* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714)) -* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764)) -* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760)) -* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716)) -* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766)) -* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715)) - -Synapse 1.85.2 (2023-06-08) -=========================== - -Bugfixes --------- - -- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) - - -Synapse 1.85.1 (2023-06-07) -=========================== - -Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0. - -Bugfixes --------- - -- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739)) - - -Synapse 1.85.0 (2023-06-06) -=========================== - -No significant changes since 1.85.0rc2. - - -## Security advisory - -The following issues are fixed in 1.85.0 (and RCs). - -- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity - - It may be possible for a deactivated user to login when using uncommon configurations. - -- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity - - A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs). - -See the advisories for more details. If you have any questions, email security@matrix.org. - - -Synapse 1.85.0rc2 (2023-06-01) -============================== - -Bugfixes --------- - -- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693)) - - -Deprecations and Removals -------------------------- - -- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703)) - - -Internal Changes ----------------- - -- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700)) - - -Synapse 1.85.0rc1 (2023-05-30) -============================== - -Features --------- - -- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585)) -- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611)) -- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644)) - - -Bugfixes --------- - -- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464)) -- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601)) -- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607)) -- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614)) -- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624)) -- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634)) - - -Improved Documentation ----------------------- - -- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613)) -- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648)) -- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668)) - - -Deprecations and Removals -------------------------- - -- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428)) - - -Internal Changes ----------------- - -- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481)) -- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537)) -- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578)) -- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597)) -- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602)) -- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604)) -- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620)) -- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647)) -- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633)) -- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615)) -- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621)) -- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625)) -- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626)) -- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636)) -- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630)) -- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646)) -- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659)) -- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663)) -- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665)) -- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678)) - -### Updates to locked dependencies - -* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642)) -* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681)) -* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682)) -* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685)) -* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643)) -* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651)) -* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641)) -* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686)) -* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640)) -* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683)) -* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684)) -* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639)) - -Synapse 1.84.1 (2023-05-26) -=========================== - -This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers. -If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release. - - -Bugfixes --------- - -- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672)) - - -Internal Changes ----------------- - -- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673)) - - -Synapse 1.84.0 (2023-05-23) -=========================== - -The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information. - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599)) - - -Synapse 1.84.0rc1 (2023-05-16) -============================== - -Features --------- - -- Add an option to prevent media downloads from configured domains. ([\#15197](https://github.com/matrix-org/synapse/issues/15197)) -- Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them. ([\#15224](https://github.com/matrix-org/synapse/issues/15224)) -- Add redis TLS configuration options. ([\#15312](https://github.com/matrix-org/synapse/issues/15312)) -- Add a config option to delay push notifications by a random amount, to discourage time-based profiling. ([\#15516](https://github.com/matrix-org/synapse/issues/15516)) -- Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15528](https://github.com/matrix-org/synapse/issues/15528)) -- Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs. ([\#15536](https://github.com/matrix-org/synapse/issues/15536)) -- Advertise support for Matrix 1.6 on `/_matrix/client/versions`. ([\#15559](https://github.com/matrix-org/synapse/issues/15559)) -- Print full error and stack-trace of any exception that occurs during startup/initialization. ([\#15569](https://github.com/matrix-org/synapse/issues/15569)) - - -Bugfixes --------- - -- Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu. ([\#15523](https://github.com/matrix-org/synapse/issues/15523)) -- Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data. ([\#15554](https://github.com/matrix-org/synapse/issues/15554), [\#15555](https://github.com/matrix-org/synapse/issues/15555)) -- Fix a long-standing bug where an invalid membership event could cause an internal server error. ([\#15564](https://github.com/matrix-org/synapse/issues/15564)) -- Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571)) - - -Deprecations and Removals -------------------------- - -- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491)) - - -Updates to the Docker image ---------------------------- - -- Add pkg-config package to Stage 0 to be able to build Dockerfile on ppc64le architecture. ([\#15567](https://github.com/matrix-org/synapse/issues/15567)) - - -Improved Documentation ----------------------- - -- Clarify documentation of the "Create or modify account" Admin API. ([\#15544](https://github.com/matrix-org/synapse/issues/15544)) -- Fix path to the `statistics/database/rooms` admin API in documentation. ([\#15560](https://github.com/matrix-org/synapse/issues/15560)) -- Update and improve Mastodon Single Sign-On documentation. ([\#15587](https://github.com/matrix-org/synapse/issues/15587)) - - -Internal Changes ----------------- - -- Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025)) -- Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470)) -- Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509)) -- Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522)) -- Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527)) -- Speed up rebuilding of the user directory for local users. ([\#15529](https://github.com/matrix-org/synapse/issues/15529)) -- Speed up deleting of old rows in `event_push_actions`. ([\#15531](https://github.com/matrix-org/synapse/issues/15531)) -- Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment. ([\#15532](https://github.com/matrix-org/synapse/issues/15532), [\#15533](https://github.com/matrix-org/synapse/issues/15533), [\#15545](https://github.com/matrix-org/synapse/issues/15545)) -- Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules. ([\#15534](https://github.com/matrix-org/synapse/issues/15534)) -- Move various module API callback registration methods to a dedicated class. ([\#15535](https://github.com/matrix-org/synapse/issues/15535)) -- Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984). ([\#15539](https://github.com/matrix-org/synapse/issues/15539)) -- Factor out an `is_mine_server_name` method. ([\#15542](https://github.com/matrix-org/synapse/issues/15542)) -- Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`. ([\#15543](https://github.com/matrix-org/synapse/issues/15543)) -- Bump serde from 1.0.160 to 1.0.162. ([\#15548](https://github.com/matrix-org/synapse/issues/15548)) -- Bump types-setuptools from 67.6.0.5 to 67.7.0.1. ([\#15549](https://github.com/matrix-org/synapse/issues/15549)) -- Bump sentry-sdk from 1.19.1 to 1.22.1. ([\#15550](https://github.com/matrix-org/synapse/issues/15550)) -- Bump ruff from 0.0.259 to 0.0.265. ([\#15551](https://github.com/matrix-org/synapse/issues/15551)) -- Bump hiredis from 2.2.2 to 2.2.3. ([\#15552](https://github.com/matrix-org/synapse/issues/15552)) -- Bump types-requests from 2.29.0.0 to 2.30.0.0. ([\#15553](https://github.com/matrix-org/synapse/issues/15553)) -- Add `org.matrix.msc3981` info to `/_matrix/client/versions`. ([\#15558](https://github.com/matrix-org/synapse/issues/15558)) -- Declare unstable support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) under `/_matrix/client/versions` if the experimental implementation is enabled. ([\#15562](https://github.com/matrix-org/synapse/issues/15562)) -- Implement [MSC3821](https://github.com/matrix-org/matrix-spec-proposals/pull/3821) to update the redaction rules. ([\#15563](https://github.com/matrix-org/synapse/issues/15563)) -- Implement updated redaction rules from [MSC3389](https://github.com/matrix-org/matrix-spec-proposals/pull/3389). ([\#15565](https://github.com/matrix-org/synapse/issues/15565)) -- Allow `pip install` to use setuptools_rust 1.6.0 when building Synapse. ([\#15570](https://github.com/matrix-org/synapse/issues/15570)) -- Deal with upcoming Github Actions deprecations. ([\#15576](https://github.com/matrix-org/synapse/issues/15576)) -- Export `run_as_background_process` from the module API. ([\#15577](https://github.com/matrix-org/synapse/issues/15577)) -- Update build system requirements to allow building with poetry-core==1.6.0. ([\#15588](https://github.com/matrix-org/synapse/issues/15588)) -- Bump serde from 1.0.162 to 1.0.163. ([\#15589](https://github.com/matrix-org/synapse/issues/15589)) -- Bump phonenumbers from 8.13.7 to 8.13.11. ([\#15590](https://github.com/matrix-org/synapse/issues/15590)) -- Bump types-psycopg2 from 2.9.21.9 to 2.9.21.10. ([\#15591](https://github.com/matrix-org/synapse/issues/15591)) -- Bump types-commonmark from 0.9.2.2 to 0.9.2.3. ([\#15592](https://github.com/matrix-org/synapse/issues/15592)) -- Bump types-setuptools from 67.7.0.1 to 67.7.0.2. ([\#15594](https://github.com/matrix-org/synapse/issues/15594)) - - -Synapse 1.83.0 (2023-05-09) -=========================== - -No significant changes since 1.83.0rc1. - - -Synapse 1.83.0rc1 (2023-05-02) -============================== - -Features --------- - -- Experimental support to recursively provide relations per [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981). ([\#15315](https://github.com/matrix-org/synapse/issues/15315)) -- Experimental support for [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970): Scope transaction IDs to devices. ([\#15318](https://github.com/matrix-org/synapse/issues/15318)) -- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/experimental_features.html) to support per-user feature flags. ([\#15344](https://github.com/matrix-org/synapse/issues/15344)) -- Add a module API to send an HTTP push notification. ([\#15387](https://github.com/matrix-org/synapse/issues/15387)) -- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/statistics.html#get-largest-rooms-by-size-in-database) to query the largest rooms by disk space used in the database. ([\#15482](https://github.com/matrix-org/synapse/issues/15482)) - - -Bugfixes --------- - -- Disable push rule evaluation for rooms excluded from sync. ([\#15361](https://github.com/matrix-org/synapse/issues/15361)) -- Fix a long-standing bug where cached server key results which were directly fetched would not be properly re-used. ([\#15417](https://github.com/matrix-org/synapse/issues/15417)) -- Fix a bug introduced in Synapse 1.73.0 where some experimental push rules were returned by default. ([\#15494](https://github.com/matrix-org/synapse/issues/15494)) - - -Improved Documentation ----------------------- - -- Add Nginx loadbalancing example with sticky mxid for workers. ([\#15411](https://github.com/matrix-org/synapse/issues/15411)) -- Update outdated development docs that mention restrictions in versions of SQLite that we no longer support. ([\#15498](https://github.com/matrix-org/synapse/issues/15498)) - - -Internal Changes ----------------- - -- Speedup tests by caching HomeServerConfig instances. ([\#15284](https://github.com/matrix-org/synapse/issues/15284)) -- Add denormalised event stream ordering column to membership state tables for future use. Contributed by Nick @ Beeper (@fizzadar). ([\#15356](https://github.com/matrix-org/synapse/issues/15356)) -- Always use multi-user device resync replication endpoints. ([\#15418](https://github.com/matrix-org/synapse/issues/15418)) -- Add column `full_user_id` to tables `profiles` and `user_filters`. ([\#15458](https://github.com/matrix-org/synapse/issues/15458)) -- Update support for [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) to allow always returning fallback-keys in a `/keys/claim` request. ([\#15462](https://github.com/matrix-org/synapse/issues/15462)) -- Improve type hints. ([\#15465](https://github.com/matrix-org/synapse/issues/15465), [\#15496](https://github.com/matrix-org/synapse/issues/15496), [\#15497](https://github.com/matrix-org/synapse/issues/15497)) -- Support claiming more than one OTK at a time. ([\#15468](https://github.com/matrix-org/synapse/issues/15468)) -- Bump types-pyyaml from 6.0.12.8 to 6.0.12.9. ([\#15471](https://github.com/matrix-org/synapse/issues/15471)) -- Bump pyasn1-modules from 0.2.8 to 0.3.0. ([\#15473](https://github.com/matrix-org/synapse/issues/15473)) -- Bump cryptography from 40.0.1 to 40.0.2. ([\#15474](https://github.com/matrix-org/synapse/issues/15474)) -- Bump types-netaddr from 0.8.0.7 to 0.8.0.8. ([\#15475](https://github.com/matrix-org/synapse/issues/15475)) -- Bump types-jsonschema from 4.17.0.6 to 4.17.0.7. ([\#15476](https://github.com/matrix-org/synapse/issues/15476)) -- Ask bug reporters to provide logs as text. ([\#15479](https://github.com/matrix-org/synapse/issues/15479)) -- Add a Nix flake for use as a development environment. ([\#15495](https://github.com/matrix-org/synapse/issues/15495)) -- Bump anyhow from 1.0.70 to 1.0.71. ([\#15507](https://github.com/matrix-org/synapse/issues/15507)) -- Bump types-pillow from 9.4.0.19 to 9.5.0.2. ([\#15508](https://github.com/matrix-org/synapse/issues/15508)) -- Bump packaging from 23.0 to 23.1. ([\#15510](https://github.com/matrix-org/synapse/issues/15510)) -- Bump types-requests from 2.28.11.16 to 2.29.0.0. ([\#15511](https://github.com/matrix-org/synapse/issues/15511)) -- Bump setuptools-rust from 1.5.2 to 1.6.0. ([\#15512](https://github.com/matrix-org/synapse/issues/15512)) -- Update the check_schema_delta script to account for when the schema version has been bumped locally. ([\#15466](https://github.com/matrix-org/synapse/issues/15466)) - - -Synapse 1.82.0 (2023-04-25) -=========================== - -No significant changes since 1.82.0rc1. - - -Synapse 1.82.0rc1 (2023-04-18) -============================== - -Features --------- - -- Allow loading the `/directory/room/{roomAlias}` endpoint on workers. ([\#15333](https://github.com/matrix-org/synapse/issues/15333)) -- Add some validation to `instance_map` configuration loading. ([\#15431](https://github.com/matrix-org/synapse/issues/15431)) -- Allow loading the `/capabilities` endpoint on workers. ([\#15436](https://github.com/matrix-org/synapse/issues/15436)) - - -Bugfixes --------- - -- Delete server-side backup keys when deactivating an account. ([\#15181](https://github.com/matrix-org/synapse/issues/15181)) -- Fix and document untold assumption that `on_logged_out` module hooks will be called before the deletion of pushers. ([\#15410](https://github.com/matrix-org/synapse/issues/15410)) -- Improve robustness when handling a perspective key response by deduplicating received server keys. ([\#15423](https://github.com/matrix-org/synapse/issues/15423)) -- Synapse now correctly fails to start if the config option `app_service_config_files` is not a list. ([\#15425](https://github.com/matrix-org/synapse/issues/15425)) -- Disable loading `RefreshTokenServlet` (`/_matrix/client/(r0|v3|unstable)/refresh`) on workers. ([\#15428](https://github.com/matrix-org/synapse/issues/15428)) - - -Improved Documentation ----------------------- - -- Note that the `delete_stale_devices_after` background job always runs on the main process. ([\#15452](https://github.com/matrix-org/synapse/issues/15452)) - - -Deprecations and Removals -------------------------- - -- Remove the broken, unspecced registration fallback. Note that the *login* fallback is unaffected by this change. ([\#15405](https://github.com/matrix-org/synapse/issues/15405)) - - -Internal Changes ----------------- - -- Bump black from 23.1.0 to 23.3.0. ([\#15372](https://github.com/matrix-org/synapse/issues/15372)) -- Bump pyopenssl from 23.1.0 to 23.1.1. ([\#15373](https://github.com/matrix-org/synapse/issues/15373)) -- Bump types-psycopg2 from 2.9.21.8 to 2.9.21.9. ([\#15374](https://github.com/matrix-org/synapse/issues/15374)) -- Bump types-netaddr from 0.8.0.6 to 0.8.0.7. ([\#15375](https://github.com/matrix-org/synapse/issues/15375)) -- Bump types-opentracing from 2.4.10.3 to 2.4.10.4. ([\#15376](https://github.com/matrix-org/synapse/issues/15376)) -- Bump dawidd6/action-download-artifact from 2.26.0 to 2.26.1. ([\#15404](https://github.com/matrix-org/synapse/issues/15404)) -- Bump parameterized from 0.8.1 to 0.9.0. ([\#15412](https://github.com/matrix-org/synapse/issues/15412)) -- Bump types-pillow from 9.4.0.17 to 9.4.0.19. ([\#15413](https://github.com/matrix-org/synapse/issues/15413)) -- Bump sentry-sdk from 1.17.0 to 1.19.1. ([\#15414](https://github.com/matrix-org/synapse/issues/15414)) -- Bump immutabledict from 2.2.3 to 2.2.4. ([\#15415](https://github.com/matrix-org/synapse/issues/15415)) -- Bump dawidd6/action-download-artifact from 2.26.1 to 2.27.0. ([\#15441](https://github.com/matrix-org/synapse/issues/15441)) -- Bump serde_json from 1.0.95 to 1.0.96. ([\#15442](https://github.com/matrix-org/synapse/issues/15442)) -- Bump serde from 1.0.159 to 1.0.160. ([\#15443](https://github.com/matrix-org/synapse/issues/15443)) -- Bump pillow from 9.4.0 to 9.5.0. ([\#15444](https://github.com/matrix-org/synapse/issues/15444)) -- Bump furo from 2023.3.23 to 2023.3.27. ([\#15445](https://github.com/matrix-org/synapse/issues/15445)) -- Bump types-pyopenssl from 23.1.0.0 to 23.1.0.2. ([\#15446](https://github.com/matrix-org/synapse/issues/15446)) -- Bump mypy from 1.0.0 to 1.0.1. ([\#15447](https://github.com/matrix-org/synapse/issues/15447)) -- Bump psycopg2 from 2.9.5 to 2.9.6. ([\#15448](https://github.com/matrix-org/synapse/issues/15448)) -- Improve DB performance of clearing out old data from `stream_ordering_to_exterm`. ([\#15382](https://github.com/matrix-org/synapse/issues/15382), [\#15429](https://github.com/matrix-org/synapse/issues/15429)) -- Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. ([\#15393](https://github.com/matrix-org/synapse/issues/15393)) -- Implement [MSC2175](https://github.com/matrix-org/matrix-doc/pull/2175) to stop adding `creator` to create events. ([\#15394](https://github.com/matrix-org/synapse/issues/15394)) -- Implement [MSC2174](https://github.com/matrix-org/matrix-spec-proposals/pull/2174) to move the `redacts` key to a `content` property. ([\#15395](https://github.com/matrix-org/synapse/issues/15395)) -- Trust dtonlay/rust-toolchain in CI. ([\#15406](https://github.com/matrix-org/synapse/issues/15406)) -- Explicitly install Synapse during typechecking in CI. ([\#15409](https://github.com/matrix-org/synapse/issues/15409)) -- Only load the SSO redirect servlet if SSO is enabled. ([\#15421](https://github.com/matrix-org/synapse/issues/15421)) -- Refactor `SimpleHttpClient` to pull out a base class. ([\#15427](https://github.com/matrix-org/synapse/issues/15427)) -- Improve type hints. ([\#15432](https://github.com/matrix-org/synapse/issues/15432)) -- Convert async to normal tests in `TestSSOHandler`. ([\#15433](https://github.com/matrix-org/synapse/issues/15433)) -- Speed up the user directory background update. ([\#15435](https://github.com/matrix-org/synapse/issues/15435)) -- Disable directory listing for static resources in `/_matrix/static/`. ([\#15438](https://github.com/matrix-org/synapse/issues/15438)) -- Move various module API callback registration methods to a dedicated class. ([\#15453](https://github.com/matrix-org/synapse/issues/15453)) - - -Synapse 1.81.0 (2023-04-11) -=========================== - -Synapse now attempts the versioned appservice paths before falling back to the -[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). -Usage of the legacy routes should be considered deprecated. - -Additionally, Synapse has supported sending the application service access token -via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) -since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` -query parameter. This is insecure and should be considered deprecated. - -A future version of Synapse (v1.88.0 or later) will remove support for legacy -application service routes and query parameter authorization. - - -No significant changes since 1.81.0rc2. - - -Synapse 1.81.0rc2 (2023-04-06) -============================== - -Bugfixes --------- - -- Fix the `set_device_id_for_pushers_txn` background update crash. ([\#15391](https://github.com/matrix-org/synapse/issues/15391)) - - -Internal Changes ----------------- - -- Update CI to run complement under the latest stable go version. ([\#15403](https://github.com/matrix-org/synapse/issues/15403)) - - -Synapse 1.81.0rc1 (2023-04-04) -============================== - -Features --------- - -- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978)) -- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) -- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) -- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) -- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) -- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) - - -Bugfixes --------- - -- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295)) -- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297)) -- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) - to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) -- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) -- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) -- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) -- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) -- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) -- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) -- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352)) -- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383)) - - -Improved Documentation ----------------------- - -- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) -- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) -- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) - - -Internal Changes ----------------- - -- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113)) -- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336)) -- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280)) -- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285)) -- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) -- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) -- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) -- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) -- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) -- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) -- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) -- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325)) -- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326)) -- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327)) -- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328)) -- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329)) -- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330)) -- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) -- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) -- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) -- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) -- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) -- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) -- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371)) -- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385)) - - -Synapse 1.80.0 (2023-03-28) -=========================== - -No significant changes since 1.80.0rc2. - - -Synapse 1.80.0rc2 (2023-03-22) -============================== - -Bugfixes --------- - -- Fix a bug in which the [`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid) endpoint would return the wrong error if the user did not have permission to view the event. This aligns Synapse's implementation with [MSC2249](https://github.com/matrix-org/matrix-spec-proposals/pull/2249). ([\#15298](https://github.com/matrix-org/synapse/issues/15298), [\#15300](https://github.com/matrix-org/synapse/issues/15300)) -- Fix a bug introduced in Synapse 1.75.0rc1 where the [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) - would fail to open the SQLite database. ([\#15301](https://github.com/matrix-org/synapse/issues/15301)) - - -Synapse 1.80.0rc1 (2023-03-21) -============================== - -Features --------- - -- Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition. ([\#15187](https://github.com/matrix-org/synapse/issues/15187)) -- Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15249](https://github.com/matrix-org/synapse/issues/15249)) -- Allow loading `/register/available` endpoint on workers. ([\#15268](https://github.com/matrix-org/synapse/issues/15268)) -- Improve performance of creating and authenticating events. ([\#15195](https://github.com/matrix-org/synapse/issues/15195)) -- Add topic and name events to group of events that are batch persisted when creating a room. ([\#15229](https://github.com/matrix-org/synapse/issues/15229)) - - -Bugfixes --------- - -- Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change. ([\#14755](https://github.com/matrix-org/synapse/issues/14755), [\#14756](https://github.com/matrix-org/synapse/issues/14756)) -- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules. ([\#15190](https://github.com/matrix-org/synapse/issues/15190)) -- Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged. ([\#15232](https://github.com/matrix-org/synapse/issues/15232)) -- Fix a long-standing error when sending message into deleted room. ([\#15235](https://github.com/matrix-org/synapse/issues/15235)) - - -Updates to the Docker image ---------------------------- - -- Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel. ([\#15239](https://github.com/matrix-org/synapse/issues/15239)) -- Mirror images to the GitHub Container Registry (`ghcr.io/matrix-org/synapse`). ([\#15281](https://github.com/matrix-org/synapse/issues/15281), [\#15282](https://github.com/matrix-org/synapse/issues/15282)) - - -Improved Documentation ----------------------- - -- Add a missing endpoint to the workers documentation. ([\#15223](https://github.com/matrix-org/synapse/issues/15223)) - - -Internal Changes ----------------- - -- Add additional functionality to declaring worker types when starting Complement in worker mode. ([\#14921](https://github.com/matrix-org/synapse/issues/14921)) -- Add `Synapse-Trace-Id` to `access-control-expose-headers` header. ([\#14974](https://github.com/matrix-org/synapse/issues/14974)) -- Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key. ([\#15200](https://github.com/matrix-org/synapse/issues/15200)) -- Improve log lines when purging rooms. ([\#15222](https://github.com/matrix-org/synapse/issues/15222)) -- Improve type hints. ([\#15230](https://github.com/matrix-org/synapse/issues/15230), [\#15231](https://github.com/matrix-org/synapse/issues/15231), [\#15238](https://github.com/matrix-org/synapse/issues/15238)) -- Move various module API callback registration methods to a dedicated class. ([\#15237](https://github.com/matrix-org/synapse/issues/15237)) -- Configure GitHub Actions for merge queues. ([\#15244](https://github.com/matrix-org/synapse/issues/15244)) -- Add schema comments about the `destinations` and `destination_rooms` tables. ([\#15247](https://github.com/matrix-org/synapse/issues/15247)) -- Skip processing of auto-join room behaviour if there are no auto-join rooms configured. ([\#15262](https://github.com/matrix-org/synapse/issues/15262)) -- Remove unused store method `_set_destination_retry_timings_emulated`. ([\#15266](https://github.com/matrix-org/synapse/issues/15266)) -- Reorganize URL preview code. ([\#15269](https://github.com/matrix-org/synapse/issues/15269)) -- Clean-up direct TCP replication code. ([\#15272](https://github.com/matrix-org/synapse/issues/15272), [\#15274](https://github.com/matrix-org/synapse/issues/15274)) -- Make `configure_workers_and_start` script used in Complement tests compatible with older versions of Python. ([\#15275](https://github.com/matrix-org/synapse/issues/15275)) -- Add a `/versions` flag for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15293](https://github.com/matrix-org/synapse/issues/15293)) -- Bump hiredis from 2.2.1 to 2.2.2. ([\#15252](https://github.com/matrix-org/synapse/issues/15252)) -- Bump serde from 1.0.152 to 1.0.155. ([\#15253](https://github.com/matrix-org/synapse/issues/15253)) -- Bump pysaml2 from 7.2.1 to 7.3.1. ([\#15254](https://github.com/matrix-org/synapse/issues/15254)) -- Bump msgpack from 1.0.4 to 1.0.5. ([\#15255](https://github.com/matrix-org/synapse/issues/15255)) -- Bump gitpython from 3.1.30 to 3.1.31. ([\#15256](https://github.com/matrix-org/synapse/issues/15256)) -- Bump cryptography from 39.0.1 to 39.0.2. ([\#15257](https://github.com/matrix-org/synapse/issues/15257)) -- Bump pydantic from 1.10.4 to 1.10.6. ([\#15286](https://github.com/matrix-org/synapse/issues/15286)) -- Bump serde from 1.0.155 to 1.0.157. ([\#15287](https://github.com/matrix-org/synapse/issues/15287)) -- Bump anyhow from 1.0.69 to 1.0.70. ([\#15288](https://github.com/matrix-org/synapse/issues/15288)) -- Bump txredisapi from 1.4.7 to 1.4.9. ([\#15289](https://github.com/matrix-org/synapse/issues/15289)) -- Bump pygithub from 1.57 to 1.58.1. ([\#15290](https://github.com/matrix-org/synapse/issues/15290)) -- Bump types-requests from 2.28.11.12 to 2.28.11.15. ([\#15291](https://github.com/matrix-org/synapse/issues/15291)) - - - -Synapse 1.79.0 (2023-03-14) -=========================== - -No significant changes since 1.79.0rc2. - - -Synapse 1.79.0rc2 (2023-03-13) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.79.0rc1 where attempting to register a `on_remove_user_third_party_identifier` module API callback would be a no-op. ([\#15227](https://github.com/matrix-org/synapse/issues/15227)) -- Fix a rare bug introduced in Synapse 1.73 where events could remain unsent to other homeservers after a faster-join to a room. ([\#15248](https://github.com/matrix-org/synapse/issues/15248)) - - -Internal Changes ----------------- - -- Refactor `filter_events_for_server`. ([\#15240](https://github.com/matrix-org/synapse/issues/15240)) - - -Synapse 1.79.0rc1 (2023-03-07) -============================== - -Features --------- - -- Add two new Third Party Rules module API callbacks: [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier) and [`on_remove_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_remove_user_third_party_identifier). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) -- Experimental support for [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967) to not require UIA for setting up cross-signing on first use. ([\#15077](https://github.com/matrix-org/synapse/issues/15077)) -- Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#15107](https://github.com/matrix-org/synapse/issues/15107)) -- Add an [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) to delete a [specific event report](https://spec.matrix.org/v1.6/client-server-api/#reporting-content). ([\#15116](https://github.com/matrix-org/synapse/issues/15116)) -- Add support for knocking to workers. ([\#15133](https://github.com/matrix-org/synapse/issues/15133)) -- Allow use of the `/filter` Client-Server APIs on workers. ([\#15134](https://github.com/matrix-org/synapse/issues/15134)) -- Update support for [MSC2677](https://github.com/matrix-org/matrix-spec-proposals/pull/2677): remove support for server-side aggregation of reactions. ([\#15172](https://github.com/matrix-org/synapse/issues/15172)) -- Stabilise support for [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758): `event_property_is` push condition. ([\#15185](https://github.com/matrix-org/synapse/issues/15185)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.75 that caused experimental support for deleting account data to raise an internal server error while using an account data writer worker. ([\#14869](https://github.com/matrix-org/synapse/issues/14869)) -- Fix a long-standing bug where Synapse handled an unspecced field on push rules. ([\#15088](https://github.com/matrix-org/synapse/issues/15088)) -- Fix a long-standing bug where a URL preview would break if the discovered oEmbed failed to download. ([\#15092](https://github.com/matrix-org/synapse/issues/15092)) -- Fix a long-standing bug where an initial sync would not respond to changes to the list of ignored users if there was an initial sync cached. ([\#15163](https://github.com/matrix-org/synapse/issues/15163)) -- Add the `transaction_id` in the events included in many endpoints' responses. ([\#15174](https://github.com/matrix-org/synapse/issues/15174)) -- Fix a bug introduced in Synapse 1.78.0 where requests to claim dehydrated devices would fail with a `405` error. ([\#15180](https://github.com/matrix-org/synapse/issues/15180)) -- Stop applying edits when bundling aggregations, per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925). ([\#15193](https://github.com/matrix-org/synapse/issues/15193)) -- Fix a long-standing bug where the user directory search was not case-insensitive for accented characters. ([\#15143](https://github.com/matrix-org/synapse/issues/15143)) - - -Updates to the Docker image ---------------------------- - -- Improve startup logging in the with-workers Docker image. ([\#15186](https://github.com/matrix-org/synapse/issues/15186)) - - -Improved Documentation ----------------------- - -- Document how to use caches in a module. ([\#14026](https://github.com/matrix-org/synapse/issues/14026)) -- Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on. ([\#15071](https://github.com/matrix-org/synapse/issues/15071)) -- Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider. ([\#15112](https://github.com/matrix-org/synapse/issues/15112)) -- Correct reference to `federation_verify_certificates` in configuration documentation. ([\#15139](https://github.com/matrix-org/synapse/issues/15139)) -- Correct small documentation errors in some `MatrixFederationHttpClient` methods. ([\#15148](https://github.com/matrix-org/synapse/issues/15148)) -- Correct the description of the behavior of `registration_shared_secret_path` on startup. ([\#15168](https://github.com/matrix-org/synapse/issues/15168)) - - -Deprecations and Removals -------------------------- - -- Deprecate the `on_threepid_bind` module callback, to be replaced by [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier). See [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.79/docs/upgrade.md#upgrading-to-v1790). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) -- Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response. ([\#15093](https://github.com/matrix-org/synapse/issues/15093)) -- Remove the unspecced `PUT` on the `/knock/{roomIdOrAlias}` endpoint. ([\#15189](https://github.com/matrix-org/synapse/issues/15189)) -- Remove the undocumented and unspecced `type` parameter to the `/thumbnail` endpoint. ([\#15137](https://github.com/matrix-org/synapse/issues/15137)) -- Remove unspecced and buggy `PUT` method on the unstable `/rooms//batch_send` endpoint. ([\#15199](https://github.com/matrix-org/synapse/issues/15199)) - - -Internal Changes ----------------- - -- Run the integration test suites with the asyncio reactor enabled in CI. ([\#14101](https://github.com/matrix-org/synapse/issues/14101)) -- Batch up storing state groups when creating a new room. ([\#14918](https://github.com/matrix-org/synapse/issues/14918)) -- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15051](https://github.com/matrix-org/synapse/issues/15051)) -- Refactor writing json data in `FileExfiltrationWriter`. ([\#15095](https://github.com/matrix-org/synapse/issues/15095)) -- Tighten the login ratelimit defaults. ([\#15135](https://github.com/matrix-org/synapse/issues/15135)) -- Fix a typo in an experimental config setting. ([\#15138](https://github.com/matrix-org/synapse/issues/15138)) -- Refactor the media modules. ([\#15146](https://github.com/matrix-org/synapse/issues/15146), [\#15175](https://github.com/matrix-org/synapse/issues/15175)) -- Improve type hints. ([\#15164](https://github.com/matrix-org/synapse/issues/15164)) -- Move `get_event_report` and `get_event_reports_paginate` from `RoomStore` to `RoomWorkerStore`. ([\#15165](https://github.com/matrix-org/synapse/issues/15165)) -- Remove dangling reference to being a reference implementation in docstring. ([\#15167](https://github.com/matrix-org/synapse/issues/15167)) -- Add an option to force a rebuild of the "editable" complement image. ([\#15184](https://github.com/matrix-org/synapse/issues/15184)) -- Use nightly rustfmt in CI. ([\#15188](https://github.com/matrix-org/synapse/issues/15188)) -- Add a `get_next_txn` method to `StreamIdGenerator` to match `MultiWriterIdGenerator`. ([\#15191](https://github.com/matrix-org/synapse/issues/15191)) -- Combine `AbstractStreamIdTracker` and `AbstractStreamIdGenerator`. ([\#15192](https://github.com/matrix-org/synapse/issues/15192)) -- Automatically fix errors with `ruff`. ([\#15194](https://github.com/matrix-org/synapse/issues/15194)) -- Refactor database transaction for query users' devices to reduce database pool contention. ([\#15215](https://github.com/matrix-org/synapse/issues/15215)) -- Correct `test_icu_word_boundary_punctuation` so that it passes with the ICU versions available in Alpine and macOS. ([\#15177](https://github.com/matrix-org/synapse/issues/15177)) - -
Locked dependency updates - - - Bump actions/checkout from 2 to 3. ([\#15155](https://github.com/matrix-org/synapse/issues/15155)) - - Bump black from 22.12.0 to 23.1.0. ([\#15103](https://github.com/matrix-org/synapse/issues/15103)) - - Bump dawidd6/action-download-artifact from 2.25.0 to 2.26.0. ([\#15152](https://github.com/matrix-org/synapse/issues/15152)) - - Bump docker/login-action from 1 to 2. ([\#15154](https://github.com/matrix-org/synapse/issues/15154)) - - Bump matrix-org/backend-meta from 1 to 2. ([\#15156](https://github.com/matrix-org/synapse/issues/15156)) - - Bump ruff from 0.0.237 to 0.0.252. ([\#15159](https://github.com/matrix-org/synapse/issues/15159)) - - Bump serde_json from 1.0.93 to 1.0.94. ([\#15214](https://github.com/matrix-org/synapse/issues/15214)) - - Bump types-commonmark from 0.9.2.1 to 0.9.2.2. ([\#15209](https://github.com/matrix-org/synapse/issues/15209)) - - Bump types-opentracing from 2.4.10.1 to 2.4.10.3. ([\#15158](https://github.com/matrix-org/synapse/issues/15158)) - - Bump types-pillow from 9.4.0.13 to 9.4.0.17. ([\#15211](https://github.com/matrix-org/synapse/issues/15211)) - - Bump types-psycopg2 from 2.9.21.4 to 2.9.21.8. ([\#15210](https://github.com/matrix-org/synapse/issues/15210)) - - Bump types-pyopenssl from 22.1.0.2 to 23.0.0.4. ([\#15213](https://github.com/matrix-org/synapse/issues/15213)) - - Bump types-setuptools from 67.3.0.1 to 67.4.0.3. ([\#15160](https://github.com/matrix-org/synapse/issues/15160)) - - Bump types-setuptools from 67.4.0.3 to 67.5.0.0. ([\#15212](https://github.com/matrix-org/synapse/issues/15212)) - - Bump typing-extensions from 4.4.0 to 4.5.0. ([\#15157](https://github.com/matrix-org/synapse/issues/15157)) -
- - -Synapse 1.78.0 (2023-02-28) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.76 where 5s delays would occasionally occur in deployments using workers. ([\#15150](https://github.com/matrix-org/synapse/issues/15150)) - - -Synapse 1.78.0rc1 (2023-02-21) -============================== - -Features --------- - -- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964)) -- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969)) -- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004)) -- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034)) -- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042)) -- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045)) -- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073)) -- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075)) - - -Bugfixes --------- - -- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779)) -- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605)) -- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038)) -- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068)) -- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074)) -- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079)) -- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080)) -- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096)) -- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108)) - - -Improved Documentation ----------------------- - -- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022)) -- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959)) -- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078)) -- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083)) - - -Internal Changes ----------------- - -- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755)) -- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606)) -- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675)) -- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742)) -- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834)) -- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840)) -- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929)) -- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973)) -- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977)) -- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980)) -- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982)) -- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084)) -- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037)) -- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040)) -- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041)) -- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043)) -- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047)) -- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053)) -- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054)) -- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069)) -- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070)) -- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114)) -- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120)) - -
Locked dependency updates - -- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059)) -- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020)) -- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033)) -- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060)) -- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061)) -- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062)) -- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063)) -- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064)) -- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065)) -- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099)) -- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100)) -- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101)) -- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102)) -- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104)) -- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105)) - - -
- - -Synapse 1.77.0 (2023-02-14) -=========================== - -No significant changes since 1.77.0rc2. - - -Synapse 1.77.0rc2 (2023-02-10) -============================== - -Bugfixes --------- - -- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024)) - - -Internal Changes ----------------- - -- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036)) - - -Synapse 1.77.0rc1 (2023-02-07) -============================== - -Features --------- - -- Experimental support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#14823](https://github.com/matrix-org/synapse/issues/14823), [\#14943](https://github.com/matrix-org/synapse/issues/14943), [\#14957](https://github.com/matrix-org/synapse/issues/14957), [\#14958](https://github.com/matrix-org/synapse/issues/14958)) -- Experimental support to suppress notifications from message edits ([MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958)). ([\#14960](https://github.com/matrix-org/synapse/issues/14960), [\#15016](https://github.com/matrix-org/synapse/issues/15016)) -- Add profile information, devices and connections to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.77/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14894](https://github.com/matrix-org/synapse/issues/14894)) -- Improve performance when joining or sending an event in large rooms. ([\#14962](https://github.com/matrix-org/synapse/issues/14962)) -- Improve performance of joining and leaving large rooms with many local users. ([\#14971](https://github.com/matrix-org/synapse/issues/14971)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.53.0 where `next_batch` tokens from `/sync` could not be used with the `/relations` endpoint. ([\#14866](https://github.com/matrix-org/synapse/issues/14866)) -- Fix a bug introduced in Synapse 1.35.0 where the module API's `send_local_online_presence_to` would fail to send presence updates over federation. ([\#14880](https://github.com/matrix-org/synapse/issues/14880)) -- Fix a bug introduced in Synapse 1.70.0 where the background updates to add non-thread unique indexes on receipts could fail when upgrading from 1.67.0 or earlier. ([\#14915](https://github.com/matrix-org/synapse/issues/14915)) -- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14926](https://github.com/matrix-org/synapse/issues/14926)) -- Fix a bug introduced in Synapse 1.68.0 where we were unable to service remote joins in rooms with `@room` notification levels set to `null` in their (malformed) power levels. ([\#14942](https://github.com/matrix-org/synapse/issues/14942)) -- Fix a bug introduced in Synapse 1.64.0 where boolean power levels were erroneously permitted in [v10 rooms](https://spec.matrix.org/v1.5/rooms/v10/). ([\#14944](https://github.com/matrix-org/synapse/issues/14944)) -- Fix a long-standing bug where sending messages on servers with presence enabled would spam "Re-starting finished log context" log lines. ([\#14947](https://github.com/matrix-org/synapse/issues/14947)) -- Fix a bug introduced in Synapse 1.68.0 where logging from the Rust module was not properly logged. ([\#14976](https://github.com/matrix-org/synapse/issues/14976)) -- Fix various long-standing bugs in Synapse's config, event and request handling where booleans were unintentionally accepted where an integer was expected. ([\#14945](https://github.com/matrix-org/synapse/issues/14945)) - - -Internal Changes ----------------- - -- Add missing type hints. ([\#14879](https://github.com/matrix-org/synapse/issues/14879), [\#14886](https://github.com/matrix-org/synapse/issues/14886), [\#14887](https://github.com/matrix-org/synapse/issues/14887), [\#14904](https://github.com/matrix-org/synapse/issues/14904), [\#14927](https://github.com/matrix-org/synapse/issues/14927), [\#14956](https://github.com/matrix-org/synapse/issues/14956), [\#14983](https://github.com/matrix-org/synapse/issues/14983), [\#14984](https://github.com/matrix-org/synapse/issues/14984), [\#14985](https://github.com/matrix-org/synapse/issues/14985), [\#14987](https://github.com/matrix-org/synapse/issues/14987), [\#14988](https://github.com/matrix-org/synapse/issues/14988), [\#14990](https://github.com/matrix-org/synapse/issues/14990), [\#14991](https://github.com/matrix-org/synapse/issues/14991), [\#14992](https://github.com/matrix-org/synapse/issues/14992), [\#15007](https://github.com/matrix-org/synapse/issues/15007)) -- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14922](https://github.com/matrix-org/synapse/issues/14922)) -- Allow running the complement tests suites with the asyncio reactor enabled. ([\#14858](https://github.com/matrix-org/synapse/issues/14858)) -- Improve performance of `/sync` in a few situations. ([\#14908](https://github.com/matrix-org/synapse/issues/14908), [\#14970](https://github.com/matrix-org/synapse/issues/14970)) -- Document how to handle Dependabot pull requests. ([\#14916](https://github.com/matrix-org/synapse/issues/14916)) -- Fix typo in release script. ([\#14920](https://github.com/matrix-org/synapse/issues/14920)) -- Update build system requirements to allow building with poetry-core 1.5.0. ([\#14949](https://github.com/matrix-org/synapse/issues/14949), [\#15019](https://github.com/matrix-org/synapse/issues/15019)) -- Add an [lnav](https://lnav.org) config file for Synapse logs to `/contrib/lnav`. ([\#14953](https://github.com/matrix-org/synapse/issues/14953)) -- Faster joins: Refactor internal handling of servers in room to never store an empty list. ([\#14954](https://github.com/matrix-org/synapse/issues/14954)) -- Faster joins: tag `v2/send_join/` requests to indicate if they served a partial join response. ([\#14950](https://github.com/matrix-org/synapse/issues/14950)) -- Allow running `cargo` without the `extension-module` option. ([\#14965](https://github.com/matrix-org/synapse/issues/14965)) -- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014)) -- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002)) - -
Locked dependency updates - -- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968)) -- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952)) -- Bump ijson from 3.1.4 to 3.2.0.post0. ([\#14935](https://github.com/matrix-org/synapse/issues/14935)) -- Bump types-pyyaml from 6.0.12.2 to 6.0.12.3. ([\#14936](https://github.com/matrix-org/synapse/issues/14936)) -- Bump types-jsonschema from 4.17.0.2 to 4.17.0.3. ([\#14937](https://github.com/matrix-org/synapse/issues/14937)) -- Bump types-pillow from 9.4.0.3 to 9.4.0.5. ([\#14938](https://github.com/matrix-org/synapse/issues/14938)) -- Bump hiredis from 2.0.0 to 2.1.1. ([\#14939](https://github.com/matrix-org/synapse/issues/14939)) -- Bump hiredis from 2.1.1 to 2.2.1. ([\#14993](https://github.com/matrix-org/synapse/issues/14993)) -- Bump types-setuptools from 65.6.0.3 to 67.1.0.0. ([\#14994](https://github.com/matrix-org/synapse/issues/14994)) -- Bump prometheus-client from 0.15.0 to 0.16.0. ([\#14995](https://github.com/matrix-org/synapse/issues/14995)) -- Bump anyhow from 1.0.68 to 1.0.69. ([\#14996](https://github.com/matrix-org/synapse/issues/14996)) -- Bump serde_json from 1.0.91 to 1.0.92. ([\#14997](https://github.com/matrix-org/synapse/issues/14997)) -- Bump isort from 5.11.4 to 5.11.5. ([\#14998](https://github.com/matrix-org/synapse/issues/14998)) -- Bump phonenumbers from 8.13.4 to 8.13.5. ([\#14999](https://github.com/matrix-org/synapse/issues/14999)) -
- -Synapse 1.76.0 (2023-01-31) -=========================== - -The 1.76 release is the first to enable faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706) and [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902)) by default. Admins can opt-out: see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#faster-joins-are-enabled-by-default) for more details. - -The upgrade from 1.75 to 1.76 changes the account data replication streams in a backwards-incompatible manner. Server operators running a multi-worker deployment should consult [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#changes-to-the-account-data-replication-streams). - -Those who are `poetry install`ing from source using our lockfile should ensure their poetry version is 1.3.2 or higher; [see upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#minimum-version-of-poetry-is-now-132). - - -Notes on faster joins ---------------------- - -The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. - -After a faster join, Synapse considers that room "partially joined". In this state, you should be able to - -- read incoming messages; -- see incoming state changes, e.g. room topic changes; and -- send messages, if the room is unencrypted. - -Synapse has to spend more effort to complete the join in the background. Once this finishes, you will be able to - -- send messages, if the room is in encrypted; -- retrieve room history from before your join, if permitted by the room settings; and -- access the full list of room members. - - -Improved Documentation ----------------------- - -- Describe the ideas and the internal machinery behind faster joins. ([\#14677](https://github.com/matrix-org/synapse/issues/14677)) - - -Synapse 1.76.0rc2 (2023-01-27) -============================== - -Bugfixes --------- - -- Faster joins: Fix a bug introduced in Synapse 1.69 where device list EDUs could fail to be handled after a restart when a faster join sync is in progress. ([\#14914](https://github.com/matrix-org/synapse/issues/14914)) - - -Internal Changes ----------------- - -- Faster joins: Improve performance of looking up partial-state status of rooms. ([\#14917](https://github.com/matrix-org/synapse/issues/14917)) - - -Synapse 1.76.0rc1 (2023-01-25) -============================== - -Features --------- - -- Update the default room version to [v10](https://spec.matrix.org/v1.5/rooms/v10/) ([MSC 3904](https://github.com/matrix-org/matrix-spec-proposals/pull/3904)). Contributed by @FSG-Cat. ([\#14111](https://github.com/matrix-org/synapse/issues/14111)) -- Add a `set_displayname()` method to the module API for setting a user's display name. ([\#14629](https://github.com/matrix-org/synapse/issues/14629)) -- Add a dedicated listener configuration for `health` endpoint. ([\#14747](https://github.com/matrix-org/synapse/issues/14747)) -- Implement support for [MSC3890](https://github.com/matrix-org/matrix-spec-proposals/pull/3890): Remotely silence local notifications. ([\#14775](https://github.com/matrix-org/synapse/issues/14775)) -- Implement experimental support for [MSC3930](https://github.com/matrix-org/matrix-spec-proposals/pull/3930): Push rules for ([MSC3381](https://github.com/matrix-org/matrix-spec-proposals/pull/3381)) Polls. ([\#14787](https://github.com/matrix-org/synapse/issues/14787)) -- Per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925), bundle the whole of the replacement with any edited events, and optionally inhibit server-side replacement. ([\#14811](https://github.com/matrix-org/synapse/issues/14811)) -- Faster joins: always serve a partial join response to servers that request it with the stable query param. ([\#14839](https://github.com/matrix-org/synapse/issues/14839)) -- Faster joins: allow non-lazy-loading ("eager") syncs to complete after a partial join by omitting partial state rooms until they become fully stated. ([\#14870](https://github.com/matrix-org/synapse/issues/14870)) -- Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. ([\#14905](https://github.com/matrix-org/synapse/issues/14905)) - - -Bugfixes --------- - -- Add index to improve performance of the `/timestamp_to_event` endpoint used for jumping to a specific date in the timeline of a room. ([\#14799](https://github.com/matrix-org/synapse/issues/14799)) -- Fix a long-standing bug where Synapse would exhaust the stack when processing many federation requests where the remote homeserver has disconencted early. ([\#14812](https://github.com/matrix-org/synapse/issues/14812), [\#14842](https://github.com/matrix-org/synapse/issues/14842)) -- Fix rare races when using workers. ([\#14820](https://github.com/matrix-org/synapse/issues/14820)) -- Fix a bug introduced in Synapse 1.64.0 when using room version 10 with frozen events enabled. ([\#14864](https://github.com/matrix-org/synapse/issues/14864)) -- Fix a long-standing bug where the `populate_room_stats` background job could fail on broken rooms. ([\#14873](https://github.com/matrix-org/synapse/issues/14873)) -- Faster joins: Fix a bug in worker deployments where the room stats and user directory would not get updated when finishing a fast join until another event is sent or received. ([\#14874](https://github.com/matrix-org/synapse/issues/14874)) -- Faster joins: Fix incompatibility with joins into restricted rooms where no local users have the ability to invite. ([\#14882](https://github.com/matrix-org/synapse/issues/14882)) -- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14910](https://github.com/matrix-org/synapse/issues/14910)) - - -Updates to the Docker image ---------------------------- - -- Bump default Python version in the Dockerfile from 3.9 to 3.11. ([\#14875](https://github.com/matrix-org/synapse/issues/14875)) - - -Improved Documentation ----------------------- - -- Include `x_forwarded` entry in the HTTP listener example configs and remove the remaining `worker_main_http_uri` entries. ([\#14667](https://github.com/matrix-org/synapse/issues/14667)) -- Remove duplicate commands from the Code Style documentation page; point to the Contributing Guide instead. ([\#14773](https://github.com/matrix-org/synapse/issues/14773)) -- Add missing documentation for `tag` to `listeners` section. ([\#14803](https://github.com/matrix-org/synapse/issues/14803)) -- Updated documentation in configuration manual for `user_directory.search_all_users`. ([\#14818](https://github.com/matrix-org/synapse/issues/14818)) -- Add `worker_manhole` to configuration manual. ([\#14824](https://github.com/matrix-org/synapse/issues/14824)) -- Fix the example config missing the `id` field in [application service documentation](https://matrix-org.github.io/synapse/latest/application_services.html). ([\#14845](https://github.com/matrix-org/synapse/issues/14845)) -- Minor corrections to the logging configuration documentation. ([\#14868](https://github.com/matrix-org/synapse/issues/14868)) -- Document the export user data command. Contributed by @thezaidbintariq. ([\#14883](https://github.com/matrix-org/synapse/issues/14883)) - - -Deprecations and Removals -------------------------- - -- Poetry 1.3.2 or higher is now required when `poetry install`ing from source. ([\#14860](https://github.com/matrix-org/synapse/issues/14860)) - - -Internal Changes ----------------- - -- Faster remote room joins (worker mode): do not populate external hosts-in-room cache when sending events as this requires blocking for full state. ([\#14749](https://github.com/matrix-org/synapse/issues/14749)) -- Enable Complement tests for Faster Remote Room Joins against worker-mode Synapse. ([\#14752](https://github.com/matrix-org/synapse/issues/14752)) -- Add some clarifying comments and refactor a portion of the `Keyring` class for readability. ([\#14804](https://github.com/matrix-org/synapse/issues/14804)) -- Add local poetry config files (`poetry.toml`) to `.gitignore`. ([\#14807](https://github.com/matrix-org/synapse/issues/14807)) -- Add missing type hints. ([\#14816](https://github.com/matrix-org/synapse/issues/14816), [\#14885](https://github.com/matrix-org/synapse/issues/14885), [\#14889](https://github.com/matrix-org/synapse/issues/14889)) -- Refactor push tests. ([\#14819](https://github.com/matrix-org/synapse/issues/14819)) -- Re-enable some linting that was disabled when we switched to ruff. ([\#14821](https://github.com/matrix-org/synapse/issues/14821)) -- Add `cargo fmt` and `cargo clippy` to the lint script. ([\#14822](https://github.com/matrix-org/synapse/issues/14822)) -- Drop unused table `presence`. ([\#14825](https://github.com/matrix-org/synapse/issues/14825)) -- Merge the two account data and the two device list replication streams. ([\#14826](https://github.com/matrix-org/synapse/issues/14826), [\#14833](https://github.com/matrix-org/synapse/issues/14833)) -- Faster joins: use stable identifiers from [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). ([\#14832](https://github.com/matrix-org/synapse/issues/14832), [\#14841](https://github.com/matrix-org/synapse/issues/14841)) -- Add a parameter to control whether the federation client performs a partial state join. ([\#14843](https://github.com/matrix-org/synapse/issues/14843)) -- Add check to avoid starting duplicate partial state syncs. ([\#14844](https://github.com/matrix-org/synapse/issues/14844)) -- Add an early return when handling no-op presence updates. ([\#14855](https://github.com/matrix-org/synapse/issues/14855)) -- Fix `wait_for_stream_position` to correctly wait for the right instance to advance its token. ([\#14856](https://github.com/matrix-org/synapse/issues/14856), [\#14872](https://github.com/matrix-org/synapse/issues/14872)) -- Always notify replication when a stream advances automatically. ([\#14877](https://github.com/matrix-org/synapse/issues/14877)) -- Reduce max time we wait for stream positions. ([\#14881](https://github.com/matrix-org/synapse/issues/14881)) -- Faster joins: allow the resync process more time to fetch `/state` ids. ([\#14912](https://github.com/matrix-org/synapse/issues/14912)) -- Bump regex from 1.7.0 to 1.7.1. ([\#14848](https://github.com/matrix-org/synapse/issues/14848)) -- Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2. ([\#14861](https://github.com/matrix-org/synapse/issues/14861)) -- Bump ruff from 0.0.215 to 0.0.224. ([\#14862](https://github.com/matrix-org/synapse/issues/14862)) -- Bump types-pillow from 9.4.0.0 to 9.4.0.3. ([\#14863](https://github.com/matrix-org/synapse/issues/14863)) -- Bump types-opentracing from 2.4.10 to 2.4.10.1. ([\#14896](https://github.com/matrix-org/synapse/issues/14896)) -- Bump ruff from 0.0.224 to 0.0.230. ([\#14897](https://github.com/matrix-org/synapse/issues/14897)) -- Bump types-requests from 2.28.11.7 to 2.28.11.8. ([\#14899](https://github.com/matrix-org/synapse/issues/14899)) -- Bump types-psycopg2 from 2.9.21.2 to 2.9.21.4. ([\#14900](https://github.com/matrix-org/synapse/issues/14900)) -- Bump types-commonmark from 0.9.2 to 0.9.2.1. ([\#14901](https://github.com/matrix-org/synapse/issues/14901)) - - -Synapse 1.75.0 (2023-01-17) -=========================== - -No significant changes since 1.75.0rc2. - - -Synapse 1.75.0rc2 (2023-01-12) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810)) -- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817)) - - -Synapse 1.75.0rc1 (2023-01-10) -============================== - -Features --------- - -- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663)) -- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714)) -- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750)) -- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753)) -- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786)) -- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263)) - - -Bugfixes --------- - -- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644)) -- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669)) -- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723)) -- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727)) -- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751)) -- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781)) -- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725)) - - -Improved Documentation ----------------------- - -- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673)) -- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674)) -- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730)) -- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744)) -- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748)) -- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778)) -- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797)) -- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801)) -- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743)) - - -Internal Changes ----------------- - -- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546)) -- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741)) -- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665)) -- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672)) -- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676)) -- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687)) -- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685)) -- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692)) -- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707)) -- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716)) -- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724)) -- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772)) -- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774)) -- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802)) - -### Dependabot updates - -
- -- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693)) -- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694)) -- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695)) -- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696)) -- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697)) -- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698)) -- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700)) -- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701)) -- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702)) -- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720)) -- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731)) -- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732)) -- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733)) -- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734)) -- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735)) -- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736)) -- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738)) -- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758)) -- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759)) -- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760)) -- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761)) -- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762)) -- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763)) -- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779)) -- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791)) -- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792)) -- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793)) -- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794)) -- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795)) -- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796)) -
- **Changelogs for older versions can be found [here](docs/changelogs/).** diff --git a/docs/changelogs/CHANGES-2023.md b/docs/changelogs/CHANGES-2023.md new file mode 100644 index 0000000000..9b6ad3de1b --- /dev/null +++ b/docs/changelogs/CHANGES-2023.md @@ -0,0 +1,2202 @@ +# Synapse 1.98.0 (2023-12-12) + +Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + + +No significant changes since 1.98.0rc1. + + + +# Synapse 1.98.0rc1 (2023-12-05) + +### Features + +- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) +- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback for when a user logs in. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) +- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) +- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) +- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) +- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) + +### Bugfixes + +- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) +- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) +- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) +- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) + +### Improved Documentation + +- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) +- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) +- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) +- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) + +### Internal Changes + +- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) +- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) +- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) +- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) +- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) +- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) +- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) +- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) +- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) +- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) + + +### Updates to locked dependencies + +* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) +* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) +* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) +* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) +* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) +* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) +* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) +* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) +* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) +* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) +* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) +* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) +* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) +* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) +* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) +* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) +* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) +* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) +* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) + +# Synapse 1.97.0 (2023-11-28) + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + + +No significant changes since 1.97.0rc1. + + +# Synapse 1.97.0rc1 (2023-11-21) + +### Features + +- Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) +- Improve the performance of some operations in multi-worker deployments. ([\#16613](https://github.com/matrix-org/synapse/issues/16613), [\#16616](https://github.com/matrix-org/synapse/issues/16616)) + +### Bugfixes + +- Fix a long-standing bug where some queries updated the same row twice. Introduced in Synapse 1.57.0. ([\#16609](https://github.com/matrix-org/synapse/issues/16609)) +- Fix a long-standing bug where Synapse would not unbind third-party identifiers for Application Service users when deactivated and would not emit a compliant response. ([\#16617](https://github.com/matrix-org/synapse/issues/16617)) +- Fix sending out of order `POSITION` over replication, causing additional database load. ([\#16639](https://github.com/matrix-org/synapse/issues/16639)) + +### Improved Documentation + +- Note that the option [`outbound_federation_restricted_to`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#outbound_federation_restricted_to) was added in Synapse 1.89.0, and fix a nearby formatting error. ([\#16628](https://github.com/matrix-org/synapse/issues/16628)) +- Update parameter information for the `/timestamp_to_event` admin API. ([\#16631](https://github.com/matrix-org/synapse/issues/16631)) +- Provide an example for a common encrypted media response from the admin user media API and mention possible null values. ([\#16654](https://github.com/matrix-org/synapse/issues/16654)) + +### Internal Changes + +- Remove whole table locks on push rule modifications. Contributed by Nick @ Beeper (@fizzadar). ([\#16051](https://github.com/matrix-org/synapse/issues/16051)) +- Support reactor tick timings on more types of event loops. ([\#16532](https://github.com/matrix-org/synapse/issues/16532)) +- Improve type hints. ([\#16564](https://github.com/matrix-org/synapse/issues/16564), [\#16611](https://github.com/matrix-org/synapse/issues/16611), [\#16612](https://github.com/matrix-org/synapse/issues/16612)) +- Avoid executing no-op queries. ([\#16583](https://github.com/matrix-org/synapse/issues/16583)) +- Simplify persistence code to be per-room. ([\#16584](https://github.com/matrix-org/synapse/issues/16584)) +- Use standard SQL helpers in persistence code. ([\#16585](https://github.com/matrix-org/synapse/issues/16585)) +- Avoid updating the stream cache unnecessarily. ([\#16586](https://github.com/matrix-org/synapse/issues/16586)) +- Improve performance when using opentracing. ([\#16589](https://github.com/matrix-org/synapse/issues/16589)) +- Run push rule evaluator setup in parallel. ([\#16590](https://github.com/matrix-org/synapse/issues/16590)) +- Improve tests of the SQL generator. ([\#16596](https://github.com/matrix-org/synapse/issues/16596)) +- Use more generic database methods. ([\#16615](https://github.com/matrix-org/synapse/issues/16615)) +- Use `dbname` instead of the deprecated `database` connection parameter for psycopg2. ([\#16618](https://github.com/matrix-org/synapse/issues/16618)) +- Add an internal [Admin API endpoint](https://matrix-org.github.io/synapse/v1.97/usage/configuration/config_documentation.html#allow-replacing-master-cross-signing-key-without-user-interactive-auth) to temporarily grant the ability to update an existing cross-signing key without UIA. ([\#16634](https://github.com/matrix-org/synapse/issues/16634)) +- Improve references to GitHub issues. ([\#16637](https://github.com/matrix-org/synapse/issues/16637), [\#16638](https://github.com/matrix-org/synapse/issues/16638)) +- More efficiently handle no-op `POSITION` over replication. ([\#16640](https://github.com/matrix-org/synapse/issues/16640), [\#16655](https://github.com/matrix-org/synapse/issues/16655)) +- Speed up deleting of device messages when deleting a device. ([\#16643](https://github.com/matrix-org/synapse/issues/16643)) +- Speed up persisting large number of outliers. ([\#16649](https://github.com/matrix-org/synapse/issues/16649)) +- Reduce max concurrency of background tasks, reducing potential max DB load. ([\#16656](https://github.com/matrix-org/synapse/issues/16656), [\#16660](https://github.com/matrix-org/synapse/issues/16660)) +- Speed up purge room by adding an index to `event_push_summary`. ([\#16657](https://github.com/matrix-org/synapse/issues/16657)) + + + +### Updates to locked dependencies + +* Bump prometheus-client from 0.17.1 to 0.18.0. ([\#16626](https://github.com/matrix-org/synapse/issues/16626)) +* Bump pyicu from 2.11 to 2.12. ([\#16603](https://github.com/matrix-org/synapse/issues/16603)) +* Bump requests-toolbelt from 0.10.1 to 1.0.0. ([\#16659](https://github.com/matrix-org/synapse/issues/16659)) +* Bump ruff from 0.0.292 to 0.1.4. ([\#16600](https://github.com/matrix-org/synapse/issues/16600)) +* Bump serde from 1.0.190 to 1.0.192. ([\#16627](https://github.com/matrix-org/synapse/issues/16627)) +* Bump serde_json from 1.0.107 to 1.0.108. ([\#16604](https://github.com/matrix-org/synapse/issues/16604)) +* Bump setuptools-rust from 1.8.0 to 1.8.1. ([\#16601](https://github.com/matrix-org/synapse/issues/16601)) +* Bump towncrier from 23.6.0 to 23.11.0. ([\#16622](https://github.com/matrix-org/synapse/issues/16622)) +* Bump treq from 22.2.0 to 23.11.0. ([\#16623](https://github.com/matrix-org/synapse/issues/16623)) +* Bump twisted from 23.8.0 to 23.10.0. ([\#16588](https://github.com/matrix-org/synapse/issues/16588)) +* Bump types-bleach from 6.1.0.0 to 6.1.0.1. ([\#16624](https://github.com/matrix-org/synapse/issues/16624)) +* Bump types-jsonschema from 4.19.0.3 to 4.19.0.4. ([\#16599](https://github.com/matrix-org/synapse/issues/16599)) +* Bump types-pyopenssl from 23.2.0.2 to 23.3.0.0. ([\#16625](https://github.com/matrix-org/synapse/issues/16625)) +* Bump types-pyyaml from 6.0.12.11 to 6.0.12.12. ([\#16602](https://github.com/matrix-org/synapse/issues/16602)) + +# Synapse 1.96.1 (2023-11-17) + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + +* https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ +* https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual +[announcements channels](https://matrix.to/#/#homeowners:matrix.org), but we are +striving to make this as seamless as possible. + +This minor release was needed only because of CI-related trouble on [v1.96.0](https://github.com/matrix-org/synapse/releases/tag/v1.96.0), which was never released. + +### Internal Changes + +- Fix building of wheels in CI. ([\#16653](https://github.com/matrix-org/synapse/issues/16653)) + +# Synapse 1.96.0 (2023-11-16) + +### Bugfixes + +- Fix "'int' object is not iterable" error in `set_device_id_for_pushers` background update introduced in Synapse 1.95.0. ([\#16594](https://github.com/matrix-org/synapse/issues/16594)) + +# Synapse 1.96.0rc1 (2023-10-31) + +### Features + +- Add experimental support to allow multiple workers to write to receipts stream. ([\#16432](https://github.com/matrix-org/synapse/issues/16432)) +- Add a new module API for controller presence. ([\#16544](https://github.com/matrix-org/synapse/issues/16544)) +- Add a new module API callback that allows adding extra fields to events' unsigned section when sent down to clients. ([\#16549](https://github.com/matrix-org/synapse/issues/16549)) +- Improve the performance of claiming encryption keys. ([\#16565](https://github.com/matrix-org/synapse/issues/16565), [\#16570](https://github.com/matrix-org/synapse/issues/16570)) + +### Bugfixes + +- Fixed a bug in the example Grafana dashboard that prevents it from finding the correct datasource. Contributed by @MichaelSasser. ([\#16471](https://github.com/matrix-org/synapse/issues/16471)) +- Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. ([\#16473](https://github.com/matrix-org/synapse/issues/16473), [\#16557](https://github.com/matrix-org/synapse/issues/16557), [\#16561](https://github.com/matrix-org/synapse/issues/16561), [\#16578](https://github.com/matrix-org/synapse/issues/16578), [\#16580](https://github.com/matrix-org/synapse/issues/16580)) +- Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events. ([\#16485](https://github.com/matrix-org/synapse/issues/16485)) +- Fix a bug introduced in Synapse 1.41 where HTTP(S) forward proxy authorization would fail when using basic HTTP authentication with a long `username:password` string. ([\#16504](https://github.com/matrix-org/synapse/issues/16504)) +- Force TLS certificate verification in user registration script. ([\#16530](https://github.com/matrix-org/synapse/issues/16530)) +- Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. ([\#16540](https://github.com/matrix-org/synapse/issues/16540)) +- Fix ratelimiting of message sending when using workers, where the ratelimit would only be applied after most of the work has been done. ([\#16558](https://github.com/matrix-org/synapse/issues/16558)) +- Fix a long-standing bug where invited/knocking users would not leave during a room purge. ([\#16559](https://github.com/matrix-org/synapse/issues/16559)) + +### Improved Documentation + +- Improve documentation of presence router. ([\#16529](https://github.com/matrix-org/synapse/issues/16529)) +- Add a sentence to the [opentracing docs](https://matrix-org.github.io/synapse/latest/opentracing.html) on how you can have jaeger in a different place than synapse. ([\#16531](https://github.com/matrix-org/synapse/issues/16531)) +- Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. ([\#16541](https://github.com/matrix-org/synapse/issues/16541)) +- Pin the recommended poetry version in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16550](https://github.com/matrix-org/synapse/issues/16550)) +- Fix a broken link to the [client breakdown](https://matrix.org/ecosystem/clients/) in the README. ([\#16569](https://github.com/matrix-org/synapse/issues/16569)) + +### Internal Changes + +- Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479). ([\#16492](https://github.com/matrix-org/synapse/issues/16492)) +- Reduce memory allocations. ([\#16505](https://github.com/matrix-org/synapse/issues/16505)) +- Improve replication performance when purging rooms. ([\#16510](https://github.com/matrix-org/synapse/issues/16510)) +- Run tests against Python 3.12. ([\#16511](https://github.com/matrix-org/synapse/issues/16511)) +- Run trial & integration tests in continuous integration when `.ci` directory is modified. ([\#16512](https://github.com/matrix-org/synapse/issues/16512)) +- Remove duplicate call to mark remote server 'awake' when using a federation sending worker. ([\#16515](https://github.com/matrix-org/synapse/issues/16515)) +- Enable dirty runs on Complement CI, which is significantly faster. ([\#16520](https://github.com/matrix-org/synapse/issues/16520)) +- Stop deleting from an unused table. ([\#16521](https://github.com/matrix-org/synapse/issues/16521)) +- Improve type hints. ([\#16526](https://github.com/matrix-org/synapse/issues/16526), [\#16551](https://github.com/matrix-org/synapse/issues/16551)) +- Fix running unit tests on Twisted trunk. ([\#16528](https://github.com/matrix-org/synapse/issues/16528)) +- Reduce some spurious logging in worker mode. ([\#16555](https://github.com/matrix-org/synapse/issues/16555)) +- Stop porting a table in port db that we're going to nuke and rebuild anyway. ([\#16563](https://github.com/matrix-org/synapse/issues/16563)) +- Deal with warnings from running complement in CI. ([\#16567](https://github.com/matrix-org/synapse/issues/16567)) +- Allow building with `setuptools_rust` 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) + +### Updates to locked dependencies + +* Bump black from 23.10.0 to 23.10.1. ([\#16575](https://github.com/matrix-org/synapse/issues/16575)) +* Bump black from 23.9.1 to 23.10.0. ([\#16538](https://github.com/matrix-org/synapse/issues/16538)) +* Bump cryptography from 41.0.4 to 41.0.5. ([\#16572](https://github.com/matrix-org/synapse/issues/16572)) +* Bump gitpython from 3.1.37 to 3.1.40. ([\#16534](https://github.com/matrix-org/synapse/issues/16534)) +* Bump phonenumbers from 8.13.22 to 8.13.23. ([\#16576](https://github.com/matrix-org/synapse/issues/16576)) +* Bump pygithub from 1.59.1 to 2.1.1. ([\#16535](https://github.com/matrix-org/synapse/issues/16535)) +- Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. ([\#16539](https://github.com/matrix-org/synapse/issues/16539)) +* Bump serde from 1.0.189 to 1.0.190. ([\#16577](https://github.com/matrix-org/synapse/issues/16577)) +* Bump setuptools-rust from 1.7.0 to 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574)) +* Bump types-pillow from 10.0.0.3 to 10.1.0.0. ([\#16536](https://github.com/matrix-org/synapse/issues/16536)) +* Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15. ([\#16573](https://github.com/matrix-org/synapse/issues/16573)) +* Bump types-requests from 2.31.0.2 to 2.31.0.10. ([\#16537](https://github.com/matrix-org/synapse/issues/16537)) +* Bump urllib3 from 1.26.17 to 1.26.18. ([\#16516](https://github.com/matrix-org/synapse/issues/16516)) + +# Synapse 1.95.1 (2023-10-31) + +## Security advisory + +The following issue is fixed in 1.95.1. + +- [GHSA-mp92-3jfm-3575](https://github.com/matrix-org/synapse/security/advisories/GHSA-mp92-3jfm-3575) / [CVE-2023-43796](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-43796) — Moderate Severity + + Cached device information of remote users can be queried from Synapse. This can be used to enumerate the remote users known to a homeserver. + +See the advisory for more details. If you have any questions, email security@matrix.org. + + + +# Synapse 1.95.0 (2023-10-24) + +### Internal Changes + +- Build Debian packages for [Ubuntu 23.10 Mantic Minotaur](https://canonical.com/blog/canonical-releases-ubuntu-23-10-mantic-minotaur). ([\#16524](https://github.com/matrix-org/synapse/issues/16524)) + + +# Synapse 1.95.0rc1 (2023-10-17) + +### Bugfixes + +- Remove legacy unspecced `knock_state_events` field returned in some responses. ([\#16403](https://github.com/matrix-org/synapse/issues/16403)) +- Fix a bug introduced in Synapse 1.81.0 where an `AttributeError` would be raised when `_matrix/client/v3/account/whoami` is called over a unix socket. Contributed by @Sir-Photch. ([\#16404](https://github.com/matrix-org/synapse/issues/16404)) +- Properly return inline media when content types have parameters. ([\#16440](https://github.com/matrix-org/synapse/issues/16440)) +- Prevent the purging of large rooms from timing out when Postgres is in use. The timeout which causes this issue was introduced in Synapse 1.88.0. ([\#16455](https://github.com/matrix-org/synapse/issues/16455)) +- Improve the performance of purging rooms, particularly encrypted rooms. ([\#16457](https://github.com/matrix-org/synapse/issues/16457)) +- Fix a bug introduced in Synapse 1.59.0 where servers could be incorrectly marked as available after an error response was received. ([\#16506](https://github.com/matrix-org/synapse/issues/16506)) + +### Improved Documentation + +- Document internal background update mechanism. ([\#16420](https://github.com/matrix-org/synapse/issues/16420)) +- Fix a typo in the sql for [useful SQL for admins document](https://matrix-org.github.io/synapse/latest/usage/administration/useful_sql_for_admins.html). ([\#16477](https://github.com/matrix-org/synapse/issues/16477)) + +### Internal Changes + +- Bump pyo3 from 0.17.1 to 0.19.2. ([\#16162](https://github.com/matrix-org/synapse/issues/16162)) +- Update registration of media repository URLs. ([\#16419](https://github.com/matrix-org/synapse/issues/16419)) +- Improve type hints. ([\#16421](https://github.com/matrix-org/synapse/issues/16421), [\#16468](https://github.com/matrix-org/synapse/issues/16468), [\#16469](https://github.com/matrix-org/synapse/issues/16469), [\#16507](https://github.com/matrix-org/synapse/issues/16507)) +- Refactor some code to simplify and better type receipts stream adjacent code. ([\#16426](https://github.com/matrix-org/synapse/issues/16426)) +- Factor out `MultiWriter` token from `RoomStreamToken`. ([\#16427](https://github.com/matrix-org/synapse/issues/16427)) +- Improve code comments. ([\#16428](https://github.com/matrix-org/synapse/issues/16428)) +- Reduce memory allocations. ([\#16429](https://github.com/matrix-org/synapse/issues/16429), [\#16431](https://github.com/matrix-org/synapse/issues/16431), [\#16433](https://github.com/matrix-org/synapse/issues/16433), [\#16434](https://github.com/matrix-org/synapse/issues/16434), [\#16438](https://github.com/matrix-org/synapse/issues/16438), [\#16444](https://github.com/matrix-org/synapse/issues/16444)) +- Remove unused method. ([\#16435](https://github.com/matrix-org/synapse/issues/16435)) +- Improve rate limiting logic. ([\#16441](https://github.com/matrix-org/synapse/issues/16441)) +- Do not block running of CI behind the check for sign-off on PRs. ([\#16454](https://github.com/matrix-org/synapse/issues/16454)) +- Update the release script to remind releaser to check for special release notes. ([\#16461](https://github.com/matrix-org/synapse/issues/16461)) +- Update complement.sh to match new public API shape. ([\#16466](https://github.com/matrix-org/synapse/issues/16466)) +- Clean up logging on event persister endpoints. ([\#16488](https://github.com/matrix-org/synapse/issues/16488)) +- Remove useless async job to delete device messages on sync, since we only deliver (and hence delete) up to 100 device messages at a time. ([\#16491](https://github.com/matrix-org/synapse/issues/16491)) + +### Updates to locked dependencies + +* Bump bleach from 6.0.0 to 6.1.0. ([\#16451](https://github.com/matrix-org/synapse/issues/16451)) +* Bump jsonschema from 4.19.0 to 4.19.1. ([\#16500](https://github.com/matrix-org/synapse/issues/16500)) +* Bump netaddr from 0.8.0 to 0.9.0. ([\#16453](https://github.com/matrix-org/synapse/issues/16453)) +* Bump packaging from 23.1 to 23.2. ([\#16497](https://github.com/matrix-org/synapse/issues/16497)) +* Bump pillow from 10.0.1 to 10.1.0. ([\#16498](https://github.com/matrix-org/synapse/issues/16498)) +* Bump psycopg2 from 2.9.8 to 2.9.9. ([\#16452](https://github.com/matrix-org/synapse/issues/16452)) +* Bump pyo3-log from 0.8.3 to 0.8.4. ([\#16495](https://github.com/matrix-org/synapse/issues/16495)) +* Bump ruff from 0.0.290 to 0.0.292. ([\#16449](https://github.com/matrix-org/synapse/issues/16449)) +* Bump sentry-sdk from 1.31.0 to 1.32.0. ([\#16496](https://github.com/matrix-org/synapse/issues/16496)) +* Bump serde from 1.0.188 to 1.0.189. ([\#16494](https://github.com/matrix-org/synapse/issues/16494)) +* Bump types-bleach from 6.0.0.4 to 6.1.0.0. ([\#16450](https://github.com/matrix-org/synapse/issues/16450)) +* Bump types-jsonschema from 4.17.0.10 to 4.19.0.3. ([\#16499](https://github.com/matrix-org/synapse/issues/16499)) + +# Synapse 1.94.0 (2023-10-10) + +No significant changes since 1.94.0rc1. +However, please take note of the security advisory that follows. + +## Security advisory + +The following issue is fixed in 1.94.0 (and RC). + +- [GHSA-5chr-wjw5-3gq4](https://github.com/matrix-org/synapse/security/advisories/GHSA-5chr-wjw5-3gq4) / [CVE-2023-45129](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-45129) — Moderate Severity + + A malicious server ACL event can impact performance temporarily or permanently leading to a persistent denial of service. + + Homeservers running on a closed federation (which presumably do not need to use server ACLs) are not affected. + +See the advisory for more details. If you have any questions, email security@matrix.org. + + +# Synapse 1.94.0rc1 (2023-10-03) + +### Features + +- Render plain, CSS, CSV, JSON and common image formats in the browser (inline) when requested through the /download endpoint. ([\#15988](https://github.com/matrix-org/synapse/issues/15988)) +- Add experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) to push all encrypted events to clients. ([\#16361](https://github.com/matrix-org/synapse/issues/16361)) +- Minor performance improvement when sending presence to federated servers. ([\#16385](https://github.com/matrix-org/synapse/issues/16385)) +- Minor performance improvement by caching server ACL checking. ([\#16360](https://github.com/matrix-org/synapse/issues/16360)) + +### Improved Documentation + +- Add developer documentation concerning gradual schema migrations with column alterations. ([\#15691](https://github.com/matrix-org/synapse/issues/15691)) +- Improve documentation of the user directory search algorithm. ([\#16320](https://github.com/matrix-org/synapse/issues/16320)) +- Fix rendering of user admin API documentation around deactivation. This was broken in Synapse 1.91.0. ([\#16355](https://github.com/matrix-org/synapse/issues/16355)) +- Update documentation around message retention policies. ([\#16382](https://github.com/matrix-org/synapse/issues/16382)) +- Add note to `federation_domain_whitelist` config option to clarify its usage. ([\#16416](https://github.com/matrix-org/synapse/issues/16416)) +- Improve legacy release notes. ([\#16418](https://github.com/matrix-org/synapse/issues/16418)) + +### Deprecations and Removals + +- Remove Python version from `/_synapse/admin/v1/server_version`. ([\#16380](https://github.com/matrix-org/synapse/issues/16380)) + +### Internal Changes + +- Avoid running CI steps when the files they check have not been changed. ([\#14745](https://github.com/matrix-org/synapse/issues/14745), [\#16387](https://github.com/matrix-org/synapse/issues/16387)) +- Improve type hints. ([\#14911](https://github.com/matrix-org/synapse/issues/14911), [\#16350](https://github.com/matrix-org/synapse/issues/16350), [\#16356](https://github.com/matrix-org/synapse/issues/16356), [\#16395](https://github.com/matrix-org/synapse/issues/16395)) +- Added support for pydantic v2 in addition to pydantic v1. Contributed by Maxwell G (@gotmax23). ([\#16332](https://github.com/matrix-org/synapse/issues/16332)) +- Get CI to check PRs have been signed-off. ([\#16348](https://github.com/matrix-org/synapse/issues/16348)) +- Add missing licence header. ([\#16359](https://github.com/matrix-org/synapse/issues/16359)) +- Improve type hints, and bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) +- Improve comments in `StateGroupBackgroundUpdateStore`. ([\#16383](https://github.com/matrix-org/synapse/issues/16383)) +- Update maturin configuration. ([\#16394](https://github.com/matrix-org/synapse/issues/16394)) +- Downgrade replication stream time out error log lines to warning. ([\#16401](https://github.com/matrix-org/synapse/issues/16401)) + +### Updates to locked dependencies + +* Bump actions/checkout from 3 to 4. ([\#16250](https://github.com/matrix-org/synapse/issues/16250)) +* Bump cryptography from 41.0.3 to 41.0.4. ([\#16362](https://github.com/matrix-org/synapse/issues/16362)) +* Bump dawidd6/action-download-artifact from 2.27.0 to 2.28.0. ([\#16374](https://github.com/matrix-org/synapse/issues/16374)) +* Bump docker/setup-buildx-action from 2 to 3. ([\#16375](https://github.com/matrix-org/synapse/issues/16375)) +* Bump gitpython from 3.1.35 to 3.1.37. ([\#16376](https://github.com/matrix-org/synapse/issues/16376)) +* Bump msgpack from 1.0.5 to 1.0.6. ([\#16377](https://github.com/matrix-org/synapse/issues/16377)) +* Bump msgpack from 1.0.6 to 1.0.7. ([\#16412](https://github.com/matrix-org/synapse/issues/16412)) +* Bump phonenumbers from 8.13.19 to 8.13.22. ([\#16413](https://github.com/matrix-org/synapse/issues/16413)) +* Bump psycopg2 from 2.9.7 to 2.9.8. ([\#16409](https://github.com/matrix-org/synapse/issues/16409)) +* Bump pydantic from 2.3.0 to 2.4.2. ([\#16410](https://github.com/matrix-org/synapse/issues/16410)) +* Bump regex from 1.9.5 to 1.9.6. ([\#16408](https://github.com/matrix-org/synapse/issues/16408)) +* Bump sentry-sdk from 1.30.0 to 1.31.0. ([\#16378](https://github.com/matrix-org/synapse/issues/16378)) +* Bump types-netaddr from 0.8.0.9 to 0.9.0.1. ([\#16411](https://github.com/matrix-org/synapse/issues/16411)) +* Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14. ([\#16381](https://github.com/matrix-org/synapse/issues/16381)) +* Bump urllib3 from 1.26.15 to 1.26.17. ([\#16422](https://github.com/matrix-org/synapse/issues/16422)) + +# Synapse 1.93.0 (2023-09-26) + +No significant changes since 1.93.0rc1. + + +## Security advisory + +The following issues are fixed in 1.93.0 (and RCs). + +- [GHSA-4f74-84v3-j9q5](https://github.com/matrix-org/synapse/security/advisories/GHSA-4f74-84v3-j9q5) / [CVE-2023-41335](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-41335) — Low Severity + + Temporary storage of plaintext passwords during password changes. + +- [GHSA-7565-cq32-vx2x](https://github.com/matrix-org/synapse/security/advisories/GHSA-7565-cq32-vx2x) / [CVE-2023-42453](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-42453) — Low Severity + + Improper validation of receipts allows forged read receipts. + +See the advisories for more details. If you have any questions, email security@matrix.org. + + +# Synapse 1.93.0rc1 (2023-09-19) + +### Features + +- Add automatic purge after all users have forgotten a room. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) +- Restore room purge/shutdown after a Synapse restart. ([\#15488](https://github.com/matrix-org/synapse/issues/15488)) +- Support resolving homeservers using `matrix-fed` DNS SRV records from [MSC4040](https://github.com/matrix-org/matrix-spec-proposals/pull/4040). ([\#16137](https://github.com/matrix-org/synapse/issues/16137)) +- Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes. ([\#16219](https://github.com/matrix-org/synapse/issues/16219)) +- Add span information to requests sent to appservices. Contributed by MTRNord. ([\#16227](https://github.com/matrix-org/synapse/issues/16227)) +- Add the ability to enable/disable registrations when using CAS. Contributed by Aurélien Grimpard. ([\#16262](https://github.com/matrix-org/synapse/issues/16262)) +- Allow the `/notifications` endpoint to be routed to workers. ([\#16265](https://github.com/matrix-org/synapse/issues/16265)) +- Enable users to easily unsubscribe to notifications emails via the `List-Unsubscribe` header. ([\#16274](https://github.com/matrix-org/synapse/issues/16274)) +- Report whether a user is `locked` in the [List Accounts admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#list-accounts), and exclude locked users by default. ([\#16328](https://github.com/matrix-org/synapse/issues/16328)) + +### Bugfixes + +- Fix a long-standing bug where multi-device accounts could cause high load due to presence. ([\#16066](https://github.com/matrix-org/synapse/issues/16066), [\#16170](https://github.com/matrix-org/synapse/issues/16170), [\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172), [\#16174](https://github.com/matrix-org/synapse/issues/16174)) +- Fix a long-standing bug where appservices using [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) to receive `to_device` messages would only get messages for one user. ([\#16251](https://github.com/matrix-org/synapse/issues/16251)) +- Fix bug when using workers where Synapse could end up re-requesting the same remote device repeatedly. ([\#16252](https://github.com/matrix-org/synapse/issues/16252)) +- Fix long-standing bug where we kept re-requesting a remote server's key repeatedly, potentially causing delays in receiving events over federation. ([\#16257](https://github.com/matrix-org/synapse/issues/16257)) +- Avoid temporary storage of sensitive information. ([\#16272](https://github.com/matrix-org/synapse/issues/16272)) +- Fix bug introduced in Synapse 1.49.0 when using dehydrated devices ([MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697)) and refresh tokens. Contributed by Hanadi. ([\#16288](https://github.com/matrix-org/synapse/issues/16288)) +- Fix a long-standing bug where invalid receipts would be accepted. ([\#16327](https://github.com/matrix-org/synapse/issues/16327)) +- Use standard name for UTF-8 charset in emails. ([\#16329](https://github.com/matrix-org/synapse/issues/16329)) +- Don't try refetching device lists for users on remote hosts that are marked as "down". ([\#16298](https://github.com/matrix-org/synapse/issues/16298)) + +### Improved Documentation + +- Fix typos in the documentation. ([\#16282](https://github.com/matrix-org/synapse/issues/16282)) +- Link to the Alpine Linux community package for Synapse. ([\#16304](https://github.com/matrix-org/synapse/issues/16304)) +- Use string for `federation_client_minimum_tls_version` documentation examples. Contributed by @jcgruenhage. ([\#16353](https://github.com/matrix-org/synapse/issues/16353)) + +### Internal Changes + +- Allow modules to delete rooms. ([\#15997](https://github.com/matrix-org/synapse/issues/15997)) +- Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled. ([\#16090](https://github.com/matrix-org/synapse/issues/16090), [\#16263](https://github.com/matrix-org/synapse/issues/16263)) +- Fix type checking when using the new version of Twisted. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) +- Delete device messages asynchronously and in staged batches using the task scheduler. ([\#16240](https://github.com/matrix-org/synapse/issues/16240), [\#16311](https://github.com/matrix-org/synapse/issues/16311), [\#16312](https://github.com/matrix-org/synapse/issues/16312), [\#16313](https://github.com/matrix-org/synapse/issues/16313)) +- Bump minimum supported Rust version to 1.61.0. ([\#16248](https://github.com/matrix-org/synapse/issues/16248)) +- Update rust to version 1.71.1 in the nix development environment. ([\#16260](https://github.com/matrix-org/synapse/issues/16260)) +- Simplify server key storage. ([\#16261](https://github.com/matrix-org/synapse/issues/16261)) +- Reduce CPU overhead of change password endpoint. ([\#16264](https://github.com/matrix-org/synapse/issues/16264)) +- Stop purging from tables slated for removal. ([\#16273](https://github.com/matrix-org/synapse/issues/16273)) +- Improve type hints. ([\#16276](https://github.com/matrix-org/synapse/issues/16276), [\#16301](https://github.com/matrix-org/synapse/issues/16301), [\#16325](https://github.com/matrix-org/synapse/issues/16325), [\#16326](https://github.com/matrix-org/synapse/issues/16326)) +- Raise `setuptools_rust` version cap to 1.7.0. ([\#16277](https://github.com/matrix-org/synapse/issues/16277)) +- Fix using the new task scheduler causing lots of CPU to be used. ([\#16278](https://github.com/matrix-org/synapse/issues/16278)) +- Upgrade CI run of Python 3.12 from rc1 to rc2. ([\#16280](https://github.com/matrix-org/synapse/issues/16280)) +- Include values in SQL debug when using `execute_values` with Postgres. ([\#16281](https://github.com/matrix-org/synapse/issues/16281)) +- Enable additional linting checks. ([\#16283](https://github.com/matrix-org/synapse/issues/16283)) +- Refactor `receipts_graph` Postgres transactions to stop error messages. ([\#16299](https://github.com/matrix-org/synapse/issues/16299)) +- Small improvements to logging in replication code. ([\#16309](https://github.com/matrix-org/synapse/issues/16309)) +- Remove a reference cycle in background processes. ([\#16314](https://github.com/matrix-org/synapse/issues/16314)) +- Only use literal strings for background process names. ([\#16315](https://github.com/matrix-org/synapse/issues/16315)) +- Refactor `get_user_by_id`. ([\#16316](https://github.com/matrix-org/synapse/issues/16316)) +- Speed up task to delete to-device messages. ([\#16318](https://github.com/matrix-org/synapse/issues/16318)) +- Avoid patching code in tests. ([\#16349](https://github.com/matrix-org/synapse/issues/16349)) +- Test against PostgreSQL 16. ([\#16351](https://github.com/matrix-org/synapse/issues/16351)) + +### Updates to locked dependencies + +* Bump mypy from 1.4.1 to 1.5.1. ([\#16300](https://github.com/matrix-org/synapse/issues/16300)) +* Bump black from 23.7.0 to 23.9.1. ([\#16295](https://github.com/matrix-org/synapse/issues/16295)) +* Bump docker/build-push-action from 4 to 5. ([\#16336](https://github.com/matrix-org/synapse/issues/16336)) +* Bump docker/login-action from 2 to 3. ([\#16339](https://github.com/matrix-org/synapse/issues/16339)) +* Bump docker/metadata-action from 4 to 5. ([\#16337](https://github.com/matrix-org/synapse/issues/16337)) +* Bump docker/setup-qemu-action from 2 to 3. ([\#16338](https://github.com/matrix-org/synapse/issues/16338)) +* Bump furo from 2023.8.19 to 2023.9.10. ([\#16340](https://github.com/matrix-org/synapse/issues/16340)) +* Bump gitpython from 3.1.32 to 3.1.35. ([\#16267](https://github.com/matrix-org/synapse/issues/16267), [\#16279](https://github.com/matrix-org/synapse/issues/16279)) +* Bump mypy-zope from 1.0.0 to 1.0.1. ([\#16291](https://github.com/matrix-org/synapse/issues/16291)) +* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) +* Bump regex from 1.9.4 to 1.9.5. ([\#16233](https://github.com/matrix-org/synapse/issues/16233)) +* Bump ruff from 0.0.286 to 0.0.290. ([\#16342](https://github.com/matrix-org/synapse/issues/16342)) +* Bump serde_json from 1.0.105 to 1.0.107. ([\#16296](https://github.com/matrix-org/synapse/issues/16296), [\#16345](https://github.com/matrix-org/synapse/issues/16345)) +* Bump twisted from 22.10.0 to 23.8.0. ([\#16235](https://github.com/matrix-org/synapse/issues/16235)) +* Bump types-pillow from 10.0.0.2 to 10.0.0.3. ([\#16293](https://github.com/matrix-org/synapse/issues/16293)) +* Bump types-setuptools from 68.0.0.3 to 68.2.0.0. ([\#16292](https://github.com/matrix-org/synapse/issues/16292)) +* Bump typing-extensions from 4.7.1 to 4.8.0. ([\#16341](https://github.com/matrix-org/synapse/issues/16341)) + +# Synapse 1.92.3 (2023-09-18) + +This is again a security update targeted at mitigating [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863). +It turns out that libwebp is bundled statically in Pillow wheels so we need to update this dependency instead of +libwebp package at the OS level. + +Unlike what was advertised in 1.92.2 changelog this release also impacts PyPI wheels and Debian packages from matrix.org. + +We encourage admins to upgrade as soon as possible. + + +### Internal Changes + +- Pillow 10.0.1 is now mandatory because of libwebp CVE-2023-4863, since Pillow provides libwebp in the wheels. ([\#16347](https://github.com/matrix-org/synapse/issues/16347)) + +### Updates to locked dependencies + +* Bump pillow from 10.0.0 to 10.0.1. ([\#16344](https://github.com/matrix-org/synapse/issues/16344)) + +# Synapse 1.92.2 (2023-09-15) + +This is a Docker-only update to mitigate [CVE-2023-4863](https://cve.org/CVERecord?id=CVE-2023-4863), a critical vulnerability in `libwebp`. Server admins not using Docker should ensure that their `libwebp` is up to date (if installed). We encourage admins to upgrade as soon as possible. + + +### Updates to the Docker image + +- Update docker image to use Debian bookworm as the base. ([\#16324](https://github.com/matrix-org/synapse/issues/16324)) + + +# Synapse 1.92.1 (2023-09-12) + +This minor release was needed only because of CI-related trouble on [v1.92.0](https://github.com/matrix-org/synapse/releases/tag/v1.92.0), which was never released. + +### Internal Changes + +- Stop building Ubuntu Kinetic since it is EOL and repos seem to be dead. + + +# Synapse 1.92.0 (2023-09-12) + +This release includes the same [bugfix](https://github.com/matrix-org/synapse/issues/16258) as Synapse 1.91.2. + +This version was never released following a CI build failure, cf [v1.92.1 changelog](https://github.com/matrix-org/synapse/releases/tag/v1.92.1). + +### Bugfixes + +- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) + +### Internal Changes + +- Fix incorrect docstring for `Ratelimiter`. ([\#16255](https://github.com/matrix-org/synapse/issues/16255)) +- Update the release script to work on macOS. ([\#16266](https://github.com/matrix-org/synapse/issues/16266)) + + +# Synapse 1.91.2 (2023-09-06) + +### Bugfixes + +- Revert [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) introspection cache, admin impersonation and account lock. ([\#16258](https://github.com/matrix-org/synapse/issues/16258)) + + +# Synapse 1.92.0rc1 (2023-09-05) + +### Features + +- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816)) +- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113)) +- Experimental support for [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041): return a `Retry-After` header with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136)) +- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218)) +- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223)) + +### Bugfixes + +- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155)) +- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185)) +- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205)) +- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210)) +- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211)) +- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221)) + +### Improved Documentation + +- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008)) +- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168)) +- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178)) + +### Internal Changes + +- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099)) +- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213)) +- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135)) +- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170)) +- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172)) +- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175)) +- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180)) +- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183)) +- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184)) +- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201)) +- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187)) +- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212)) +- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141)) +* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238)) +* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237)) +* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196)) +* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195)) +* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198)) +* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236)) +* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194)) +* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140)) +* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200)) +* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199)) + + +# Synapse 1.91.1 (2023-09-04) + +### Bugfixes + +- Fix a performance regression introduced in Synapse 1.91.0 where event persistence would cause an excessive linear growth in CPU usage. ([\#16220](https://github.com/matrix-org/synapse/issues/16220)) + + +# Synapse 1.91.0 (2023-08-30) + +No significant changes since 1.91.0rc1. + + +# Synapse 1.91.0rc1 (2023-08-23) + +### Features + +- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) +- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) +- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) +- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) +- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) + +### Bugfixes + +- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) +- Fix a long-standing bu in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) +- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) +- Filter out user agent references to the sliding sync proxy and rust-sdk from the user_daily_visits table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) +- User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) +- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) +- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) +- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) + +### Improved Documentation + +- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) + +### Internal Changes + +- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) +- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) +- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) +- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) +- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) +- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) +- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) +- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) +- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) +- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) +- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) +- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) +- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) +- MSC3861: allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) +- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) +- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) +- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) +- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) +- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) +- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) +- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) + +### Updates to locked dependencies + +* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) +* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) +* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) +* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) +* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) +* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) +* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) +* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) +* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) +* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) +* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) +* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) + +# Synapse 1.91.0rc1 (2023-08-23) + +### Features + +- Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939). ([\#15870](https://github.com/matrix-org/synapse/issues/15870)) +- Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27. ([\#16030](https://github.com/matrix-org/synapse/issues/16030)) +- Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider). ([\#16094](https://github.com/matrix-org/synapse/issues/16094)) +- Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries. ([\#16114](https://github.com/matrix-org/synapse/issues/16114)) + +### Bugfixes + +- Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar). ([\#16052](https://github.com/matrix-org/synapse/issues/16052)) +- Fix a long-standing bug in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec. ([\#16080](https://github.com/matrix-org/synapse/issues/16080)) +- Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted. ([\#16116](https://github.com/matrix-org/synapse/issues/16116)) +- Filter out user agent references to the sliding sync proxy and rust-sdk from the `user_daily_visits` table to ensure that Element X can be represented fully. ([\#16124](https://github.com/matrix-org/synapse/issues/16124)) +- User constent and third-party ID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support. ([\#16127](https://github.com/matrix-org/synapse/issues/16127), [\#16134](https://github.com/matrix-org/synapse/issues/16134)) +- Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. ([\#16133](https://github.com/matrix-org/synapse/issues/16133), [\#16169](https://github.com/matrix-org/synapse/issues/16169)) +- Fix performance degredation when there are a lot of in-flight replication requests. ([\#16148](https://github.com/matrix-org/synapse/issues/16148)) +- Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico. ([\#16156](https://github.com/matrix-org/synapse/issues/16156), [\#16164](https://github.com/matrix-org/synapse/issues/16164)) + +### Improved Documentation + +- Structured logging docs: add a link to explain the ELK stack ([\#16091](https://github.com/matrix-org/synapse/issues/16091)) + +### Internal Changes + +- Update dehydrated devices implementation. ([\#16010](https://github.com/matrix-org/synapse/issues/16010)) +- Fix database performance of read/write worker locks. ([\#16061](https://github.com/matrix-org/synapse/issues/16061)) +- Fix building the nix development environment on MacOS systems. ([\#16063](https://github.com/matrix-org/synapse/issues/16063)) +- Override global statement timeout when creating indexes in Postgres. ([\#16085](https://github.com/matrix-org/synapse/issues/16085)) +- Fix the type annotation on `run_db_interaction` in the Module API. ([\#16089](https://github.com/matrix-org/synapse/issues/16089)) +- Clean-up the presence code. ([\#16092](https://github.com/matrix-org/synapse/issues/16092)) +- Run `pyupgrade` for Python 3.8+. ([\#16110](https://github.com/matrix-org/synapse/issues/16110)) +- Rename pagination and purge locks and add comments to explain why they exist and how they work. ([\#16112](https://github.com/matrix-org/synapse/issues/16112)) +- Attempt to fix the twisted trunk job. ([\#16115](https://github.com/matrix-org/synapse/issues/16115)) +- Cache token introspection response from OIDC provider. ([\#16117](https://github.com/matrix-org/synapse/issues/16117)) +- Add cache to `get_server_keys_json_for_remote`. ([\#16123](https://github.com/matrix-org/synapse/issues/16123)) +- Add an admin endpoint to allow authorizing server to signal token revocations. ([\#16125](https://github.com/matrix-org/synapse/issues/16125)) +- Add response time metrics for introspection requests for delegated auth. ([\#16131](https://github.com/matrix-org/synapse/issues/16131)) +- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter. ([\#16132](https://github.com/matrix-org/synapse/issues/16132)) +- Increase performance of read/write locks. ([\#16149](https://github.com/matrix-org/synapse/issues/16149)) +- Improve presence tests. ([\#16150](https://github.com/matrix-org/synapse/issues/16150), [\#16151](https://github.com/matrix-org/synapse/issues/16151), [\#16158](https://github.com/matrix-org/synapse/issues/16158)) +- Raised the poetry-core version cap to 1.7.0. ([\#16152](https://github.com/matrix-org/synapse/issues/16152)) +- Fix assertion in user directory unit tests. ([\#16157](https://github.com/matrix-org/synapse/issues/16157)) +- Reduce scope of locks when paginating to alleviate DB contention. ([\#16159](https://github.com/matrix-org/synapse/issues/16159)) +- Reduce DB contention on worker locks. ([\#16160](https://github.com/matrix-org/synapse/issues/16160)) +- Task scheduler: mark task as active if we are scheduling as soon as possible. ([\#16165](https://github.com/matrix-org/synapse/issues/16165)) +- Implements a task scheduler for resumable potentially long running tasks. ([\#15891](https://github.com/matrix-org/synapse/issues/15891)) + +### Updates to locked dependencies + +* Bump click from 8.1.6 to 8.1.7. ([\#16145](https://github.com/matrix-org/synapse/issues/16145)) +* Bump gitpython from 3.1.31 to 3.1.32. ([\#16103](https://github.com/matrix-org/synapse/issues/16103)) +* Bump ijson from 3.2.1 to 3.2.3. ([\#16143](https://github.com/matrix-org/synapse/issues/16143)) +* Bump isort from 5.11.5 to 5.12.0. ([\#16108](https://github.com/matrix-org/synapse/issues/16108)) +* Bump log from 0.4.19 to 0.4.20. ([\#16109](https://github.com/matrix-org/synapse/issues/16109)) +* Bump pygithub from 1.59.0 to 1.59.1. ([\#16144](https://github.com/matrix-org/synapse/issues/16144)) +* Bump sentry-sdk from 1.28.1 to 1.29.2. ([\#16142](https://github.com/matrix-org/synapse/issues/16142)) +* Bump serde from 1.0.183 to 1.0.184. ([\#16139](https://github.com/matrix-org/synapse/issues/16139)) +* Bump txredisapi from 1.4.9 to 1.4.10. ([\#16107](https://github.com/matrix-org/synapse/issues/16107)) +* Bump types-bleach from 6.0.0.3 to 6.0.0.4. ([\#16106](https://github.com/matrix-org/synapse/issues/16106)) +* Bump types-pillow from 10.0.0.1 to 10.0.0.2. ([\#16105](https://github.com/matrix-org/synapse/issues/16105)) +* Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2. ([\#16146](https://github.com/matrix-org/synapse/issues/16146)) + +# Synapse 1.90.0 (2023-08-15) + +No significant changes since 1.90.0rc1. + + +# Synapse 1.90.0rc1 (2023-08-08) + +### Features + +- Scope transaction IDs to devices (implement [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970)). ([\#15629](https://github.com/matrix-org/synapse/issues/15629)) +- Remove old rows from the `cache_invalidation_stream_by_instance` table automatically (this table is unused in SQLite). ([\#15868](https://github.com/matrix-org/synapse/issues/15868)) + +### Bugfixes + +- Fix a long-standing bug where purging history and paginating simultaneously could lead to database corruption when using workers. ([\#15791](https://github.com/matrix-org/synapse/issues/15791)) +- Fix a long-standing bug where profile endpoint returned a 404 when the user's display name was empty. ([\#16012](https://github.com/matrix-org/synapse/issues/16012)) +- Fix a long-standing bug where the `synapse_port_db` failed to configure sequences for application services and partial stated rooms. ([\#16043](https://github.com/matrix-org/synapse/issues/16043)) +- Fix long-standing bug with deletion in dehydrated devices v2. ([\#16046](https://github.com/matrix-org/synapse/issues/16046)) + +### Updates to the Docker image + +- Add `org.opencontainers.image.version` labels to Docker containers [published by Matrix.org](https://hub.docker.com/r/matrixdotorg/synapse). Contributed by Mo Balaa. ([\#15972](https://github.com/matrix-org/synapse/issues/15972), [\#16009](https://github.com/matrix-org/synapse/issues/16009)) + +### Improved Documentation + +- Add a internal documentation page describing the ["streams" used within Synapse](https://matrix-org.github.io/synapse/v1.90/development/synapse_architecture/streams.html). ([\#16015](https://github.com/matrix-org/synapse/issues/16015)) +- Clarify comment on the keys/upload over replication enpoint. ([\#16016](https://github.com/matrix-org/synapse/issues/16016)) +- Do not expose Admin API in caddy reverse proxy example. Contributed by @NilsIrl. ([\#16027](https://github.com/matrix-org/synapse/issues/16027)) + +### Deprecations and Removals + +- Remove support for legacy application service paths. ([\#15964](https://github.com/matrix-org/synapse/issues/15964)) +- Move support for application service query parameter authorization behind a configuration option. ([\#16017](https://github.com/matrix-org/synapse/issues/16017)) + +### Internal Changes + +- Update SQL queries to inline boolean parameters as supported in SQLite 3.27. ([\#15525](https://github.com/matrix-org/synapse/issues/15525)) +- Allow for the configuration of the backoff algorithm for federation destinations. ([\#15754](https://github.com/matrix-org/synapse/issues/15754)) +- Allow modules to check whether the current worker is configured to run background tasks. ([\#15991](https://github.com/matrix-org/synapse/issues/15991)) +- Update support for [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958) to match the latest revision of the MSC. ([\#15992](https://github.com/matrix-org/synapse/issues/15992)) +- Allow modules to schedule delayed background calls. ([\#15993](https://github.com/matrix-org/synapse/issues/15993)) +- Properly overwrite the `redacts` content-property for forwards-compatibility with room versions 1 through 10. ([\#16013](https://github.com/matrix-org/synapse/issues/16013)) +- Fix building the nix development environment on MacOS systems. ([\#16019](https://github.com/matrix-org/synapse/issues/16019)) +- Remove leading and trailing spaces when setting a display name. ([\#16031](https://github.com/matrix-org/synapse/issues/16031)) +- Combine duplicated code. ([\#16023](https://github.com/matrix-org/synapse/issues/16023)) +- Collect additional metrics from `ResponseCache` for eviction. ([\#16028](https://github.com/matrix-org/synapse/issues/16028)) +- Fix endpoint improperly declaring support for MSC3814. ([\#16068](https://github.com/matrix-org/synapse/issues/16068)) +- Drop backwards compat hack for event serialization. ([\#16069](https://github.com/matrix-org/synapse/issues/16069)) + +### Updates to locked dependencies + +* Update PyYAML to 6.0.1. ([\#16011](https://github.com/matrix-org/synapse/issues/16011)) +* Bump cryptography from 41.0.2 to 41.0.3. ([\#16048](https://github.com/matrix-org/synapse/issues/16048)) +* Bump furo from 2023.5.20 to 2023.7.26. ([\#16077](https://github.com/matrix-org/synapse/issues/16077)) +* Bump immutabledict from 2.2.4 to 3.0.0. ([\#16034](https://github.com/matrix-org/synapse/issues/16034)) +* Update certifi to 2023.7.22 and pygments to 2.15.1. ([\#16044](https://github.com/matrix-org/synapse/issues/16044)) +* Bump jsonschema from 4.18.3 to 4.19.0. ([\#16081](https://github.com/matrix-org/synapse/issues/16081)) +* Bump phonenumbers from 8.13.14 to 8.13.18. ([\#16076](https://github.com/matrix-org/synapse/issues/16076)) +* Bump regex from 1.9.1 to 1.9.3. ([\#16073](https://github.com/matrix-org/synapse/issues/16073)) +* Bump serde from 1.0.171 to 1.0.175. ([\#15982](https://github.com/matrix-org/synapse/issues/15982)) +* Bump serde from 1.0.175 to 1.0.179. ([\#16033](https://github.com/matrix-org/synapse/issues/16033)) +* Bump serde from 1.0.179 to 1.0.183. ([\#16074](https://github.com/matrix-org/synapse/issues/16074)) +* Bump serde_json from 1.0.103 to 1.0.104. ([\#16032](https://github.com/matrix-org/synapse/issues/16032)) +* Bump service-identity from 21.1.0 to 23.1.0. ([\#16038](https://github.com/matrix-org/synapse/issues/16038)) +* Bump types-commonmark from 0.9.2.3 to 0.9.2.4. ([\#16037](https://github.com/matrix-org/synapse/issues/16037)) +* Bump types-jsonschema from 4.17.0.8 to 4.17.0.10. ([\#16036](https://github.com/matrix-org/synapse/issues/16036)) +* Bump types-netaddr from 0.8.0.8 to 0.8.0.9. ([\#16035](https://github.com/matrix-org/synapse/issues/16035)) +* Bump types-opentracing from 2.4.10.5 to 2.4.10.6. ([\#16078](https://github.com/matrix-org/synapse/issues/16078)) +* Bump types-setuptools from 68.0.0.0 to 68.0.0.3. ([\#16079](https://github.com/matrix-org/synapse/issues/16079)) + +# Synapse 1.89.0 (2023-08-01) + +No significant changes since 1.89.0rc1. + + +# Synapse 1.89.0rc1 (2023-07-25) + +### Features + +- Add Unix Socket support for HTTP Replication Listeners. [Document and provide usage instructions](https://matrix-org.github.io/synapse/v1.89/usage/configuration/config_documentation.html#listeners) for utilizing Unix sockets in Synapse. Contributed by Jason Little. ([\#15708](https://github.com/matrix-org/synapse/issues/15708), [\#15924](https://github.com/matrix-org/synapse/issues/15924)) +- Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009). ([\#15911](https://github.com/matrix-org/synapse/issues/15911)) +- Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820). ([\#15912](https://github.com/matrix-org/synapse/issues/15912)) +- Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`. ([\#15913](https://github.com/matrix-org/synapse/issues/15913), [\#15969](https://github.com/matrix-org/synapse/issues/15969)) +- Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly, H-Shay and poljar. ([\#15929](https://github.com/matrix-org/synapse/issues/15929)) + +### Bugfixes + +- Fix a long-standing bug where remote invites weren't correctly pushed. ([\#15820](https://github.com/matrix-org/synapse/issues/15820)) +- Fix background schema updates failing over a large upgrade gap. ([\#15887](https://github.com/matrix-org/synapse/issues/15887)) +- Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting. ([\#15925](https://github.com/matrix-org/synapse/issues/15925)) +- Fixed deploy annotations in the provided Grafana dashboard config, so that it shows for any homeserver and not just matrix.org. Contributed by @wrjlewis. ([\#15957](https://github.com/matrix-org/synapse/issues/15957)) +- Ensure a long state res does not starve CPU by occasionally yielding to the reactor. ([\#15960](https://github.com/matrix-org/synapse/issues/15960)) +- Properly handle redactions of creation events. ([\#15973](https://github.com/matrix-org/synapse/issues/15973)) +- Fix a bug where resyncing stale device lists could block responding to federation transactions, and thus delay receiving new data from the remote server. ([\#15975](https://github.com/matrix-org/synapse/issues/15975)) + +### Improved Documentation + +- Better clarify how to run a worker instance (pass both configs). ([\#15921](https://github.com/matrix-org/synapse/issues/15921)) +- Improve [the documentation](https://matrix-org.github.io/synapse/v1.89/admin_api/user_admin_api.html#login-as-a-user) for the login as a user admin API. ([\#15938](https://github.com/matrix-org/synapse/issues/15938)) +- Fix broken Arch Linux package link. Contributed by @SnipeXandrej. ([\#15981](https://github.com/matrix-org/synapse/issues/15981)) + +### Deprecations and Removals + +- Remove support for calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15928](https://github.com/matrix-org/synapse/issues/15928)) + +### Internal Changes + +- Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it. ([\#15884](https://github.com/matrix-org/synapse/issues/15884)) +- Document which Python version runs on a given Linux distribution so we can more easily clean up later. ([\#15909](https://github.com/matrix-org/synapse/issues/15909)) +- Add details to warning in log when we fail to fetch an alias. ([\#15922](https://github.com/matrix-org/synapse/issues/15922)) +- Remove unneeded `__init__`. ([\#15926](https://github.com/matrix-org/synapse/issues/15926)) +- Fix bug with read/write lock implementation. This is currently unused so has no observable effects. ([\#15933](https://github.com/matrix-org/synapse/issues/15933), [\#15958](https://github.com/matrix-org/synapse/issues/15958)) +- Unbreak the nix development environment by pinning the Rust version to 1.70.0. ([\#15940](https://github.com/matrix-org/synapse/issues/15940)) +- Update presence metrics to differentiate remote vs local users. ([\#15952](https://github.com/matrix-org/synapse/issues/15952)) +- Stop reading from column `user_id` of table `profiles`. ([\#15955](https://github.com/matrix-org/synapse/issues/15955)) +- Build packages for Debian Trixie. ([\#15961](https://github.com/matrix-org/synapse/issues/15961)) +- Reduce the amount of state we pull out. ([\#15968](https://github.com/matrix-org/synapse/issues/15968)) +- Speed up updating state in large rooms. ([\#15971](https://github.com/matrix-org/synapse/issues/15971)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.71 to 1.0.72. ([\#15949](https://github.com/matrix-org/synapse/issues/15949)) +* Bump click from 8.1.3 to 8.1.6. ([\#15984](https://github.com/matrix-org/synapse/issues/15984)) +* Bump cryptography from 41.0.1 to 41.0.2. ([\#15943](https://github.com/matrix-org/synapse/issues/15943)) +* Bump jsonschema from 4.17.3 to 4.18.3. ([\#15948](https://github.com/matrix-org/synapse/issues/15948)) +* Bump pillow from 9.4.0 to 10.0.0. ([\#15986](https://github.com/matrix-org/synapse/issues/15986)) +* Bump prometheus-client from 0.17.0 to 0.17.1. ([\#15945](https://github.com/matrix-org/synapse/issues/15945)) +* Bump pydantic from 1.10.10 to 1.10.11. ([\#15946](https://github.com/matrix-org/synapse/issues/15946)) +* Bump pygithub from 1.58.2 to 1.59.0. ([\#15834](https://github.com/matrix-org/synapse/issues/15834)) +* Bump pyo3-log from 0.8.2 to 0.8.3. ([\#15951](https://github.com/matrix-org/synapse/issues/15951)) +* Bump sentry-sdk from 1.26.0 to 1.28.1. ([\#15985](https://github.com/matrix-org/synapse/issues/15985)) +* Bump serde_json from 1.0.100 to 1.0.103. ([\#15950](https://github.com/matrix-org/synapse/issues/15950)) +* Bump types-pillow from 9.5.0.4 to 10.0.0.1. ([\#15932](https://github.com/matrix-org/synapse/issues/15932)) +* Bump types-requests from 2.31.0.1 to 2.31.0.2. ([\#15983](https://github.com/matrix-org/synapse/issues/15983)) +* Bump typing-extensions from 4.5.0 to 4.7.1. ([\#15947](https://github.com/matrix-org/synapse/issues/15947)) + +# Synapse 1.88.0 (2023-07-18) + +This release + - raises the minimum supported version of Python to 3.8, as Python 3.7 is now [end-of-life](https://devguide.python.org/versions/), and + - removes deprecated config options related to worker deployment. + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#upgrading-to-v1880) for more information. + + +### Bugfixes + +- Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters`", which was introduced in Synapse 1.88.0rc1. ([\#15953](https://github.com/matrix-org/synapse/issues/15953)) + + +# Synapse 1.88.0rc1 (2023-07-11) + +### Features + +- Add `not_user_type` param to the [list accounts admin API](https://matrix-org.github.io/synapse/v1.88/admin_api/user_admin_api.html#list-accounts). ([\#15844](https://github.com/matrix-org/synapse/issues/15844)) + +### Bugfixes + +- Pin `pydantic` to `^=1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. + Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) +- Correctly resize thumbnails with pillow version >=10. ([\#15876](https://github.com/matrix-org/synapse/issues/15876)) + +### Improved Documentation + +- Fixed header levels on the [Admin API "Users"](https://matrix-org.github.io/synapse/v1.87/admin_api/user_admin_api.html) documentation page. Contributed by @sumnerevans at @beeper. ([\#15852](https://github.com/matrix-org/synapse/issues/15852)) +- Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options. ([\#15872](https://github.com/matrix-org/synapse/issues/15872)) + +### Deprecations and Removals + +- **Remove deprecated `worker_replication_host`, `worker_replication_http_port` and `worker_replication_http_tls` configuration options.** See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.88/docs/upgrade.md#removal-of-worker_replication_-settings) for more details. ([\#15860](https://github.com/matrix-org/synapse/issues/15860)) +- Remove support for Python 3.7 and hence for Debian Buster. ([\#15851](https://github.com/matrix-org/synapse/issues/15851), [\#15892](https://github.com/matrix-org/synapse/issues/15892), [\#15893](https://github.com/matrix-org/synapse/issues/15893), [\#15917](https://github.com/matrix-org/synapse/pull/15917)) + +### Internal Changes + +- Add foreign key constraint to `event_forward_extremities`. ([\#15751](https://github.com/matrix-org/synapse/issues/15751), [\#15907](https://github.com/matrix-org/synapse/issues/15907)) +- Add read/write style cross-worker locks. ([\#15782](https://github.com/matrix-org/synapse/issues/15782)) +- Stop writing to column `user_id` of tables `profiles` and `user_filters`. ([\#15787](https://github.com/matrix-org/synapse/issues/15787)) +- Use lower isolation level when cleaning old presence stream data to avoid serialization errors. ([\#15826](https://github.com/matrix-org/synapse/issues/15826)) +- Add tracing to media `/upload` code paths. ([\#15850](https://github.com/matrix-org/synapse/issues/15850), [\#15888](https://github.com/matrix-org/synapse/issues/15888)) +- Add a timeout that aborts any Postgres statement taking more than 1 hour. ([\#15853](https://github.com/matrix-org/synapse/issues/15853)) +- Fix the `devenv up` configuration which was ignoring the config overrides. ([\#15854](https://github.com/matrix-org/synapse/issues/15854)) +- Optimised cleanup of old entries in `device_lists_stream`. ([\#15861](https://github.com/matrix-org/synapse/issues/15861)) +- Update the Matrix clients link in the _It works! Synapse is running_ landing page. ([\#15874](https://github.com/matrix-org/synapse/issues/15874)) +- Fix building Synapse with the nightly Rust compiler. ([\#15906](https://github.com/matrix-org/synapse/issues/15906)) +- Add `Server` to Access-Control-Expose-Headers header. ([\#15908](https://github.com/matrix-org/synapse/issues/15908)) + +### Updates to locked dependencies + +* Bump authlib from 1.2.0 to 1.2.1. ([\#15864](https://github.com/matrix-org/synapse/issues/15864)) +* Bump importlib-metadata from 6.6.0 to 6.7.0. ([\#15865](https://github.com/matrix-org/synapse/issues/15865)) +* Bump lxml from 4.9.2 to 4.9.3. ([\#15897](https://github.com/matrix-org/synapse/issues/15897)) +* Bump regex from 1.8.4 to 1.9.1. ([\#15902](https://github.com/matrix-org/synapse/issues/15902)) +* Bump ruff from 0.0.275 to 0.0.277. ([\#15900](https://github.com/matrix-org/synapse/issues/15900)) +* Bump sentry-sdk from 1.25.1 to 1.26.0. ([\#15867](https://github.com/matrix-org/synapse/issues/15867)) +* Bump serde_json from 1.0.99 to 1.0.100. ([\#15901](https://github.com/matrix-org/synapse/issues/15901)) +* Bump types-pyopenssl from 23.2.0.0 to 23.2.0.1. ([\#15866](https://github.com/matrix-org/synapse/issues/15866)) + +# Synapse 1.87.0 (2023-07-04) + +Please note that this will be the last release of Synapse that is compatible with +Python 3.7 and earlier. +This is due to Python 3.7 now having reached End of Life; see our [deprecation policy](https://matrix-org.github.io/synapse/v1.87/deprecation_policy.html) +for more details. + +### Bugfixes + +- Pin `pydantic` to `^1.7.4` to avoid backwards-incompatible API changes from the 2.0.0 release. + Resolves https://github.com/matrix-org/synapse/issues/15858. + Contributed by @PaarthShah. ([\#15862](https://github.com/matrix-org/synapse/issues/15862)) + +### Internal Changes + +- Split out 2022 changes from the changelog so the rendered version in GitHub doesn't timeout as much. ([\#15846](https://github.com/matrix-org/synapse/issues/15846)) + + +# Synapse 1.87.0rc1 (2023-06-27) + +### Features + +- Improve `/messages` response time by avoiding backfill when we already have messages to return. ([\#15737](https://github.com/matrix-org/synapse/issues/15737)) +- Add spam checker module API for logins. ([\#15838](https://github.com/matrix-org/synapse/issues/15838)) + +### Bugfixes + +- Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou. ([\#15680](https://github.com/matrix-org/synapse/issues/15680)) +- Avoid invalidating a cache that was just prefilled. ([\#15758](https://github.com/matrix-org/synapse/issues/15758)) +- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15770](https://github.com/matrix-org/synapse/issues/15770)) +- Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper. ([\#15776](https://github.com/matrix-org/synapse/issues/15776)) +- Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`. ([\#15781](https://github.com/matrix-org/synapse/issues/15781)) +- Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend. ([\#15788](https://github.com/matrix-org/synapse/issues/15788)) +- Fix Sytest environmental variable evaluation in CI. ([\#15804](https://github.com/matrix-org/synapse/issues/15804)) +- Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly. ([\#15815](https://github.com/matrix-org/synapse/issues/15815)) +- Fix sqlite `user_filters` upgrade introduced in v1.86.0. ([\#15817](https://github.com/matrix-org/synapse/issues/15817)) + +### Improved Documentation + +- Document `looping_call()` functionality that will wait for the given function to finish before scheduling another. ([\#15772](https://github.com/matrix-org/synapse/issues/15772)) +- Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html). ([\#15805](https://github.com/matrix-org/synapse/issues/15805)) +- Fix typo in MSC number in faster remote room join architecture doc. ([\#15812](https://github.com/matrix-org/synapse/issues/15812)) + +### Deprecations and Removals + +- Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms. ([\#15748](https://github.com/matrix-org/synapse/issues/15748)) + +### Internal Changes + +- Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`. ([\#15233](https://github.com/matrix-org/synapse/issues/15233)) +- Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying. ([\#15743](https://github.com/matrix-org/synapse/issues/15743)) +- Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983). ([\#15755](https://github.com/matrix-org/synapse/issues/15755)) +- Allow for the configuration of max request retries and min/max retry delays in the matrix federation client. ([\#15783](https://github.com/matrix-org/synapse/issues/15783)) +- Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic. ([\#15806](https://github.com/matrix-org/synapse/issues/15806)) +- Fix harmless exceptions being printed when running the port DB script. ([\#15814](https://github.com/matrix-org/synapse/issues/15814)) + +### Updates to locked dependencies + +* Bump attrs from 22.2.0 to 23.1.0. ([\#15801](https://github.com/matrix-org/synapse/issues/15801)) +* Bump cryptography from 40.0.2 to 41.0.1. ([\#15800](https://github.com/matrix-org/synapse/issues/15800)) +* Bump ijson from 3.2.0.post0 to 3.2.1. ([\#15802](https://github.com/matrix-org/synapse/issues/15802)) +* Bump phonenumbers from 8.13.13 to 8.13.14. ([\#15798](https://github.com/matrix-org/synapse/issues/15798)) +* Bump ruff from 0.0.265 to 0.0.272. ([\#15799](https://github.com/matrix-org/synapse/issues/15799)) +* Bump ruff from 0.0.272 to 0.0.275. ([\#15833](https://github.com/matrix-org/synapse/issues/15833)) +* Bump serde_json from 1.0.96 to 1.0.97. ([\#15797](https://github.com/matrix-org/synapse/issues/15797)) +* Bump serde_json from 1.0.97 to 1.0.99. ([\#15832](https://github.com/matrix-org/synapse/issues/15832)) +* Bump towncrier from 22.12.0 to 23.6.0. ([\#15831](https://github.com/matrix-org/synapse/issues/15831)) +* Bump types-opentracing from 2.4.10.4 to 2.4.10.5. ([\#15830](https://github.com/matrix-org/synapse/issues/15830)) +* Bump types-setuptools from 67.8.0.0 to 68.0.0.0. ([\#15835](https://github.com/matrix-org/synapse/issues/15835)) + +Synapse 1.86.0 (2023-06-20) +=========================== + +No significant changes since 1.86.0rc2. + + +Synapse 1.86.0rc2 (2023-06-14) +============================== + +Bugfixes +-------- + +- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774)) + + +Synapse 1.86.0rc1 (2023-06-13) +============================== + +This version was tagged but never released. + +Features +-------- + +- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388)) +- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450)) +- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520)) +- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582)) +- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674)) +- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705)) +- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710)) +- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740)) + + +Bugfixes +-------- + +- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609)) +- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695)) + + +Improved Documentation +---------------------- + +- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732)) + + +Internal Changes +---------------- + +- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213)) +- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649)) +- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650)) +- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675)) +- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689)) +- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690)) +- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697)) +- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709)) +- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721)) +- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723)) +- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725)) +- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726)) +- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729)) +- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731)) +- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733)) +- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752)) +- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768)) + +### Updates to locked dependencies + +* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711)) +* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712)) +* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761)) +* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763)) +* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713)) +* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762)) +* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759)) +* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765)) +* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769)) +* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714)) +* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764)) +* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760)) +* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716)) +* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766)) +* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715)) + +Synapse 1.85.2 (2023-06-08) +=========================== + +Bugfixes +-------- + +- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) + + +Synapse 1.85.1 (2023-06-07) +=========================== + +Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0. + +Bugfixes +-------- + +- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739)) + + +Synapse 1.85.0 (2023-06-06) +=========================== + +No significant changes since 1.85.0rc2. + + +## Security advisory + +The following issues are fixed in 1.85.0 (and RCs). + +- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity + + It may be possible for a deactivated user to login when using uncommon configurations. + +- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity + + A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs). + +See the advisories for more details. If you have any questions, email security@matrix.org. + + +Synapse 1.85.0rc2 (2023-06-01) +============================== + +Bugfixes +-------- + +- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693)) + + +Deprecations and Removals +------------------------- + +- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703)) + + +Internal Changes +---------------- + +- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700)) + + +Synapse 1.85.0rc1 (2023-05-30) +============================== + +Features +-------- + +- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585)) +- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611)) +- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644)) + + +Bugfixes +-------- + +- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464)) +- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601)) +- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607)) +- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614)) +- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624)) +- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634)) + + +Improved Documentation +---------------------- + +- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613)) +- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648)) +- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668)) + + +Deprecations and Removals +------------------------- + +- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428)) + + +Internal Changes +---------------- + +- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481)) +- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537)) +- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578)) +- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597)) +- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602)) +- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604)) +- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620)) +- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647)) +- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633)) +- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615)) +- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621)) +- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625)) +- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626)) +- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636)) +- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630)) +- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646)) +- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659)) +- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663)) +- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665)) +- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678)) + +### Updates to locked dependencies + +* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642)) +* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681)) +* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682)) +* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685)) +* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643)) +* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651)) +* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641)) +* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686)) +* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640)) +* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683)) +* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684)) +* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639)) + +Synapse 1.84.1 (2023-05-26) +=========================== + +This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers. +If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release. + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672)) + + +Internal Changes +---------------- + +- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673)) + + +Synapse 1.84.0 (2023-05-23) +=========================== + +The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599)) + + +Synapse 1.84.0rc1 (2023-05-16) +============================== + +Features +-------- + +- Add an option to prevent media downloads from configured domains. ([\#15197](https://github.com/matrix-org/synapse/issues/15197)) +- Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them. ([\#15224](https://github.com/matrix-org/synapse/issues/15224)) +- Add redis TLS configuration options. ([\#15312](https://github.com/matrix-org/synapse/issues/15312)) +- Add a config option to delay push notifications by a random amount, to discourage time-based profiling. ([\#15516](https://github.com/matrix-org/synapse/issues/15516)) +- Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15528](https://github.com/matrix-org/synapse/issues/15528)) +- Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs. ([\#15536](https://github.com/matrix-org/synapse/issues/15536)) +- Advertise support for Matrix 1.6 on `/_matrix/client/versions`. ([\#15559](https://github.com/matrix-org/synapse/issues/15559)) +- Print full error and stack-trace of any exception that occurs during startup/initialization. ([\#15569](https://github.com/matrix-org/synapse/issues/15569)) + + +Bugfixes +-------- + +- Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu. ([\#15523](https://github.com/matrix-org/synapse/issues/15523)) +- Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data. ([\#15554](https://github.com/matrix-org/synapse/issues/15554), [\#15555](https://github.com/matrix-org/synapse/issues/15555)) +- Fix a long-standing bug where an invalid membership event could cause an internal server error. ([\#15564](https://github.com/matrix-org/synapse/issues/15564)) +- Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571)) + + +Deprecations and Removals +------------------------- + +- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491)) + + +Updates to the Docker image +--------------------------- + +- Add pkg-config package to Stage 0 to be able to build Dockerfile on ppc64le architecture. ([\#15567](https://github.com/matrix-org/synapse/issues/15567)) + + +Improved Documentation +---------------------- + +- Clarify documentation of the "Create or modify account" Admin API. ([\#15544](https://github.com/matrix-org/synapse/issues/15544)) +- Fix path to the `statistics/database/rooms` admin API in documentation. ([\#15560](https://github.com/matrix-org/synapse/issues/15560)) +- Update and improve Mastodon Single Sign-On documentation. ([\#15587](https://github.com/matrix-org/synapse/issues/15587)) + + +Internal Changes +---------------- + +- Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025)) +- Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470)) +- Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509)) +- Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522)) +- Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527)) +- Speed up rebuilding of the user directory for local users. ([\#15529](https://github.com/matrix-org/synapse/issues/15529)) +- Speed up deleting of old rows in `event_push_actions`. ([\#15531](https://github.com/matrix-org/synapse/issues/15531)) +- Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment. ([\#15532](https://github.com/matrix-org/synapse/issues/15532), [\#15533](https://github.com/matrix-org/synapse/issues/15533), [\#15545](https://github.com/matrix-org/synapse/issues/15545)) +- Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules. ([\#15534](https://github.com/matrix-org/synapse/issues/15534)) +- Move various module API callback registration methods to a dedicated class. ([\#15535](https://github.com/matrix-org/synapse/issues/15535)) +- Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984). ([\#15539](https://github.com/matrix-org/synapse/issues/15539)) +- Factor out an `is_mine_server_name` method. ([\#15542](https://github.com/matrix-org/synapse/issues/15542)) +- Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`. ([\#15543](https://github.com/matrix-org/synapse/issues/15543)) +- Bump serde from 1.0.160 to 1.0.162. ([\#15548](https://github.com/matrix-org/synapse/issues/15548)) +- Bump types-setuptools from 67.6.0.5 to 67.7.0.1. ([\#15549](https://github.com/matrix-org/synapse/issues/15549)) +- Bump sentry-sdk from 1.19.1 to 1.22.1. ([\#15550](https://github.com/matrix-org/synapse/issues/15550)) +- Bump ruff from 0.0.259 to 0.0.265. ([\#15551](https://github.com/matrix-org/synapse/issues/15551)) +- Bump hiredis from 2.2.2 to 2.2.3. ([\#15552](https://github.com/matrix-org/synapse/issues/15552)) +- Bump types-requests from 2.29.0.0 to 2.30.0.0. ([\#15553](https://github.com/matrix-org/synapse/issues/15553)) +- Add `org.matrix.msc3981` info to `/_matrix/client/versions`. ([\#15558](https://github.com/matrix-org/synapse/issues/15558)) +- Declare unstable support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) under `/_matrix/client/versions` if the experimental implementation is enabled. ([\#15562](https://github.com/matrix-org/synapse/issues/15562)) +- Implement [MSC3821](https://github.com/matrix-org/matrix-spec-proposals/pull/3821) to update the redaction rules. ([\#15563](https://github.com/matrix-org/synapse/issues/15563)) +- Implement updated redaction rules from [MSC3389](https://github.com/matrix-org/matrix-spec-proposals/pull/3389). ([\#15565](https://github.com/matrix-org/synapse/issues/15565)) +- Allow `pip install` to use setuptools_rust 1.6.0 when building Synapse. ([\#15570](https://github.com/matrix-org/synapse/issues/15570)) +- Deal with upcoming Github Actions deprecations. ([\#15576](https://github.com/matrix-org/synapse/issues/15576)) +- Export `run_as_background_process` from the module API. ([\#15577](https://github.com/matrix-org/synapse/issues/15577)) +- Update build system requirements to allow building with poetry-core==1.6.0. ([\#15588](https://github.com/matrix-org/synapse/issues/15588)) +- Bump serde from 1.0.162 to 1.0.163. ([\#15589](https://github.com/matrix-org/synapse/issues/15589)) +- Bump phonenumbers from 8.13.7 to 8.13.11. ([\#15590](https://github.com/matrix-org/synapse/issues/15590)) +- Bump types-psycopg2 from 2.9.21.9 to 2.9.21.10. ([\#15591](https://github.com/matrix-org/synapse/issues/15591)) +- Bump types-commonmark from 0.9.2.2 to 0.9.2.3. ([\#15592](https://github.com/matrix-org/synapse/issues/15592)) +- Bump types-setuptools from 67.7.0.1 to 67.7.0.2. ([\#15594](https://github.com/matrix-org/synapse/issues/15594)) + + +Synapse 1.83.0 (2023-05-09) +=========================== + +No significant changes since 1.83.0rc1. + + +Synapse 1.83.0rc1 (2023-05-02) +============================== + +Features +-------- + +- Experimental support to recursively provide relations per [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981). ([\#15315](https://github.com/matrix-org/synapse/issues/15315)) +- Experimental support for [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970): Scope transaction IDs to devices. ([\#15318](https://github.com/matrix-org/synapse/issues/15318)) +- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/experimental_features.html) to support per-user feature flags. ([\#15344](https://github.com/matrix-org/synapse/issues/15344)) +- Add a module API to send an HTTP push notification. ([\#15387](https://github.com/matrix-org/synapse/issues/15387)) +- Add an [admin API endpoint](https://matrix-org.github.io/synapse/v1.83/admin_api/statistics.html#get-largest-rooms-by-size-in-database) to query the largest rooms by disk space used in the database. ([\#15482](https://github.com/matrix-org/synapse/issues/15482)) + + +Bugfixes +-------- + +- Disable push rule evaluation for rooms excluded from sync. ([\#15361](https://github.com/matrix-org/synapse/issues/15361)) +- Fix a long-standing bug where cached server key results which were directly fetched would not be properly re-used. ([\#15417](https://github.com/matrix-org/synapse/issues/15417)) +- Fix a bug introduced in Synapse 1.73.0 where some experimental push rules were returned by default. ([\#15494](https://github.com/matrix-org/synapse/issues/15494)) + + +Improved Documentation +---------------------- + +- Add Nginx loadbalancing example with sticky mxid for workers. ([\#15411](https://github.com/matrix-org/synapse/issues/15411)) +- Update outdated development docs that mention restrictions in versions of SQLite that we no longer support. ([\#15498](https://github.com/matrix-org/synapse/issues/15498)) + + +Internal Changes +---------------- + +- Speedup tests by caching HomeServerConfig instances. ([\#15284](https://github.com/matrix-org/synapse/issues/15284)) +- Add denormalised event stream ordering column to membership state tables for future use. Contributed by Nick @ Beeper (@fizzadar). ([\#15356](https://github.com/matrix-org/synapse/issues/15356)) +- Always use multi-user device resync replication endpoints. ([\#15418](https://github.com/matrix-org/synapse/issues/15418)) +- Add column `full_user_id` to tables `profiles` and `user_filters`. ([\#15458](https://github.com/matrix-org/synapse/issues/15458)) +- Update support for [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) to allow always returning fallback-keys in a `/keys/claim` request. ([\#15462](https://github.com/matrix-org/synapse/issues/15462)) +- Improve type hints. ([\#15465](https://github.com/matrix-org/synapse/issues/15465), [\#15496](https://github.com/matrix-org/synapse/issues/15496), [\#15497](https://github.com/matrix-org/synapse/issues/15497)) +- Support claiming more than one OTK at a time. ([\#15468](https://github.com/matrix-org/synapse/issues/15468)) +- Bump types-pyyaml from 6.0.12.8 to 6.0.12.9. ([\#15471](https://github.com/matrix-org/synapse/issues/15471)) +- Bump pyasn1-modules from 0.2.8 to 0.3.0. ([\#15473](https://github.com/matrix-org/synapse/issues/15473)) +- Bump cryptography from 40.0.1 to 40.0.2. ([\#15474](https://github.com/matrix-org/synapse/issues/15474)) +- Bump types-netaddr from 0.8.0.7 to 0.8.0.8. ([\#15475](https://github.com/matrix-org/synapse/issues/15475)) +- Bump types-jsonschema from 4.17.0.6 to 4.17.0.7. ([\#15476](https://github.com/matrix-org/synapse/issues/15476)) +- Ask bug reporters to provide logs as text. ([\#15479](https://github.com/matrix-org/synapse/issues/15479)) +- Add a Nix flake for use as a development environment. ([\#15495](https://github.com/matrix-org/synapse/issues/15495)) +- Bump anyhow from 1.0.70 to 1.0.71. ([\#15507](https://github.com/matrix-org/synapse/issues/15507)) +- Bump types-pillow from 9.4.0.19 to 9.5.0.2. ([\#15508](https://github.com/matrix-org/synapse/issues/15508)) +- Bump packaging from 23.0 to 23.1. ([\#15510](https://github.com/matrix-org/synapse/issues/15510)) +- Bump types-requests from 2.28.11.16 to 2.29.0.0. ([\#15511](https://github.com/matrix-org/synapse/issues/15511)) +- Bump setuptools-rust from 1.5.2 to 1.6.0. ([\#15512](https://github.com/matrix-org/synapse/issues/15512)) +- Update the check_schema_delta script to account for when the schema version has been bumped locally. ([\#15466](https://github.com/matrix-org/synapse/issues/15466)) + + +Synapse 1.82.0 (2023-04-25) +=========================== + +No significant changes since 1.82.0rc1. + + +Synapse 1.82.0rc1 (2023-04-18) +============================== + +Features +-------- + +- Allow loading the `/directory/room/{roomAlias}` endpoint on workers. ([\#15333](https://github.com/matrix-org/synapse/issues/15333)) +- Add some validation to `instance_map` configuration loading. ([\#15431](https://github.com/matrix-org/synapse/issues/15431)) +- Allow loading the `/capabilities` endpoint on workers. ([\#15436](https://github.com/matrix-org/synapse/issues/15436)) + + +Bugfixes +-------- + +- Delete server-side backup keys when deactivating an account. ([\#15181](https://github.com/matrix-org/synapse/issues/15181)) +- Fix and document untold assumption that `on_logged_out` module hooks will be called before the deletion of pushers. ([\#15410](https://github.com/matrix-org/synapse/issues/15410)) +- Improve robustness when handling a perspective key response by deduplicating received server keys. ([\#15423](https://github.com/matrix-org/synapse/issues/15423)) +- Synapse now correctly fails to start if the config option `app_service_config_files` is not a list. ([\#15425](https://github.com/matrix-org/synapse/issues/15425)) +- Disable loading `RefreshTokenServlet` (`/_matrix/client/(r0|v3|unstable)/refresh`) on workers. ([\#15428](https://github.com/matrix-org/synapse/issues/15428)) + + +Improved Documentation +---------------------- + +- Note that the `delete_stale_devices_after` background job always runs on the main process. ([\#15452](https://github.com/matrix-org/synapse/issues/15452)) + + +Deprecations and Removals +------------------------- + +- Remove the broken, unspecced registration fallback. Note that the *login* fallback is unaffected by this change. ([\#15405](https://github.com/matrix-org/synapse/issues/15405)) + + +Internal Changes +---------------- + +- Bump black from 23.1.0 to 23.3.0. ([\#15372](https://github.com/matrix-org/synapse/issues/15372)) +- Bump pyopenssl from 23.1.0 to 23.1.1. ([\#15373](https://github.com/matrix-org/synapse/issues/15373)) +- Bump types-psycopg2 from 2.9.21.8 to 2.9.21.9. ([\#15374](https://github.com/matrix-org/synapse/issues/15374)) +- Bump types-netaddr from 0.8.0.6 to 0.8.0.7. ([\#15375](https://github.com/matrix-org/synapse/issues/15375)) +- Bump types-opentracing from 2.4.10.3 to 2.4.10.4. ([\#15376](https://github.com/matrix-org/synapse/issues/15376)) +- Bump dawidd6/action-download-artifact from 2.26.0 to 2.26.1. ([\#15404](https://github.com/matrix-org/synapse/issues/15404)) +- Bump parameterized from 0.8.1 to 0.9.0. ([\#15412](https://github.com/matrix-org/synapse/issues/15412)) +- Bump types-pillow from 9.4.0.17 to 9.4.0.19. ([\#15413](https://github.com/matrix-org/synapse/issues/15413)) +- Bump sentry-sdk from 1.17.0 to 1.19.1. ([\#15414](https://github.com/matrix-org/synapse/issues/15414)) +- Bump immutabledict from 2.2.3 to 2.2.4. ([\#15415](https://github.com/matrix-org/synapse/issues/15415)) +- Bump dawidd6/action-download-artifact from 2.26.1 to 2.27.0. ([\#15441](https://github.com/matrix-org/synapse/issues/15441)) +- Bump serde_json from 1.0.95 to 1.0.96. ([\#15442](https://github.com/matrix-org/synapse/issues/15442)) +- Bump serde from 1.0.159 to 1.0.160. ([\#15443](https://github.com/matrix-org/synapse/issues/15443)) +- Bump pillow from 9.4.0 to 9.5.0. ([\#15444](https://github.com/matrix-org/synapse/issues/15444)) +- Bump furo from 2023.3.23 to 2023.3.27. ([\#15445](https://github.com/matrix-org/synapse/issues/15445)) +- Bump types-pyopenssl from 23.1.0.0 to 23.1.0.2. ([\#15446](https://github.com/matrix-org/synapse/issues/15446)) +- Bump mypy from 1.0.0 to 1.0.1. ([\#15447](https://github.com/matrix-org/synapse/issues/15447)) +- Bump psycopg2 from 2.9.5 to 2.9.6. ([\#15448](https://github.com/matrix-org/synapse/issues/15448)) +- Improve DB performance of clearing out old data from `stream_ordering_to_exterm`. ([\#15382](https://github.com/matrix-org/synapse/issues/15382), [\#15429](https://github.com/matrix-org/synapse/issues/15429)) +- Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. ([\#15393](https://github.com/matrix-org/synapse/issues/15393)) +- Implement [MSC2175](https://github.com/matrix-org/matrix-doc/pull/2175) to stop adding `creator` to create events. ([\#15394](https://github.com/matrix-org/synapse/issues/15394)) +- Implement [MSC2174](https://github.com/matrix-org/matrix-spec-proposals/pull/2174) to move the `redacts` key to a `content` property. ([\#15395](https://github.com/matrix-org/synapse/issues/15395)) +- Trust dtonlay/rust-toolchain in CI. ([\#15406](https://github.com/matrix-org/synapse/issues/15406)) +- Explicitly install Synapse during typechecking in CI. ([\#15409](https://github.com/matrix-org/synapse/issues/15409)) +- Only load the SSO redirect servlet if SSO is enabled. ([\#15421](https://github.com/matrix-org/synapse/issues/15421)) +- Refactor `SimpleHttpClient` to pull out a base class. ([\#15427](https://github.com/matrix-org/synapse/issues/15427)) +- Improve type hints. ([\#15432](https://github.com/matrix-org/synapse/issues/15432)) +- Convert async to normal tests in `TestSSOHandler`. ([\#15433](https://github.com/matrix-org/synapse/issues/15433)) +- Speed up the user directory background update. ([\#15435](https://github.com/matrix-org/synapse/issues/15435)) +- Disable directory listing for static resources in `/_matrix/static/`. ([\#15438](https://github.com/matrix-org/synapse/issues/15438)) +- Move various module API callback registration methods to a dedicated class. ([\#15453](https://github.com/matrix-org/synapse/issues/15453)) + + +Synapse 1.81.0 (2023-04-11) +=========================== + +Synapse now attempts the versioned appservice paths before falling back to the +[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). +Usage of the legacy routes should be considered deprecated. + +Additionally, Synapse has supported sending the application service access token +via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) +since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` +query parameter. This is insecure and should be considered deprecated. + +A future version of Synapse (v1.88.0 or later) will remove support for legacy +application service routes and query parameter authorization. + + +No significant changes since 1.81.0rc2. + + +Synapse 1.81.0rc2 (2023-04-06) +============================== + +Bugfixes +-------- + +- Fix the `set_device_id_for_pushers_txn` background update crash. ([\#15391](https://github.com/matrix-org/synapse/issues/15391)) + + +Internal Changes +---------------- + +- Update CI to run complement under the latest stable go version. ([\#15403](https://github.com/matrix-org/synapse/issues/15403)) + + +Synapse 1.81.0rc1 (2023-04-04) +============================== + +Features +-------- + +- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978)) +- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) +- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) +- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) +- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) +- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) + + +Bugfixes +-------- + +- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295)) +- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297)) +- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) + to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) +- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) +- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) +- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) +- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) +- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) +- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) +- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352)) +- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383)) + + +Improved Documentation +---------------------- + +- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) +- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) +- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) + + +Internal Changes +---------------- + +- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113)) +- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336)) +- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280)) +- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285)) +- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) +- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) +- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) +- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) +- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) +- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) +- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) +- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325)) +- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326)) +- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327)) +- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328)) +- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329)) +- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330)) +- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) +- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) +- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) +- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) +- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) +- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) +- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371)) +- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385)) + + +Synapse 1.80.0 (2023-03-28) +=========================== + +No significant changes since 1.80.0rc2. + + +Synapse 1.80.0rc2 (2023-03-22) +============================== + +Bugfixes +-------- + +- Fix a bug in which the [`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid) endpoint would return the wrong error if the user did not have permission to view the event. This aligns Synapse's implementation with [MSC2249](https://github.com/matrix-org/matrix-spec-proposals/pull/2249). ([\#15298](https://github.com/matrix-org/synapse/issues/15298), [\#15300](https://github.com/matrix-org/synapse/issues/15300)) +- Fix a bug introduced in Synapse 1.75.0rc1 where the [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) + would fail to open the SQLite database. ([\#15301](https://github.com/matrix-org/synapse/issues/15301)) + + +Synapse 1.80.0rc1 (2023-03-21) +============================== + +Features +-------- + +- Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition. ([\#15187](https://github.com/matrix-org/synapse/issues/15187)) +- Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15249](https://github.com/matrix-org/synapse/issues/15249)) +- Allow loading `/register/available` endpoint on workers. ([\#15268](https://github.com/matrix-org/synapse/issues/15268)) +- Improve performance of creating and authenticating events. ([\#15195](https://github.com/matrix-org/synapse/issues/15195)) +- Add topic and name events to group of events that are batch persisted when creating a room. ([\#15229](https://github.com/matrix-org/synapse/issues/15229)) + + +Bugfixes +-------- + +- Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change. ([\#14755](https://github.com/matrix-org/synapse/issues/14755), [\#14756](https://github.com/matrix-org/synapse/issues/14756)) +- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules. ([\#15190](https://github.com/matrix-org/synapse/issues/15190)) +- Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged. ([\#15232](https://github.com/matrix-org/synapse/issues/15232)) +- Fix a long-standing error when sending message into deleted room. ([\#15235](https://github.com/matrix-org/synapse/issues/15235)) + + +Updates to the Docker image +--------------------------- + +- Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel. ([\#15239](https://github.com/matrix-org/synapse/issues/15239)) +- Mirror images to the GitHub Container Registry (`ghcr.io/matrix-org/synapse`). ([\#15281](https://github.com/matrix-org/synapse/issues/15281), [\#15282](https://github.com/matrix-org/synapse/issues/15282)) + + +Improved Documentation +---------------------- + +- Add a missing endpoint to the workers documentation. ([\#15223](https://github.com/matrix-org/synapse/issues/15223)) + + +Internal Changes +---------------- + +- Add additional functionality to declaring worker types when starting Complement in worker mode. ([\#14921](https://github.com/matrix-org/synapse/issues/14921)) +- Add `Synapse-Trace-Id` to `access-control-expose-headers` header. ([\#14974](https://github.com/matrix-org/synapse/issues/14974)) +- Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key. ([\#15200](https://github.com/matrix-org/synapse/issues/15200)) +- Improve log lines when purging rooms. ([\#15222](https://github.com/matrix-org/synapse/issues/15222)) +- Improve type hints. ([\#15230](https://github.com/matrix-org/synapse/issues/15230), [\#15231](https://github.com/matrix-org/synapse/issues/15231), [\#15238](https://github.com/matrix-org/synapse/issues/15238)) +- Move various module API callback registration methods to a dedicated class. ([\#15237](https://github.com/matrix-org/synapse/issues/15237)) +- Configure GitHub Actions for merge queues. ([\#15244](https://github.com/matrix-org/synapse/issues/15244)) +- Add schema comments about the `destinations` and `destination_rooms` tables. ([\#15247](https://github.com/matrix-org/synapse/issues/15247)) +- Skip processing of auto-join room behaviour if there are no auto-join rooms configured. ([\#15262](https://github.com/matrix-org/synapse/issues/15262)) +- Remove unused store method `_set_destination_retry_timings_emulated`. ([\#15266](https://github.com/matrix-org/synapse/issues/15266)) +- Reorganize URL preview code. ([\#15269](https://github.com/matrix-org/synapse/issues/15269)) +- Clean-up direct TCP replication code. ([\#15272](https://github.com/matrix-org/synapse/issues/15272), [\#15274](https://github.com/matrix-org/synapse/issues/15274)) +- Make `configure_workers_and_start` script used in Complement tests compatible with older versions of Python. ([\#15275](https://github.com/matrix-org/synapse/issues/15275)) +- Add a `/versions` flag for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15293](https://github.com/matrix-org/synapse/issues/15293)) +- Bump hiredis from 2.2.1 to 2.2.2. ([\#15252](https://github.com/matrix-org/synapse/issues/15252)) +- Bump serde from 1.0.152 to 1.0.155. ([\#15253](https://github.com/matrix-org/synapse/issues/15253)) +- Bump pysaml2 from 7.2.1 to 7.3.1. ([\#15254](https://github.com/matrix-org/synapse/issues/15254)) +- Bump msgpack from 1.0.4 to 1.0.5. ([\#15255](https://github.com/matrix-org/synapse/issues/15255)) +- Bump gitpython from 3.1.30 to 3.1.31. ([\#15256](https://github.com/matrix-org/synapse/issues/15256)) +- Bump cryptography from 39.0.1 to 39.0.2. ([\#15257](https://github.com/matrix-org/synapse/issues/15257)) +- Bump pydantic from 1.10.4 to 1.10.6. ([\#15286](https://github.com/matrix-org/synapse/issues/15286)) +- Bump serde from 1.0.155 to 1.0.157. ([\#15287](https://github.com/matrix-org/synapse/issues/15287)) +- Bump anyhow from 1.0.69 to 1.0.70. ([\#15288](https://github.com/matrix-org/synapse/issues/15288)) +- Bump txredisapi from 1.4.7 to 1.4.9. ([\#15289](https://github.com/matrix-org/synapse/issues/15289)) +- Bump pygithub from 1.57 to 1.58.1. ([\#15290](https://github.com/matrix-org/synapse/issues/15290)) +- Bump types-requests from 2.28.11.12 to 2.28.11.15. ([\#15291](https://github.com/matrix-org/synapse/issues/15291)) + + + +Synapse 1.79.0 (2023-03-14) +=========================== + +No significant changes since 1.79.0rc2. + + +Synapse 1.79.0rc2 (2023-03-13) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.79.0rc1 where attempting to register a `on_remove_user_third_party_identifier` module API callback would be a no-op. ([\#15227](https://github.com/matrix-org/synapse/issues/15227)) +- Fix a rare bug introduced in Synapse 1.73 where events could remain unsent to other homeservers after a faster-join to a room. ([\#15248](https://github.com/matrix-org/synapse/issues/15248)) + + +Internal Changes +---------------- + +- Refactor `filter_events_for_server`. ([\#15240](https://github.com/matrix-org/synapse/issues/15240)) + + +Synapse 1.79.0rc1 (2023-03-07) +============================== + +Features +-------- + +- Add two new Third Party Rules module API callbacks: [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier) and [`on_remove_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_remove_user_third_party_identifier). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) +- Experimental support for [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967) to not require UIA for setting up cross-signing on first use. ([\#15077](https://github.com/matrix-org/synapse/issues/15077)) +- Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#15107](https://github.com/matrix-org/synapse/issues/15107)) +- Add an [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) to delete a [specific event report](https://spec.matrix.org/v1.6/client-server-api/#reporting-content). ([\#15116](https://github.com/matrix-org/synapse/issues/15116)) +- Add support for knocking to workers. ([\#15133](https://github.com/matrix-org/synapse/issues/15133)) +- Allow use of the `/filter` Client-Server APIs on workers. ([\#15134](https://github.com/matrix-org/synapse/issues/15134)) +- Update support for [MSC2677](https://github.com/matrix-org/matrix-spec-proposals/pull/2677): remove support for server-side aggregation of reactions. ([\#15172](https://github.com/matrix-org/synapse/issues/15172)) +- Stabilise support for [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758): `event_property_is` push condition. ([\#15185](https://github.com/matrix-org/synapse/issues/15185)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.75 that caused experimental support for deleting account data to raise an internal server error while using an account data writer worker. ([\#14869](https://github.com/matrix-org/synapse/issues/14869)) +- Fix a long-standing bug where Synapse handled an unspecced field on push rules. ([\#15088](https://github.com/matrix-org/synapse/issues/15088)) +- Fix a long-standing bug where a URL preview would break if the discovered oEmbed failed to download. ([\#15092](https://github.com/matrix-org/synapse/issues/15092)) +- Fix a long-standing bug where an initial sync would not respond to changes to the list of ignored users if there was an initial sync cached. ([\#15163](https://github.com/matrix-org/synapse/issues/15163)) +- Add the `transaction_id` in the events included in many endpoints' responses. ([\#15174](https://github.com/matrix-org/synapse/issues/15174)) +- Fix a bug introduced in Synapse 1.78.0 where requests to claim dehydrated devices would fail with a `405` error. ([\#15180](https://github.com/matrix-org/synapse/issues/15180)) +- Stop applying edits when bundling aggregations, per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925). ([\#15193](https://github.com/matrix-org/synapse/issues/15193)) +- Fix a long-standing bug where the user directory search was not case-insensitive for accented characters. ([\#15143](https://github.com/matrix-org/synapse/issues/15143)) + + +Updates to the Docker image +--------------------------- + +- Improve startup logging in the with-workers Docker image. ([\#15186](https://github.com/matrix-org/synapse/issues/15186)) + + +Improved Documentation +---------------------- + +- Document how to use caches in a module. ([\#14026](https://github.com/matrix-org/synapse/issues/14026)) +- Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on. ([\#15071](https://github.com/matrix-org/synapse/issues/15071)) +- Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider. ([\#15112](https://github.com/matrix-org/synapse/issues/15112)) +- Correct reference to `federation_verify_certificates` in configuration documentation. ([\#15139](https://github.com/matrix-org/synapse/issues/15139)) +- Correct small documentation errors in some `MatrixFederationHttpClient` methods. ([\#15148](https://github.com/matrix-org/synapse/issues/15148)) +- Correct the description of the behavior of `registration_shared_secret_path` on startup. ([\#15168](https://github.com/matrix-org/synapse/issues/15168)) + + +Deprecations and Removals +------------------------- + +- Deprecate the `on_threepid_bind` module callback, to be replaced by [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.79/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier). See [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.79/docs/upgrade.md#upgrading-to-v1790). ([\#15044](https://github.com/matrix-org/synapse/issues/15044)) +- Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response. ([\#15093](https://github.com/matrix-org/synapse/issues/15093)) +- Remove the unspecced `PUT` on the `/knock/{roomIdOrAlias}` endpoint. ([\#15189](https://github.com/matrix-org/synapse/issues/15189)) +- Remove the undocumented and unspecced `type` parameter to the `/thumbnail` endpoint. ([\#15137](https://github.com/matrix-org/synapse/issues/15137)) +- Remove unspecced and buggy `PUT` method on the unstable `/rooms//batch_send` endpoint. ([\#15199](https://github.com/matrix-org/synapse/issues/15199)) + + +Internal Changes +---------------- + +- Run the integration test suites with the asyncio reactor enabled in CI. ([\#14101](https://github.com/matrix-org/synapse/issues/14101)) +- Batch up storing state groups when creating a new room. ([\#14918](https://github.com/matrix-org/synapse/issues/14918)) +- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15051](https://github.com/matrix-org/synapse/issues/15051)) +- Refactor writing json data in `FileExfiltrationWriter`. ([\#15095](https://github.com/matrix-org/synapse/issues/15095)) +- Tighten the login ratelimit defaults. ([\#15135](https://github.com/matrix-org/synapse/issues/15135)) +- Fix a typo in an experimental config setting. ([\#15138](https://github.com/matrix-org/synapse/issues/15138)) +- Refactor the media modules. ([\#15146](https://github.com/matrix-org/synapse/issues/15146), [\#15175](https://github.com/matrix-org/synapse/issues/15175)) +- Improve type hints. ([\#15164](https://github.com/matrix-org/synapse/issues/15164)) +- Move `get_event_report` and `get_event_reports_paginate` from `RoomStore` to `RoomWorkerStore`. ([\#15165](https://github.com/matrix-org/synapse/issues/15165)) +- Remove dangling reference to being a reference implementation in docstring. ([\#15167](https://github.com/matrix-org/synapse/issues/15167)) +- Add an option to force a rebuild of the "editable" complement image. ([\#15184](https://github.com/matrix-org/synapse/issues/15184)) +- Use nightly rustfmt in CI. ([\#15188](https://github.com/matrix-org/synapse/issues/15188)) +- Add a `get_next_txn` method to `StreamIdGenerator` to match `MultiWriterIdGenerator`. ([\#15191](https://github.com/matrix-org/synapse/issues/15191)) +- Combine `AbstractStreamIdTracker` and `AbstractStreamIdGenerator`. ([\#15192](https://github.com/matrix-org/synapse/issues/15192)) +- Automatically fix errors with `ruff`. ([\#15194](https://github.com/matrix-org/synapse/issues/15194)) +- Refactor database transaction for query users' devices to reduce database pool contention. ([\#15215](https://github.com/matrix-org/synapse/issues/15215)) +- Correct `test_icu_word_boundary_punctuation` so that it passes with the ICU versions available in Alpine and macOS. ([\#15177](https://github.com/matrix-org/synapse/issues/15177)) + +
Locked dependency updates + + - Bump actions/checkout from 2 to 3. ([\#15155](https://github.com/matrix-org/synapse/issues/15155)) + - Bump black from 22.12.0 to 23.1.0. ([\#15103](https://github.com/matrix-org/synapse/issues/15103)) + - Bump dawidd6/action-download-artifact from 2.25.0 to 2.26.0. ([\#15152](https://github.com/matrix-org/synapse/issues/15152)) + - Bump docker/login-action from 1 to 2. ([\#15154](https://github.com/matrix-org/synapse/issues/15154)) + - Bump matrix-org/backend-meta from 1 to 2. ([\#15156](https://github.com/matrix-org/synapse/issues/15156)) + - Bump ruff from 0.0.237 to 0.0.252. ([\#15159](https://github.com/matrix-org/synapse/issues/15159)) + - Bump serde_json from 1.0.93 to 1.0.94. ([\#15214](https://github.com/matrix-org/synapse/issues/15214)) + - Bump types-commonmark from 0.9.2.1 to 0.9.2.2. ([\#15209](https://github.com/matrix-org/synapse/issues/15209)) + - Bump types-opentracing from 2.4.10.1 to 2.4.10.3. ([\#15158](https://github.com/matrix-org/synapse/issues/15158)) + - Bump types-pillow from 9.4.0.13 to 9.4.0.17. ([\#15211](https://github.com/matrix-org/synapse/issues/15211)) + - Bump types-psycopg2 from 2.9.21.4 to 2.9.21.8. ([\#15210](https://github.com/matrix-org/synapse/issues/15210)) + - Bump types-pyopenssl from 22.1.0.2 to 23.0.0.4. ([\#15213](https://github.com/matrix-org/synapse/issues/15213)) + - Bump types-setuptools from 67.3.0.1 to 67.4.0.3. ([\#15160](https://github.com/matrix-org/synapse/issues/15160)) + - Bump types-setuptools from 67.4.0.3 to 67.5.0.0. ([\#15212](https://github.com/matrix-org/synapse/issues/15212)) + - Bump typing-extensions from 4.4.0 to 4.5.0. ([\#15157](https://github.com/matrix-org/synapse/issues/15157)) +
+ + +Synapse 1.78.0 (2023-02-28) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.76 where 5s delays would occasionally occur in deployments using workers. ([\#15150](https://github.com/matrix-org/synapse/issues/15150)) + + +Synapse 1.78.0rc1 (2023-02-21) +============================== + +Features +-------- + +- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964)) +- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969)) +- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004)) +- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034)) +- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042)) +- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045)) +- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073)) +- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075)) + + +Bugfixes +-------- + +- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779)) +- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605)) +- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038)) +- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068)) +- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074)) +- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079)) +- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080)) +- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096)) +- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108)) + + +Improved Documentation +---------------------- + +- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022)) +- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959)) +- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078)) +- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083)) + + +Internal Changes +---------------- + +- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755)) +- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606)) +- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675)) +- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742)) +- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834)) +- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840)) +- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929)) +- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973)) +- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977)) +- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980)) +- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982)) +- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084)) +- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037)) +- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040)) +- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041)) +- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043)) +- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047)) +- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053)) +- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054)) +- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069)) +- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070)) +- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114)) +- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120)) + +
Locked dependency updates + +- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059)) +- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020)) +- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033)) +- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060)) +- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061)) +- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062)) +- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063)) +- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064)) +- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065)) +- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099)) +- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100)) +- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101)) +- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102)) +- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104)) +- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105)) + + +
+ + +Synapse 1.77.0 (2023-02-14) +=========================== + +No significant changes since 1.77.0rc2. + + +Synapse 1.77.0rc2 (2023-02-10) +============================== + +Bugfixes +-------- + +- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024)) + + +Internal Changes +---------------- + +- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036)) + + +Synapse 1.77.0rc1 (2023-02-07) +============================== + +Features +-------- + +- Experimental support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#14823](https://github.com/matrix-org/synapse/issues/14823), [\#14943](https://github.com/matrix-org/synapse/issues/14943), [\#14957](https://github.com/matrix-org/synapse/issues/14957), [\#14958](https://github.com/matrix-org/synapse/issues/14958)) +- Experimental support to suppress notifications from message edits ([MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958)). ([\#14960](https://github.com/matrix-org/synapse/issues/14960), [\#15016](https://github.com/matrix-org/synapse/issues/15016)) +- Add profile information, devices and connections to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.77/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14894](https://github.com/matrix-org/synapse/issues/14894)) +- Improve performance when joining or sending an event in large rooms. ([\#14962](https://github.com/matrix-org/synapse/issues/14962)) +- Improve performance of joining and leaving large rooms with many local users. ([\#14971](https://github.com/matrix-org/synapse/issues/14971)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.53.0 where `next_batch` tokens from `/sync` could not be used with the `/relations` endpoint. ([\#14866](https://github.com/matrix-org/synapse/issues/14866)) +- Fix a bug introduced in Synapse 1.35.0 where the module API's `send_local_online_presence_to` would fail to send presence updates over federation. ([\#14880](https://github.com/matrix-org/synapse/issues/14880)) +- Fix a bug introduced in Synapse 1.70.0 where the background updates to add non-thread unique indexes on receipts could fail when upgrading from 1.67.0 or earlier. ([\#14915](https://github.com/matrix-org/synapse/issues/14915)) +- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14926](https://github.com/matrix-org/synapse/issues/14926)) +- Fix a bug introduced in Synapse 1.68.0 where we were unable to service remote joins in rooms with `@room` notification levels set to `null` in their (malformed) power levels. ([\#14942](https://github.com/matrix-org/synapse/issues/14942)) +- Fix a bug introduced in Synapse 1.64.0 where boolean power levels were erroneously permitted in [v10 rooms](https://spec.matrix.org/v1.5/rooms/v10/). ([\#14944](https://github.com/matrix-org/synapse/issues/14944)) +- Fix a long-standing bug where sending messages on servers with presence enabled would spam "Re-starting finished log context" log lines. ([\#14947](https://github.com/matrix-org/synapse/issues/14947)) +- Fix a bug introduced in Synapse 1.68.0 where logging from the Rust module was not properly logged. ([\#14976](https://github.com/matrix-org/synapse/issues/14976)) +- Fix various long-standing bugs in Synapse's config, event and request handling where booleans were unintentionally accepted where an integer was expected. ([\#14945](https://github.com/matrix-org/synapse/issues/14945)) + + +Internal Changes +---------------- + +- Add missing type hints. ([\#14879](https://github.com/matrix-org/synapse/issues/14879), [\#14886](https://github.com/matrix-org/synapse/issues/14886), [\#14887](https://github.com/matrix-org/synapse/issues/14887), [\#14904](https://github.com/matrix-org/synapse/issues/14904), [\#14927](https://github.com/matrix-org/synapse/issues/14927), [\#14956](https://github.com/matrix-org/synapse/issues/14956), [\#14983](https://github.com/matrix-org/synapse/issues/14983), [\#14984](https://github.com/matrix-org/synapse/issues/14984), [\#14985](https://github.com/matrix-org/synapse/issues/14985), [\#14987](https://github.com/matrix-org/synapse/issues/14987), [\#14988](https://github.com/matrix-org/synapse/issues/14988), [\#14990](https://github.com/matrix-org/synapse/issues/14990), [\#14991](https://github.com/matrix-org/synapse/issues/14991), [\#14992](https://github.com/matrix-org/synapse/issues/14992), [\#15007](https://github.com/matrix-org/synapse/issues/15007)) +- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14922](https://github.com/matrix-org/synapse/issues/14922)) +- Allow running the complement tests suites with the asyncio reactor enabled. ([\#14858](https://github.com/matrix-org/synapse/issues/14858)) +- Improve performance of `/sync` in a few situations. ([\#14908](https://github.com/matrix-org/synapse/issues/14908), [\#14970](https://github.com/matrix-org/synapse/issues/14970)) +- Document how to handle Dependabot pull requests. ([\#14916](https://github.com/matrix-org/synapse/issues/14916)) +- Fix typo in release script. ([\#14920](https://github.com/matrix-org/synapse/issues/14920)) +- Update build system requirements to allow building with poetry-core 1.5.0. ([\#14949](https://github.com/matrix-org/synapse/issues/14949), [\#15019](https://github.com/matrix-org/synapse/issues/15019)) +- Add an [lnav](https://lnav.org) config file for Synapse logs to `/contrib/lnav`. ([\#14953](https://github.com/matrix-org/synapse/issues/14953)) +- Faster joins: Refactor internal handling of servers in room to never store an empty list. ([\#14954](https://github.com/matrix-org/synapse/issues/14954)) +- Faster joins: tag `v2/send_join/` requests to indicate if they served a partial join response. ([\#14950](https://github.com/matrix-org/synapse/issues/14950)) +- Allow running `cargo` without the `extension-module` option. ([\#14965](https://github.com/matrix-org/synapse/issues/14965)) +- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014)) +- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002)) + +
Locked dependency updates + +- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968)) +- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952)) +- Bump ijson from 3.1.4 to 3.2.0.post0. ([\#14935](https://github.com/matrix-org/synapse/issues/14935)) +- Bump types-pyyaml from 6.0.12.2 to 6.0.12.3. ([\#14936](https://github.com/matrix-org/synapse/issues/14936)) +- Bump types-jsonschema from 4.17.0.2 to 4.17.0.3. ([\#14937](https://github.com/matrix-org/synapse/issues/14937)) +- Bump types-pillow from 9.4.0.3 to 9.4.0.5. ([\#14938](https://github.com/matrix-org/synapse/issues/14938)) +- Bump hiredis from 2.0.0 to 2.1.1. ([\#14939](https://github.com/matrix-org/synapse/issues/14939)) +- Bump hiredis from 2.1.1 to 2.2.1. ([\#14993](https://github.com/matrix-org/synapse/issues/14993)) +- Bump types-setuptools from 65.6.0.3 to 67.1.0.0. ([\#14994](https://github.com/matrix-org/synapse/issues/14994)) +- Bump prometheus-client from 0.15.0 to 0.16.0. ([\#14995](https://github.com/matrix-org/synapse/issues/14995)) +- Bump anyhow from 1.0.68 to 1.0.69. ([\#14996](https://github.com/matrix-org/synapse/issues/14996)) +- Bump serde_json from 1.0.91 to 1.0.92. ([\#14997](https://github.com/matrix-org/synapse/issues/14997)) +- Bump isort from 5.11.4 to 5.11.5. ([\#14998](https://github.com/matrix-org/synapse/issues/14998)) +- Bump phonenumbers from 8.13.4 to 8.13.5. ([\#14999](https://github.com/matrix-org/synapse/issues/14999)) +
+ +Synapse 1.76.0 (2023-01-31) +=========================== + +The 1.76 release is the first to enable faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706) and [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902)) by default. Admins can opt-out: see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#faster-joins-are-enabled-by-default) for more details. + +The upgrade from 1.75 to 1.76 changes the account data replication streams in a backwards-incompatible manner. Server operators running a multi-worker deployment should consult [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#changes-to-the-account-data-replication-streams). + +Those who are `poetry install`ing from source using our lockfile should ensure their poetry version is 1.3.2 or higher; [see upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.76/docs/upgrade.md#minimum-version-of-poetry-is-now-132). + + +Notes on faster joins +--------------------- + +The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. + +After a faster join, Synapse considers that room "partially joined". In this state, you should be able to + +- read incoming messages; +- see incoming state changes, e.g. room topic changes; and +- send messages, if the room is unencrypted. + +Synapse has to spend more effort to complete the join in the background. Once this finishes, you will be able to + +- send messages, if the room is in encrypted; +- retrieve room history from before your join, if permitted by the room settings; and +- access the full list of room members. + + +Improved Documentation +---------------------- + +- Describe the ideas and the internal machinery behind faster joins. ([\#14677](https://github.com/matrix-org/synapse/issues/14677)) + + +Synapse 1.76.0rc2 (2023-01-27) +============================== + +Bugfixes +-------- + +- Faster joins: Fix a bug introduced in Synapse 1.69 where device list EDUs could fail to be handled after a restart when a faster join sync is in progress. ([\#14914](https://github.com/matrix-org/synapse/issues/14914)) + + +Internal Changes +---------------- + +- Faster joins: Improve performance of looking up partial-state status of rooms. ([\#14917](https://github.com/matrix-org/synapse/issues/14917)) + + +Synapse 1.76.0rc1 (2023-01-25) +============================== + +Features +-------- + +- Update the default room version to [v10](https://spec.matrix.org/v1.5/rooms/v10/) ([MSC 3904](https://github.com/matrix-org/matrix-spec-proposals/pull/3904)). Contributed by @FSG-Cat. ([\#14111](https://github.com/matrix-org/synapse/issues/14111)) +- Add a `set_displayname()` method to the module API for setting a user's display name. ([\#14629](https://github.com/matrix-org/synapse/issues/14629)) +- Add a dedicated listener configuration for `health` endpoint. ([\#14747](https://github.com/matrix-org/synapse/issues/14747)) +- Implement support for [MSC3890](https://github.com/matrix-org/matrix-spec-proposals/pull/3890): Remotely silence local notifications. ([\#14775](https://github.com/matrix-org/synapse/issues/14775)) +- Implement experimental support for [MSC3930](https://github.com/matrix-org/matrix-spec-proposals/pull/3930): Push rules for ([MSC3381](https://github.com/matrix-org/matrix-spec-proposals/pull/3381)) Polls. ([\#14787](https://github.com/matrix-org/synapse/issues/14787)) +- Per [MSC3925](https://github.com/matrix-org/matrix-spec-proposals/pull/3925), bundle the whole of the replacement with any edited events, and optionally inhibit server-side replacement. ([\#14811](https://github.com/matrix-org/synapse/issues/14811)) +- Faster joins: always serve a partial join response to servers that request it with the stable query param. ([\#14839](https://github.com/matrix-org/synapse/issues/14839)) +- Faster joins: allow non-lazy-loading ("eager") syncs to complete after a partial join by omitting partial state rooms until they become fully stated. ([\#14870](https://github.com/matrix-org/synapse/issues/14870)) +- Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. ([\#14905](https://github.com/matrix-org/synapse/issues/14905)) + + +Bugfixes +-------- + +- Add index to improve performance of the `/timestamp_to_event` endpoint used for jumping to a specific date in the timeline of a room. ([\#14799](https://github.com/matrix-org/synapse/issues/14799)) +- Fix a long-standing bug where Synapse would exhaust the stack when processing many federation requests where the remote homeserver has disconencted early. ([\#14812](https://github.com/matrix-org/synapse/issues/14812), [\#14842](https://github.com/matrix-org/synapse/issues/14842)) +- Fix rare races when using workers. ([\#14820](https://github.com/matrix-org/synapse/issues/14820)) +- Fix a bug introduced in Synapse 1.64.0 when using room version 10 with frozen events enabled. ([\#14864](https://github.com/matrix-org/synapse/issues/14864)) +- Fix a long-standing bug where the `populate_room_stats` background job could fail on broken rooms. ([\#14873](https://github.com/matrix-org/synapse/issues/14873)) +- Faster joins: Fix a bug in worker deployments where the room stats and user directory would not get updated when finishing a fast join until another event is sent or received. ([\#14874](https://github.com/matrix-org/synapse/issues/14874)) +- Faster joins: Fix incompatibility with joins into restricted rooms where no local users have the ability to invite. ([\#14882](https://github.com/matrix-org/synapse/issues/14882)) +- Fix a regression introduced in Synapse 1.69.0 which can result in database corruption when database migrations are interrupted on sqlite. ([\#14910](https://github.com/matrix-org/synapse/issues/14910)) + + +Updates to the Docker image +--------------------------- + +- Bump default Python version in the Dockerfile from 3.9 to 3.11. ([\#14875](https://github.com/matrix-org/synapse/issues/14875)) + + +Improved Documentation +---------------------- + +- Include `x_forwarded` entry in the HTTP listener example configs and remove the remaining `worker_main_http_uri` entries. ([\#14667](https://github.com/matrix-org/synapse/issues/14667)) +- Remove duplicate commands from the Code Style documentation page; point to the Contributing Guide instead. ([\#14773](https://github.com/matrix-org/synapse/issues/14773)) +- Add missing documentation for `tag` to `listeners` section. ([\#14803](https://github.com/matrix-org/synapse/issues/14803)) +- Updated documentation in configuration manual for `user_directory.search_all_users`. ([\#14818](https://github.com/matrix-org/synapse/issues/14818)) +- Add `worker_manhole` to configuration manual. ([\#14824](https://github.com/matrix-org/synapse/issues/14824)) +- Fix the example config missing the `id` field in [application service documentation](https://matrix-org.github.io/synapse/latest/application_services.html). ([\#14845](https://github.com/matrix-org/synapse/issues/14845)) +- Minor corrections to the logging configuration documentation. ([\#14868](https://github.com/matrix-org/synapse/issues/14868)) +- Document the export user data command. Contributed by @thezaidbintariq. ([\#14883](https://github.com/matrix-org/synapse/issues/14883)) + + +Deprecations and Removals +------------------------- + +- Poetry 1.3.2 or higher is now required when `poetry install`ing from source. ([\#14860](https://github.com/matrix-org/synapse/issues/14860)) + + +Internal Changes +---------------- + +- Faster remote room joins (worker mode): do not populate external hosts-in-room cache when sending events as this requires blocking for full state. ([\#14749](https://github.com/matrix-org/synapse/issues/14749)) +- Enable Complement tests for Faster Remote Room Joins against worker-mode Synapse. ([\#14752](https://github.com/matrix-org/synapse/issues/14752)) +- Add some clarifying comments and refactor a portion of the `Keyring` class for readability. ([\#14804](https://github.com/matrix-org/synapse/issues/14804)) +- Add local poetry config files (`poetry.toml`) to `.gitignore`. ([\#14807](https://github.com/matrix-org/synapse/issues/14807)) +- Add missing type hints. ([\#14816](https://github.com/matrix-org/synapse/issues/14816), [\#14885](https://github.com/matrix-org/synapse/issues/14885), [\#14889](https://github.com/matrix-org/synapse/issues/14889)) +- Refactor push tests. ([\#14819](https://github.com/matrix-org/synapse/issues/14819)) +- Re-enable some linting that was disabled when we switched to ruff. ([\#14821](https://github.com/matrix-org/synapse/issues/14821)) +- Add `cargo fmt` and `cargo clippy` to the lint script. ([\#14822](https://github.com/matrix-org/synapse/issues/14822)) +- Drop unused table `presence`. ([\#14825](https://github.com/matrix-org/synapse/issues/14825)) +- Merge the two account data and the two device list replication streams. ([\#14826](https://github.com/matrix-org/synapse/issues/14826), [\#14833](https://github.com/matrix-org/synapse/issues/14833)) +- Faster joins: use stable identifiers from [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). ([\#14832](https://github.com/matrix-org/synapse/issues/14832), [\#14841](https://github.com/matrix-org/synapse/issues/14841)) +- Add a parameter to control whether the federation client performs a partial state join. ([\#14843](https://github.com/matrix-org/synapse/issues/14843)) +- Add check to avoid starting duplicate partial state syncs. ([\#14844](https://github.com/matrix-org/synapse/issues/14844)) +- Add an early return when handling no-op presence updates. ([\#14855](https://github.com/matrix-org/synapse/issues/14855)) +- Fix `wait_for_stream_position` to correctly wait for the right instance to advance its token. ([\#14856](https://github.com/matrix-org/synapse/issues/14856), [\#14872](https://github.com/matrix-org/synapse/issues/14872)) +- Always notify replication when a stream advances automatically. ([\#14877](https://github.com/matrix-org/synapse/issues/14877)) +- Reduce max time we wait for stream positions. ([\#14881](https://github.com/matrix-org/synapse/issues/14881)) +- Faster joins: allow the resync process more time to fetch `/state` ids. ([\#14912](https://github.com/matrix-org/synapse/issues/14912)) +- Bump regex from 1.7.0 to 1.7.1. ([\#14848](https://github.com/matrix-org/synapse/issues/14848)) +- Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2. ([\#14861](https://github.com/matrix-org/synapse/issues/14861)) +- Bump ruff from 0.0.215 to 0.0.224. ([\#14862](https://github.com/matrix-org/synapse/issues/14862)) +- Bump types-pillow from 9.4.0.0 to 9.4.0.3. ([\#14863](https://github.com/matrix-org/synapse/issues/14863)) +- Bump types-opentracing from 2.4.10 to 2.4.10.1. ([\#14896](https://github.com/matrix-org/synapse/issues/14896)) +- Bump ruff from 0.0.224 to 0.0.230. ([\#14897](https://github.com/matrix-org/synapse/issues/14897)) +- Bump types-requests from 2.28.11.7 to 2.28.11.8. ([\#14899](https://github.com/matrix-org/synapse/issues/14899)) +- Bump types-psycopg2 from 2.9.21.2 to 2.9.21.4. ([\#14900](https://github.com/matrix-org/synapse/issues/14900)) +- Bump types-commonmark from 0.9.2 to 0.9.2.1. ([\#14901](https://github.com/matrix-org/synapse/issues/14901)) + + +Synapse 1.75.0 (2023-01-17) +=========================== + +No significant changes since 1.75.0rc2. + + +Synapse 1.75.0rc2 (2023-01-12) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810)) +- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817)) + + +Synapse 1.75.0rc1 (2023-01-10) +============================== + +Features +-------- + +- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663)) +- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714)) +- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750)) +- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753)) +- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786)) +- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263)) + + +Bugfixes +-------- + +- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644)) +- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669)) +- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723)) +- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727)) +- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751)) +- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781)) +- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725)) + + +Improved Documentation +---------------------- + +- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673)) +- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674)) +- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730)) +- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744)) +- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748)) +- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778)) +- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797)) +- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801)) +- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743)) + + +Internal Changes +---------------- + +- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546)) +- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741)) +- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665)) +- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672)) +- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676)) +- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687)) +- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685)) +- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692)) +- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707)) +- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716)) +- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724)) +- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772)) +- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774)) +- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802)) + +### Dependabot updates + +
+ +- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693)) +- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694)) +- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695)) +- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696)) +- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697)) +- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698)) +- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700)) +- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701)) +- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702)) +- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720)) +- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731)) +- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732)) +- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733)) +- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734)) +- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735)) +- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736)) +- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738)) +- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758)) +- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759)) +- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760)) +- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761)) +- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762)) +- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763)) +- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779)) +- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791)) +- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792)) +- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793)) +- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794)) +- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795)) +- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796)) +
diff --git a/docs/changelogs/CHANGES-2024.md b/docs/changelogs/CHANGES-2024.md new file mode 100644 index 0000000000..42c151b6ea --- /dev/null +++ b/docs/changelogs/CHANGES-2024.md @@ -0,0 +1,1576 @@ +# Synapse 1.121.0 (2024-12-11) + +### Internal Changes + +- Fix release process to not create duplicate releases. ([\#18025](https://github.com/element-hq/synapse/issues/18025)) + + + +# Synapse 1.121.0rc1 (2024-12-04) + +### Features + +- Support for [MSC4190](https://github.com/matrix-org/matrix-spec-proposals/pull/4190): device management for Application Services. ([\#17705](https://github.com/element-hq/synapse/issues/17705)) +- Update [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync to include invite, ban, kick, targets when `$LAZY`-loading room members. ([\#17947](https://github.com/element-hq/synapse/issues/17947)) +- Use stable `M_USER_LOCKED` error code for locked accounts, as per [Matrix 1.12](https://spec.matrix.org/v1.12/client-server-api/#account-locking). ([\#17965](https://github.com/element-hq/synapse/issues/17965)) +- [MSC4076](https://github.com/matrix-org/matrix-spec-proposals/pull/4076): Add `disable_badge_count` to pusher configuration. ([\#17975](https://github.com/element-hq/synapse/issues/17975)) + +### Bugfixes + +- Fix long-standing bug where read receipts could get overly delayed being sent over federation. ([\#17933](https://github.com/element-hq/synapse/issues/17933)) + +### Improved Documentation + +- Add OIDC example configuration for Forgejo (fork of Gitea). ([\#17872](https://github.com/element-hq/synapse/issues/17872)) +- Link to element-docker-demo from contrib/docker*. ([\#17953](https://github.com/element-hq/synapse/issues/17953)) + +### Internal Changes + +- [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108): Add a `Content-Type` header on the `PUT` response to work around a faulty behavior in some caching reverse proxies. ([\#17253](https://github.com/element-hq/synapse/issues/17253)) +- Fix incorrect comment in new schema delta. ([\#17936](https://github.com/element-hq/synapse/issues/17936)) +- Raise setuptools_rust version cap to 1.10.2. ([\#17944](https://github.com/element-hq/synapse/issues/17944)) +- Enable encrypted appservice related experimental features in the complement docker image. ([\#17945](https://github.com/element-hq/synapse/issues/17945)) +- Return whether the user is suspended when querying the user account in the Admin API. ([\#17952](https://github.com/element-hq/synapse/issues/17952)) +- Fix new scheduled tasks jumping the queue. ([\#17962](https://github.com/element-hq/synapse/issues/17962)) +- Bump pyo3 and dependencies to v0.23.2. ([\#17966](https://github.com/element-hq/synapse/issues/17966)) +- Update setuptools-rust and fix building abi3 wheels in latest version. ([\#17969](https://github.com/element-hq/synapse/issues/17969)) +- Consolidate SSO redirects through `/_matrix/client/v3/login/sso/redirect(/{idpId})`. ([\#17972](https://github.com/element-hq/synapse/issues/17972)) +- Fix Docker and Complement config to be able to use `public_baseurl`. ([\#17986](https://github.com/element-hq/synapse/issues/17986)) +- Fix building wheels for MacOS which was temporarily disabled in Synapse 1.120.2. ([\#17993](https://github.com/element-hq/synapse/issues/17993)) +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970), [\#17995](https://github.com/element-hq/synapse/issues/17995)) + + +### Updates to locked dependencies + +* Bump bytes from 1.8.0 to 1.9.0. ([\#17982](https://github.com/element-hq/synapse/issues/17982)) +* Bump pysaml2 from 7.3.1 to 7.5.0. ([\#17978](https://github.com/element-hq/synapse/issues/17978)) +* Bump serde_json from 1.0.132 to 1.0.133. ([\#17939](https://github.com/element-hq/synapse/issues/17939)) +* Bump tomli from 2.0.2 to 2.1.0. ([\#17959](https://github.com/element-hq/synapse/issues/17959)) +* Bump tomli from 2.1.0 to 2.2.1. ([\#17979](https://github.com/element-hq/synapse/issues/17979)) +* Bump tornado from 6.4.1 to 6.4.2. ([\#17955](https://github.com/element-hq/synapse/issues/17955)) + +# Synapse 1.120.2 (2024-12-03) + +This version has building of wheels for macOS disabled. +It is functionally identical to 1.120.1, which contains multiple security fixes. +If you are already using 1.120.1, there is no need to upgrade to this version. + + + +# Synapse 1.120.1 (2024-12-03) + +This patch release fixes multiple security vulnerabilities, some affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. + +Administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. + +### Security advisory + +The following issues are fixed in 1.120.1. + +- [GHSA-rfq8-j7rh-8hf2](https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2) / [CVE-2024-52805](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52805): **Unsupported content types can lead to memory exhaustion** + + Synapse instances which have a high `max_upload_size` and which don't have a reverse proxy in front of them that would otherwise limit upload size are affected. + + Fixed by [4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf](https://github.com/element-hq/synapse/commit/4b7154c58501b4bf5e1c2d6c11ebef96529f2fdf). + +- [GHSA-f3r3-h2mq-hx2h](https://github.com/element-hq/synapse/security/advisories/GHSA-f3r3-h2mq-hx2h) / [CVE-2024-52815](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-52815): **Malicious invites via federation can break a user's sync** + + Fixed by [d82e1ed357b7ee21dff83d06cba7a67840cfd464](https://github.com/element-hq/synapse/commit/d82e1ed357b7ee21dff83d06cba7a67840cfd464). + +- [GHSA-vp6v-whfm-rv3g](https://github.com/element-hq/synapse/security/advisories/GHSA-vp6v-whfm-rv3g) / [CVE-2024-53863](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53863): **Synapse can be forced to thumbnail unexpected file formats, invoking potentially untrustworthy decoders** + + Synapse instances can disable dynamic thumbnailing by setting `dynamic_thumbnails` to `false` in the configuration file. + + Fixed by [b64a4e5fbbbf119b6c65aedf0d999b4237d55503](https://github.com/element-hq/synapse/commit/b64a4e5fbbbf119b6c65aedf0d999b4237d55503). + +- [GHSA-56w4-5538-8v8h](https://github.com/element-hq/synapse/security/advisories/GHSA-56w4-5538-8v8h) / [CVE-2024-53867](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-53867): **The Sliding Sync feature on Synapse versions between 1.113.0rc1 and 1.120.0 can leak partial room state changes to users no longer in a room** + + Non-state events, like messages, are unaffected. + + Synapse instances can disable the Sliding Sync feature by setting `experimental_features.msc3575_enabled` to `false` in the configuration file. + + Fixed by [4daa533e82f345ce87b9495d31781af570ba3ead](https://github.com/element-hq/synapse/commit/4daa533e82f345ce87b9495d31781af570ba3ead). + +See the advisories for more details. If you have any questions, email [security at element.io](mailto:security@element.io). + +### Bugfixes + +- Fix release process to not create duplicate releases. ([\#17970](https://github.com/element-hq/synapse/issues/17970)) + + + +# Synapse 1.120.0 (2024-11-26) + +### Bugfixes + +- Fix a bug introduced in Synapse v1.120rc1 which would cause the newly-introduced `delete_old_otks` job to fail in worker-mode deployments. ([\#17960](https://github.com/element-hq/synapse/issues/17960)) + + + + +# Synapse 1.120.0rc1 (2024-11-20) + +This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the +homeserver's media store. + +Most homeservers operating in the public federation will not be impacted by this change, given that +the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers +will already have updated as a result. + +Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients +and older federated homeservers. +See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information. + +### Features + +- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889)) +- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934)) + +### Improved Documentation + +- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913)) +- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931)) + +### Deprecations and Removals + +- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638)) + +### Internal Changes + +- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865)) +- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923)) +- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924)) +- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928)) +- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932)) +- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920)) +* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918)) +* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941)) +* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940)) +* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942)) +* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917)) +* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919)) +* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938)) + +# Synapse 1.119.0 (2024-11-13) + +No significant changes since 1.119.0rc2. + +### Python 3.8 support dropped + +Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9. + +If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse. + + +# Synapse 1.119.0rc2 (2024-11-11) + +Note that due to packaging issues there was no v1.119.0rc1. + + +### Features + +- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374)) +- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888)) + +### Bugfixes + +- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809)) +- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839)) +- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in + the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847)) +- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850)) +- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861)) +- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903)) +- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915)) + +### Internal Changes + +- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908)) +- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786)) +- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890)) +- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830)) +- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852)) +- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884)) +- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887)) +- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894)) +- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905)) +- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911)) +- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902)) +- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909)) +- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912)) +- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921)) + +### Updates to locked dependencies + +* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657)) +* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901)) +* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877)) +* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853)) +* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898)) +* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899)) +* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879)) +* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874)) +* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897)) +* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900)) +* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857)) +* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855)) +* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856)) + +# Synapse 1.118.0 (2024-10-29) + +No significant changes since 1.118.0rc1. + +### Python 3.8 support will be dropped in the next release + +Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0. + +Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month. + +### Python 3.13 and PostgreSQL 17 support + +On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release. + + +# Synapse 1.118.0rc1 (2024-10-22) + +### Features + +- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708)) +- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783)) + +### Bugfixes + +- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736)) +- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805)) +- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835)) + +### Improved Documentation + +- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627)) +- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802)) +- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825)) + +### Internal Changes + +- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718)) +- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752)) +- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803)) +- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824)) +- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826)) +- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) + + + +### Updates to locked dependencies + +* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842)) +* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827)) +* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797)) +* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843)) +* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794)) +* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795)) +* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829)) +* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844)) +* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798)) +* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796)) +* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841)) +* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828)) + +# Synapse 1.117.0 (2024-10-15) + +No significant changes since 1.117.0rc1. + + + + +# Synapse 1.117.0rc1 (2024-10-08) + +### Features + +- Add config option `redis.password_path`. ([\#17717](https://github.com/element-hq/synapse/issues/17717)) + +### Bugfixes + +- Fix a rare bug introduced in v1.29.0 where invalidating a user's access token from a worker could raise an error. ([\#17779](https://github.com/element-hq/synapse/issues/17779)) +- In the response to `GET /_matrix/client/versions`, set the `unstable_features` flag for [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) to `false` when server configuration disables support for delayed events. ([\#17780](https://github.com/element-hq/synapse/issues/17780)) +- Improve input validation and room membership checks in admin redaction API. ([\#17792](https://github.com/element-hq/synapse/issues/17792)) + +### Improved Documentation + +- Clarify the docstring of `test_forget_when_not_left`. ([\#17628](https://github.com/element-hq/synapse/issues/17628)) +- Add documentation note about PYTHONMALLOC for accurate jemalloc memory tracking. Contributed by @hensg. ([\#17709](https://github.com/element-hq/synapse/issues/17709)) +- Remove spurious "TODO UPDATE ALL THIS" note in the Debian installation docs. ([\#17749](https://github.com/element-hq/synapse/issues/17749)) +- Explain how load balancing works for `federation_sender_instances`. ([\#17776](https://github.com/element-hq/synapse/issues/17776)) + +### Internal Changes + +- Minor performance increase for large accounts using sliding sync. ([\#17751](https://github.com/element-hq/synapse/issues/17751)) +- Increase performance of the notifier when there are many syncing users. ([\#17765](https://github.com/element-hq/synapse/issues/17765), [\#17766](https://github.com/element-hq/synapse/issues/17766)) +- Fix performance of streams that don't change often. ([\#17767](https://github.com/element-hq/synapse/issues/17767)) +- Improve performance of sliding sync connections that do not ask for any rooms. ([\#17768](https://github.com/element-hq/synapse/issues/17768)) +- Reduce overhead of sliding sync E2EE loops. ([\#17771](https://github.com/element-hq/synapse/issues/17771)) +- Sliding sync minor performance speed up using new table. ([\#17787](https://github.com/element-hq/synapse/issues/17787)) +- Sliding sync minor performance improvement by omitting unchanged data from incremental responses. ([\#17788](https://github.com/element-hq/synapse/issues/17788)) +- Speed up sliding sync when there are many active subscriptions. ([\#17789](https://github.com/element-hq/synapse/issues/17789)) +- Add missing license headers on new source files. ([\#17799](https://github.com/element-hq/synapse/issues/17799)) + + + +### Updates to locked dependencies + +* Bump phonenumbers from 8.13.45 to 8.13.46. ([\#17773](https://github.com/element-hq/synapse/issues/17773)) +* Bump python-multipart from 0.0.10 to 0.0.12. ([\#17772](https://github.com/element-hq/synapse/issues/17772)) +* Bump regex from 1.10.6 to 1.11.0. ([\#17770](https://github.com/element-hq/synapse/issues/17770)) +* Bump ruff from 0.6.7 to 0.6.8. ([\#17774](https://github.com/element-hq/synapse/issues/17774)) + +# Synapse 1.116.0 (2024-10-01) + +No significant changes since 1.116.0rc2. + + + + +# Synapse 1.116.0rc2 (2024-09-26) + +### Features + +- Add implementation of restricting who can overwrite a state event as proposed by [MSC3757](https://github.com/matrix-org/matrix-spec-proposals/pull/3757). ([\#17513](https://github.com/element-hq/synapse/issues/17513)) + + + + +# Synapse 1.116.0rc1 (2024-09-25) + +### Features + +- Add initial implementation of delayed events as proposed by [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). ([\#17326](https://github.com/element-hq/synapse/issues/17326)) +- Add an asynchronous Admin API endpoint [to redact all a user's events](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#redact-all-the-events-of-a-user), + and [an endpoint to check on the status of that redaction task](https://element-hq.github.io/synapse/v1.116/admin_api/user_admin_api.html#check-the-status-of-a-redaction-process). ([\#17506](https://github.com/element-hq/synapse/issues/17506)) +- Add support for the `tags` and `not_tags` filters for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17662](https://github.com/element-hq/synapse/issues/17662)) +- Guests can use the new media endpoints to download media, as described by [MSC4189](https://github.com/matrix-org/matrix-spec-proposals/pull/4189). ([\#17675](https://github.com/element-hq/synapse/issues/17675)) +- Add config option `turn_shared_secret_path`. ([\#17690](https://github.com/element-hq/synapse/issues/17690)) +- Return room tags in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync account data extension. ([\#17707](https://github.com/element-hq/synapse/issues/17707)) + +### Bugfixes + +- Make sure we get up-to-date state information when using the new [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables to derive room membership. ([\#17692](https://github.com/element-hq/synapse/issues/17692)) +- Fix bug where room account data would not correctly be sent down [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync for old rooms. ([\#17695](https://github.com/element-hq/synapse/issues/17695)) +- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync which could prevent /sync from working for certain user accounts. ([\#17727](https://github.com/element-hq/synapse/issues/17727), [\#17733](https://github.com/element-hq/synapse/issues/17733)) +- Ignore invites from ignored users in Sliding Sync. ([\#17729](https://github.com/element-hq/synapse/issues/17729)) +- Fix bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync where the server would incorrectly return a negative bump stamp, which caused Element X apps to stop syncing. ([\#17748](https://github.com/element-hq/synapse/issues/17748)) + +### Internal Changes + +- Import pydantic objects from the `_pydantic_compat` module. + This allows `check_pydantic_models.py` to mock those pydantic objects + only in the synapse module, and not interfere with pydantic objects in + external dependencies. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) +- Use [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables as a bulk shortcut for getting the max `event_stream_ordering` of rooms. ([\#17693](https://github.com/element-hq/synapse/issues/17693)) +- Speed up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests a bit where there are many room changes. ([\#17696](https://github.com/element-hq/synapse/issues/17696)) +- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync filter unit tests so the sliding sync API has better test coverage. ([\#17703](https://github.com/element-hq/synapse/issues/17703)) +- Fetch `bump_stamp`s more efficiently in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17723](https://github.com/element-hq/synapse/issues/17723)) +- Shortcut for checking if certain background updates have completed (utilized in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync). ([\#17724](https://github.com/element-hq/synapse/issues/17724)) +- More efficiently fetch rooms for [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17725](https://github.com/element-hq/synapse/issues/17725)) +- Fix `_bulk_get_max_event_pos` being inefficient. ([\#17728](https://github.com/element-hq/synapse/issues/17728)) +- Add cache to `get_tags_for_room(...)`. ([\#17730](https://github.com/element-hq/synapse/issues/17730)) +- Small performance improvement in speeding up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync. ([\#17731](https://github.com/element-hq/synapse/issues/17731)) +- Minor speed up of initial [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests. ([\#17734](https://github.com/element-hq/synapse/issues/17734)) +- Remove usage of the deprecated `cgi` module, deprecated in Python 3.11 and removed in Python 3.13. ([\#17741](https://github.com/element-hq/synapse/issues/17741)) +- Fix typing of a variable that is not `Unknown` anymore after updating `treq`. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.86 to 1.0.89. ([\#17685](https://github.com/element-hq/synapse/issues/17685), [\#17716](https://github.com/element-hq/synapse/issues/17716)) +* Bump bytes from 1.7.1 to 1.7.2. ([\#17743](https://github.com/element-hq/synapse/issues/17743)) +* Bump cryptography from 43.0.0 to 43.0.1. ([\#17689](https://github.com/element-hq/synapse/issues/17689)) +* Bump idna from 3.8 to 3.10. ([\#17758](https://github.com/element-hq/synapse/issues/17758)) +* Bump msgpack from 1.0.8 to 1.1.0. ([\#17759](https://github.com/element-hq/synapse/issues/17759)) +* Bump phonenumbers from 8.13.44 to 8.13.45. ([\#17762](https://github.com/element-hq/synapse/issues/17762)) +* Bump prometheus-client from 0.20.0 to 0.21.0. ([\#17746](https://github.com/element-hq/synapse/issues/17746)) +* Bump pyasn1 from 0.6.0 to 0.6.1. ([\#17714](https://github.com/element-hq/synapse/issues/17714)) +* Bump pyasn1-modules from 0.4.0 to 0.4.1. ([\#17747](https://github.com/element-hq/synapse/issues/17747)) +* Bump pydantic from 2.8.2 to 2.9.2. ([\#17756](https://github.com/element-hq/synapse/issues/17756)) +* Bump python-multipart from 0.0.9 to 0.0.10. ([\#17745](https://github.com/element-hq/synapse/issues/17745)) +* Bump ruff from 0.6.4 to 0.6.7. ([\#17715](https://github.com/element-hq/synapse/issues/17715), [\#17760](https://github.com/element-hq/synapse/issues/17760)) +* Bump sentry-sdk from 2.13.0 to 2.14.0. ([\#17712](https://github.com/element-hq/synapse/issues/17712)) +* Bump serde from 1.0.209 to 1.0.210. ([\#17686](https://github.com/element-hq/synapse/issues/17686)) +* Bump serde_json from 1.0.127 to 1.0.128. ([\#17687](https://github.com/element-hq/synapse/issues/17687)) +* Bump treq from 23.11.0 to 24.9.1. ([\#17744](https://github.com/element-hq/synapse/issues/17744)) +* Bump types-pyyaml from 6.0.12.20240808 to 6.0.12.20240917. ([\#17755](https://github.com/element-hq/synapse/issues/17755)) +* Bump types-requests from 2.32.0.20240712 to 2.32.0.20240914. ([\#17713](https://github.com/element-hq/synapse/issues/17713)) +* Bump types-setuptools from 74.1.0.20240907 to 75.1.0.20240917. ([\#17757](https://github.com/element-hq/synapse/issues/17757)) + +# Synapse 1.115.0 (2024-09-17) + +No significant changes since 1.115.0rc2. + + + + +# Synapse 1.115.0rc2 (2024-09-12) + +### Internal Changes + +- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17652](https://github.com/element-hq/synapse/issues/17652)) +- Speed up sliding sync by reducing amount of data pulled out of the database for large rooms. ([\#17683](https://github.com/element-hq/synapse/issues/17683)) + + + + +# Synapse 1.115.0rc1 (2024-09-10) + +### Features + +- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) + +### Bugfixes + +- Return `400 M_BAD_JSON` upon attempting to complete various room actions with a non-local user ID and unknown room ID, rather than an internal server error. ([\#17607](https://github.com/element-hq/synapse/issues/17607)) +- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) +- Fix bug where we returned the wrong `bump_stamp` for invites in sliding sync response, causing incorrect ordering of invites in the room list. ([\#17674](https://github.com/element-hq/synapse/issues/17674)) + +### Improved Documentation + +- Clarify that the admin api resource is only loaded on the main process and not workers. ([\#17590](https://github.com/element-hq/synapse/issues/17590)) +- Fixed typo in `saml2_config` config [example](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#saml2_config). ([\#17594](https://github.com/element-hq/synapse/issues/17594)) + +### Deprecations and Removals + +- Stabilise [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156) by removing the `msc4156_enabled` config setting and defaulting it to `true`. ([\#17650](https://github.com/element-hq/synapse/issues/17650)) + +### Internal Changes + +- Update [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) +- Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting. ([\#17512](https://github.com/element-hq/synapse/issues/17512), [\#17632](https://github.com/element-hq/synapse/issues/17632), [\#17633](https://github.com/element-hq/synapse/issues/17633), [\#17634](https://github.com/element-hq/synapse/issues/17634), [\#17635](https://github.com/element-hq/synapse/issues/17635), [\#17636](https://github.com/element-hq/synapse/issues/17636), [\#17641](https://github.com/element-hq/synapse/issues/17641), [\#17654](https://github.com/element-hq/synapse/issues/17654), [\#17673](https://github.com/element-hq/synapse/issues/17673)) +- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599), [\#17631](https://github.com/element-hq/synapse/issues/17631)) +- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) +- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620), [\#17643](https://github.com/element-hq/synapse/issues/17643)) +- Sliding Sync: Split up `get_room_membership_for_user_at_to_token`. ([\#17629](https://github.com/element-hq/synapse/issues/17629)) +- Use new database tables for sliding sync. ([\#17630](https://github.com/element-hq/synapse/issues/17630), [\#17649](https://github.com/element-hq/synapse/issues/17649)) +- Prevent duplicate tags being added to Sliding Sync traces. ([\#17655](https://github.com/element-hq/synapse/issues/17655)) +- Get `bump_stamp` from [new sliding sync tables](https://github.com/element-hq/synapse/pull/17512) which should be faster. ([\#17658](https://github.com/element-hq/synapse/issues/17658)) +- Speed up incremental Sliding Sync requests by avoiding extra work. ([\#17665](https://github.com/element-hq/synapse/issues/17665)) +- Small performance improvement in speeding up sliding sync. ([\#17666](https://github.com/element-hq/synapse/issues/17666), [\#17670](https://github.com/element-hq/synapse/issues/17670), [\#17672](https://github.com/element-hq/synapse/issues/17672)) +- Speed up sliding sync by reducing number of database calls. ([\#17684](https://github.com/element-hq/synapse/issues/17684)) +- Speed up sync by pulling out fewer events from the database. ([\#17688](https://github.com/element-hq/synapse/issues/17688)) + + + +### Updates to locked dependencies + +* Bump authlib from 1.3.1 to 1.3.2. ([\#17679](https://github.com/element-hq/synapse/issues/17679)) +* Bump idna from 3.7 to 3.8. ([\#17682](https://github.com/element-hq/synapse/issues/17682)) +* Bump ruff from 0.6.2 to 0.6.4. ([\#17680](https://github.com/element-hq/synapse/issues/17680)) +* Bump towncrier from 24.7.1 to 24.8.0. ([\#17645](https://github.com/element-hq/synapse/issues/17645)) +* Bump twisted from 24.7.0rc1 to 24.7.0. ([\#17647](https://github.com/element-hq/synapse/issues/17647)) +* Bump types-pillow from 10.2.0.20240520 to 10.2.0.20240822. ([\#17644](https://github.com/element-hq/synapse/issues/17644)) +* Bump types-psycopg2 from 2.9.21.20240417 to 2.9.21.20240819. ([\#17646](https://github.com/element-hq/synapse/issues/17646)) +* Bump types-setuptools from 71.1.0.20240818 to 74.1.0.20240907. ([\#17681](https://github.com/element-hq/synapse/issues/17681)) + +# Synapse 1.114.0 (2024-09-02) + +This release enables support for +[MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) — +Simplified Sliding Sync. This allows using the upcoming releases of the Element +X mobile apps without having to run a Sliding Sync Proxy. + + +### Features + +- Enable native sliding sync support ([MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) and [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186)) by default. ([\#17648](https://github.com/element-hq/synapse/issues/17648)) + + + + +# Synapse 1.114.0rc3 (2024-08-30) + +### Bugfixes + +- Fix regression in v1.114.0rc2 that caused workers to fail to start. ([\#17626](https://github.com/element-hq/synapse/issues/17626)) + + + + +# Synapse 1.114.0rc2 (2024-08-30) + +### Features + +- Improve cross-signing upload when using [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) to use a custom UIA flow stage, with web fallback support. ([\#17509](https://github.com/element-hq/synapse/issues/17509)) +- Make `hash_password` script accept password input from stdin. ([\#17608](https://github.com/element-hq/synapse/issues/17608)) + +### Bugfixes + +- Fix hierarchy returning 403 when room is accessible through federation. Contributed by Krishan (@kfiven). ([\#17194](https://github.com/element-hq/synapse/issues/17194)) +- Fix content-length on federation `/thumbnail` responses. ([\#17532](https://github.com/element-hq/synapse/issues/17532)) +- Fix authenticated media responses using a wrong limit when following redirects over federation. ([\#17543](https://github.com/element-hq/synapse/issues/17543)) + +### Internal Changes + +- MSC3861: load the issuer and account management URLs from OIDC discovery. ([\#17407](https://github.com/element-hq/synapse/issues/17407)) +- Refactor sliding sync class into multiple files. ([\#17595](https://github.com/element-hq/synapse/issues/17595)) +- Store sliding sync per-connection state in the database. ([\#17599](https://github.com/element-hq/synapse/issues/17599)) +- Make the sliding sync `PerConnectionState` class immutable. ([\#17600](https://github.com/element-hq/synapse/issues/17600)) +- Add support to `@tag_args` for standalone functions. ([\#17604](https://github.com/element-hq/synapse/issues/17604)) +- Speed up incremental syncs in sliding sync by adding some more caching. ([\#17606](https://github.com/element-hq/synapse/issues/17606)) +- Always return the user's own read receipts in sliding sync. ([\#17617](https://github.com/element-hq/synapse/issues/17617)) +- Replace `isort` and `black` with `ruff`. ([\#17620](https://github.com/element-hq/synapse/issues/17620)) +- Refactor sliding sync code to move room list logic out into a separate class. ([\#17622](https://github.com/element-hq/synapse/issues/17622)) + + + +### Updates to locked dependencies + +* Bump attrs from 23.2.0 to 24.2.0. ([\#17609](https://github.com/element-hq/synapse/issues/17609)) +* Bump cryptography from 42.0.8 to 43.0.0. ([\#17584](https://github.com/element-hq/synapse/issues/17584)) +* Bump phonenumbers from 8.13.43 to 8.13.44. ([\#17610](https://github.com/element-hq/synapse/issues/17610)) +* Bump pygithub from 2.3.0 to 2.4.0. ([\#17612](https://github.com/element-hq/synapse/issues/17612)) +* Bump pyyaml from 6.0.1 to 6.0.2. ([\#17611](https://github.com/element-hq/synapse/issues/17611)) +* Bump sentry-sdk from 2.12.0 to 2.13.0. ([\#17585](https://github.com/element-hq/synapse/issues/17585)) +* Bump serde from 1.0.206 to 1.0.208. ([\#17581](https://github.com/element-hq/synapse/issues/17581)) +* Bump serde from 1.0.208 to 1.0.209. ([\#17613](https://github.com/element-hq/synapse/issues/17613)) +* Bump serde_json from 1.0.124 to 1.0.125. ([\#17582](https://github.com/element-hq/synapse/issues/17582)) +* Bump serde_json from 1.0.125 to 1.0.127. ([\#17614](https://github.com/element-hq/synapse/issues/17614)) +* Bump types-jsonschema from 4.23.0.20240712 to 4.23.0.20240813. ([\#17583](https://github.com/element-hq/synapse/issues/17583)) +* Bump types-setuptools from 71.1.0.20240726 to 71.1.0.20240818. ([\#17586](https://github.com/element-hq/synapse/issues/17586)) + +# Synapse 1.114.0rc1 (2024-08-20) + +### Features + +- Add a flag to `/versions`, `org.matrix.simplified_msc3575`, to indicate whether experimental sliding sync support has been enabled. ([\#17571](https://github.com/element-hq/synapse/issues/17571)) +- Handle changes in `timeline_limit` in experimental sliding sync. ([\#17579](https://github.com/element-hq/synapse/issues/17579)) +- Correctly track read receipts that should be sent down in experimental sliding sync. ([\#17575](https://github.com/element-hq/synapse/issues/17575), [\#17589](https://github.com/element-hq/synapse/issues/17589), [\#17592](https://github.com/element-hq/synapse/issues/17592)) + +### Bugfixes + +- Start handlers for new media endpoints when media resource configured. ([\#17483](https://github.com/element-hq/synapse/issues/17483)) +- Fix timeline ordering (using `stream_ordering` instead of topological ordering) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17510](https://github.com/element-hq/synapse/issues/17510)) +- Fix experimental sliding sync implementation to remember any updates in rooms that were not sent down immediately. ([\#17535](https://github.com/element-hq/synapse/issues/17535)) +- Better exclude partially stated rooms if we must await full state in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17538](https://github.com/element-hq/synapse/issues/17538)) +- Handle lower-case http headers in `_Mulitpart_Parser_Protocol`. ([\#17545](https://github.com/element-hq/synapse/issues/17545)) +- Fix fetching federation signing keys from servers that omit `old_verify_keys`. Contributed by @tulir @ Beeper. ([\#17568](https://github.com/element-hq/synapse/issues/17568)) +- Fix bug where we would respond with an error when a remote server asked for media that had a length of 0, using the new multipart federation media endpoint. ([\#17570](https://github.com/element-hq/synapse/issues/17570)) + +### Improved Documentation + +- Clarify default behaviour of the + [`auto_accept_invites.worker_to_run_on`](https://element-hq.github.io/synapse/develop/usage/configuration/config_documentation.html#auto-accept-invites) + option. ([\#17515](https://github.com/element-hq/synapse/issues/17515)) +- Improve docstrings for profile methods. ([\#17559](https://github.com/element-hq/synapse/issues/17559)) + +### Internal Changes + +- Add more tracing to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17514](https://github.com/element-hq/synapse/issues/17514)) +- Fixup comment in sliding sync implementation. ([\#17531](https://github.com/element-hq/synapse/issues/17531)) +- Replace override of deprecated method `HTTPAdapter.get_connection` with `get_connection_with_tls_context`. ([\#17536](https://github.com/element-hq/synapse/issues/17536)) +- Fix performance of device lists in `/key/changes` and sliding sync. ([\#17537](https://github.com/element-hq/synapse/issues/17537), [\#17548](https://github.com/element-hq/synapse/issues/17548)) +- Bump setuptools from 67.6.0 to 72.1.0. ([\#17542](https://github.com/element-hq/synapse/issues/17542)) +- Add a utility function for generating random event IDs. ([\#17557](https://github.com/element-hq/synapse/issues/17557)) +- Speed up responding to media requests. ([\#17558](https://github.com/element-hq/synapse/issues/17558), [\#17561](https://github.com/element-hq/synapse/issues/17561), [\#17564](https://github.com/element-hq/synapse/issues/17564), [\#17566](https://github.com/element-hq/synapse/issues/17566), [\#17567](https://github.com/element-hq/synapse/issues/17567), [\#17569](https://github.com/element-hq/synapse/issues/17569)) +- Test github token before running release script steps. ([\#17562](https://github.com/element-hq/synapse/issues/17562)) +- Reduce log spam of multipart files. ([\#17563](https://github.com/element-hq/synapse/issues/17563)) +- Refactor per-connection state in experimental sliding sync handler. ([\#17574](https://github.com/element-hq/synapse/issues/17574)) +- Add histogram metrics for sliding sync processing time. ([\#17593](https://github.com/element-hq/synapse/issues/17593)) + + + +### Updates to locked dependencies + +* Bump bytes from 1.6.1 to 1.7.1. ([\#17526](https://github.com/element-hq/synapse/issues/17526)) +* Bump lxml from 5.2.2 to 5.3.0. ([\#17550](https://github.com/element-hq/synapse/issues/17550)) +* Bump phonenumbers from 8.13.42 to 8.13.43. ([\#17551](https://github.com/element-hq/synapse/issues/17551)) +* Bump regex from 1.10.5 to 1.10.6. ([\#17527](https://github.com/element-hq/synapse/issues/17527)) +* Bump sentry-sdk from 2.10.0 to 2.12.0. ([\#17553](https://github.com/element-hq/synapse/issues/17553)) +* Bump serde from 1.0.204 to 1.0.206. ([\#17556](https://github.com/element-hq/synapse/issues/17556)) +* Bump serde_json from 1.0.122 to 1.0.124. ([\#17555](https://github.com/element-hq/synapse/issues/17555)) +* Bump sigstore/cosign-installer from 3.5.0 to 3.6.0. ([\#17549](https://github.com/element-hq/synapse/issues/17549)) +* Bump types-pyyaml from 6.0.12.20240311 to 6.0.12.20240808. ([\#17552](https://github.com/element-hq/synapse/issues/17552)) +* Bump types-requests from 2.31.0.20240406 to 2.32.0.20240712. ([\#17524](https://github.com/element-hq/synapse/issues/17524)) + +# Synapse 1.113.0 (2024-08-13) + +No significant changes since 1.113.0rc1. + + + + +# Synapse 1.113.0rc1 (2024-08-06) + +### Features + +- Track which rooms have been sent to clients in the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17447](https://github.com/element-hq/synapse/issues/17447)) +- Add Account Data extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17477](https://github.com/element-hq/synapse/issues/17477)) +- Add receipts extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17489](https://github.com/element-hq/synapse/issues/17489)) +- Add typing notification extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17505](https://github.com/element-hq/synapse/issues/17505)) + +### Bugfixes + +- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to handle invite/knock rooms when filtering. ([\#17450](https://github.com/element-hq/synapse/issues/17450)) +- Fix a bug introduced in v1.110.0 which caused `/keys/query` to return incomplete results, leading to high network activity and CPU usage on Matrix clients. ([\#17499](https://github.com/element-hq/synapse/issues/17499)) + +### Improved Documentation + +- Update the [`allowed_local_3pids`](https://element-hq.github.io/synapse/v1.112/usage/configuration/config_documentation.html#allowed_local_3pids) config option's msisdn address to a working example. ([\#17476](https://github.com/element-hq/synapse/issues/17476)) + +### Internal Changes + +- Change sliding sync to use their own token format in preparation for storing per-connection state. ([\#17452](https://github.com/element-hq/synapse/issues/17452)) +- Ensure we don't send down negative `bump_stamp` in experimental sliding sync endpoint. ([\#17478](https://github.com/element-hq/synapse/issues/17478)) +- Do not send down empty room entries down experimental sliding sync endpoint. ([\#17479](https://github.com/element-hq/synapse/issues/17479)) +- Refactor Sliding Sync tests to better utilize the `SlidingSyncBase`. ([\#17481](https://github.com/element-hq/synapse/issues/17481), [\#17482](https://github.com/element-hq/synapse/issues/17482)) +- Add some opentracing tags and logging to the experimental sliding sync implementation. ([\#17501](https://github.com/element-hq/synapse/issues/17501)) +- Split and move Sliding Sync tests so we have some more sane test file sizes. ([\#17504](https://github.com/element-hq/synapse/issues/17504)) +- Update the `limited` field description in the Sliding Sync response to accurately describe what it actually represents. ([\#17507](https://github.com/element-hq/synapse/issues/17507)) +- Easier to understand `timeline` assertions in Sliding Sync tests. ([\#17511](https://github.com/element-hq/synapse/issues/17511)) +- Reset the sliding sync connection if we don't recognize the per-connection state position. ([\#17529](https://github.com/element-hq/synapse/issues/17529)) + + + +### Updates to locked dependencies + +* Bump bcrypt from 4.1.3 to 4.2.0. ([\#17495](https://github.com/element-hq/synapse/issues/17495)) +* Bump black from 24.4.2 to 24.8.0. ([\#17522](https://github.com/element-hq/synapse/issues/17522)) +* Bump phonenumbers from 8.13.39 to 8.13.42. ([\#17521](https://github.com/element-hq/synapse/issues/17521)) +* Bump ruff from 0.5.4 to 0.5.5. ([\#17494](https://github.com/element-hq/synapse/issues/17494)) +* Bump serde_json from 1.0.120 to 1.0.121. ([\#17493](https://github.com/element-hq/synapse/issues/17493)) +* Bump serde_json from 1.0.121 to 1.0.122. ([\#17525](https://github.com/element-hq/synapse/issues/17525)) +* Bump towncrier from 23.11.0 to 24.7.1. ([\#17523](https://github.com/element-hq/synapse/issues/17523)) +* Bump types-pyopenssl from 24.1.0.20240425 to 24.1.0.20240722. ([\#17496](https://github.com/element-hq/synapse/issues/17496)) +* Bump types-setuptools from 70.1.0.20240627 to 71.1.0.20240726. ([\#17497](https://github.com/element-hq/synapse/issues/17497)) + +# Synapse 1.112.0 (2024-07-30) + +This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). + +Note that this security fix is also available as **Synapse 1.111.1**, which does not include the rest of the changes in Synapse 1.112.0. + +This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. +If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. + +With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. +The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. + +Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. + +**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. + +### Internal Changes + +- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) + + +# Synapse 1.112.0rc1 (2024-07-23) + +Please note that this release candidate does not include the security dependency update +included in version 1.111.1 as this version was released before 1.111.1. +The same security fix can be found in the full release of 1.112.0. + +### Features + +- Add to-device extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17416](https://github.com/element-hq/synapse/issues/17416)) +- Populate `name`/`avatar` fields in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17418](https://github.com/element-hq/synapse/issues/17418)) +- Populate `heroes` and room summary fields (`joined_count`, `invited_count`) in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17419](https://github.com/element-hq/synapse/issues/17419)) +- Populate `is_dm` room field in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17429](https://github.com/element-hq/synapse/issues/17429)) +- Add room subscriptions to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17432](https://github.com/element-hq/synapse/issues/17432)) +- Prepare for authenticated media freeze. ([\#17433](https://github.com/element-hq/synapse/issues/17433)) +- Add E2EE extension support to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17454](https://github.com/element-hq/synapse/issues/17454)) + +### Bugfixes + +- Add configurable option to always include offline users in presence sync results. Contributed by @Michael-Hollister. ([\#17231](https://github.com/element-hq/synapse/issues/17231)) +- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using room type filters and the user has one or more remote invites. ([\#17434](https://github.com/element-hq/synapse/issues/17434)) +- Order `heroes` by `stream_ordering` as the Matrix specification states (applies to `/sync`). ([\#17435](https://github.com/element-hq/synapse/issues/17435)) +- Fix rare bug where `/sync` would break for a user when using workers with multiple stream writers. ([\#17438](https://github.com/element-hq/synapse/issues/17438)) + +### Improved Documentation + +- Update the readme image to have a white background, so that it is readable in dark mode. ([\#17387](https://github.com/element-hq/synapse/issues/17387)) +- Add Red Hat Enterprise Linux and Rocky Linux 8 and 9 installation instructions. ([\#17423](https://github.com/element-hq/synapse/issues/17423)) +- Improve documentation for the [`default_power_level_content_override`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#default_power_level_content_override) config option. ([\#17451](https://github.com/element-hq/synapse/issues/17451)) + +### Internal Changes + +- Make sure we always use the right logic for enabling the media repo. ([\#17424](https://github.com/element-hq/synapse/issues/17424)) +- Fix argument documentation for method `RateLimiter.record_action`. ([\#17426](https://github.com/element-hq/synapse/issues/17426)) +- Reduce volume of 'Waiting for current token' logs, which were introduced in v1.109.0. ([\#17428](https://github.com/element-hq/synapse/issues/17428)) +- Limit concurrent remote downloads to 6 per IP address, and decrement remote downloads without a content-length from the ratelimiter after the download is complete. ([\#17439](https://github.com/element-hq/synapse/issues/17439)) +- Remove unnecessary call to resume producing in fake channel. ([\#17449](https://github.com/element-hq/synapse/issues/17449)) +- Update experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint to bump room when it is created. ([\#17453](https://github.com/element-hq/synapse/issues/17453)) +- Speed up generating sliding sync responses. ([\#17458](https://github.com/element-hq/synapse/issues/17458)) +- Add cache to `get_rooms_for_local_user_where_membership_is` to speed up sliding sync. ([\#17460](https://github.com/element-hq/synapse/issues/17460)) +- Speed up fetching room keys from backup. ([\#17461](https://github.com/element-hq/synapse/issues/17461)) +- Speed up sorting of the room list in sliding sync. ([\#17468](https://github.com/element-hq/synapse/issues/17468)) +- Implement handling of `$ME` as a state key in sliding sync. ([\#17469](https://github.com/element-hq/synapse/issues/17469)) + + + +### Updates to locked dependencies + +* Bump bytes from 1.6.0 to 1.6.1. ([\#17441](https://github.com/element-hq/synapse/issues/17441)) +* Bump hiredis from 2.3.2 to 3.0.0. ([\#17464](https://github.com/element-hq/synapse/issues/17464)) +* Bump jsonschema from 4.22.0 to 4.23.0. ([\#17444](https://github.com/element-hq/synapse/issues/17444)) +* Bump matrix-org/done-action from 2 to 3. ([\#17440](https://github.com/element-hq/synapse/issues/17440)) +* Bump mypy from 1.9.0 to 1.10.1. ([\#17445](https://github.com/element-hq/synapse/issues/17445)) +* Bump pyopenssl from 24.1.0 to 24.2.1. ([\#17465](https://github.com/element-hq/synapse/issues/17465)) +* Bump ruff from 0.5.0 to 0.5.4. ([\#17466](https://github.com/element-hq/synapse/issues/17466)) +* Bump sentry-sdk from 2.6.0 to 2.8.0. ([\#17456](https://github.com/element-hq/synapse/issues/17456)) +* Bump sentry-sdk from 2.8.0 to 2.10.0. ([\#17467](https://github.com/element-hq/synapse/issues/17467)) +* Bump setuptools from 67.6.0 to 70.0.0. ([\#17448](https://github.com/element-hq/synapse/issues/17448)) +* Bump twine from 5.1.0 to 5.1.1. ([\#17443](https://github.com/element-hq/synapse/issues/17443)) +* Bump types-jsonschema from 4.22.0.20240610 to 4.23.0.20240712. ([\#17446](https://github.com/element-hq/synapse/issues/17446)) +* Bump ulid from 1.1.2 to 1.1.3. ([\#17442](https://github.com/element-hq/synapse/issues/17442)) +* Bump zipp from 3.15.0 to 3.19.1. ([\#17427](https://github.com/element-hq/synapse/issues/17427)) + + +# Synapse 1.111.1 (2024-07-30) + +This security release is to update our locked dependency on Twisted to 24.7.0rc1, which includes a security fix for [CVE-2024-41671 / GHSA-c8m8-j448-xjx7: Disordered HTTP pipeline response in twisted.web, again](https://github.com/twisted/twisted/security/advisories/GHSA-c8m8-j448-xjx7). + +This issue means that, if multiple HTTP requests are pipelined in the same TCP connection, Synapse can send responses to the wrong HTTP request. +If a reverse proxy was configured to use HTTP pipelining, this could result in responses being sent to the wrong user, severely harming confidentiality. + +With that said, despite being a high severity issue, **we consider it unlikely that Synapse installations will be affected**. +The use of HTTP pipelining in this fashion would cause worse performance for clients (request-response latencies would be increased as users' responses would be artificially blocked behind other users' slow requests). Further, Nginx and Haproxy, two common reverse proxies, do not appear to support configuring their upstreams to use HTTP pipelining and thus would not be affected. For both of these reasons, we consider it unlikely that a Synapse deployment would be set up in such a configuration. + +Despite that, we cannot rule out that some installations may exist with this unusual setup and so we are releasing this security update today. + +**pip users:** Note that by default, upgrading Synapse using pip will not automatically upgrade Twisted. **Please manually install the new version of Twisted** using `pip install Twisted==24.7.0rc1`. Note also that even the `--upgrade-strategy=eager` flag to `pip install -U matrix-synapse` will not upgrade Twisted to a patched version because it is only a release candidate at this time. + + +### Internal Changes + +- Upgrade locked dependency on Twisted to 24.7.0rc1. ([\#17502](https://github.com/element-hq/synapse/issues/17502)) + + +# Synapse 1.111.0 (2024-07-16) + +No significant changes since 1.111.0rc2. + + + + +# Synapse 1.111.0rc2 (2024-07-10) + +### Bugfixes + +- Fix bug where using `synapse.app.media_repository` worker configuration would break the new media endpoints. ([\#17420](https://github.com/element-hq/synapse/issues/17420)) + +### Improved Documentation + +- Document the new federation media worker endpoints in the [upgrade notes](https://element-hq.github.io/synapse/v1.111/upgrade.html) and [worker docs](https://element-hq.github.io/synapse/v1.111/workers.html). ([\#17421](https://github.com/element-hq/synapse/issues/17421)) + +### Internal Changes + +- Route authenticated federation media requests to media repository workers in Complement tests. ([\#17422](https://github.com/element-hq/synapse/issues/17422)) + + + + +# Synapse 1.111.0rc1 (2024-07-09) + +### Features + +- Add `rooms` data to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17320](https://github.com/element-hq/synapse/issues/17320)) +- Add `room_types`/`not_room_types` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17337](https://github.com/element-hq/synapse/issues/17337)) +- Return "required state" in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17342](https://github.com/element-hq/synapse/issues/17342)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding [`_matrix/client/v1/media/download`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediadownloadservernamemediaid) endpoint. ([\#17365](https://github.com/element-hq/synapse/issues/17365)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/rav/authentication-for-media/proposals/3916-authentication-for-media.md) + by adding [`_matrix/client/v1/media/thumbnail`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid), [`_matrix/federation/v1/media/thumbnail`](https://spec.matrix.org/v1.11/server-server-api/#get_matrixfederationv1mediathumbnailmediaid) endpoints and stabilizing the + remaining [`_matrix/client/v1/media`](https://spec.matrix.org/v1.11/client-server-api/#get_matrixclientv1mediaconfig) endpoints. ([\#17388](https://github.com/element-hq/synapse/issues/17388)) +- Add `rooms.bump_stamp` for easier client-side sorting in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17395](https://github.com/element-hq/synapse/issues/17395)) +- Forget all of a user's rooms upon deactivation, preventing local room purges from being blocked on deactivated users. ([\#17400](https://github.com/element-hq/synapse/issues/17400)) +- Declare support for [Matrix 1.11](https://matrix.org/blog/2024/06/20/matrix-v1.11-release/). ([\#17403](https://github.com/element-hq/synapse/issues/17403)) +- [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861): allow overriding the introspection endpoint. ([\#17406](https://github.com/element-hq/synapse/issues/17406)) + +### Bugfixes + +- Fix rare race which caused no new to-device messages to be received from remote server. ([\#17362](https://github.com/element-hq/synapse/issues/17362)) +- Fix bug in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint when using an old database. ([\#17398](https://github.com/element-hq/synapse/issues/17398)) + +### Improved Documentation + +- Clarify that `url_preview_url_blacklist` is a usability feature. ([\#17356](https://github.com/element-hq/synapse/issues/17356)) +- Fix broken links in README. ([\#17379](https://github.com/element-hq/synapse/issues/17379)) +- Clarify that changelog content *and file extension* need to match in order for entries to merge. ([\#17399](https://github.com/element-hq/synapse/issues/17399)) + +### Internal Changes + +- Make the release script create a release branch for Complement as well. ([\#17318](https://github.com/element-hq/synapse/issues/17318)) +- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) +- Add CI check for the README. ([\#17367](https://github.com/element-hq/synapse/issues/17367)) +- Fix linting errors from new `ruff` version. ([\#17381](https://github.com/element-hq/synapse/issues/17381), [\#17411](https://github.com/element-hq/synapse/issues/17411)) +- Fix building debian packages on non-clean checkouts. ([\#17390](https://github.com/element-hq/synapse/issues/17390)) +- Finish up work to allow per-user feature flags. ([\#17392](https://github.com/element-hq/synapse/issues/17392), [\#17410](https://github.com/element-hq/synapse/issues/17410)) +- Allow enabling sliding sync per-user. ([\#17393](https://github.com/element-hq/synapse/issues/17393)) + + + +### Updates to locked dependencies + +* Bump certifi from 2023.7.22 to 2024.7.4. ([\#17404](https://github.com/element-hq/synapse/issues/17404)) +* Bump cryptography from 42.0.7 to 42.0.8. ([\#17382](https://github.com/element-hq/synapse/issues/17382)) +* Bump ijson from 3.2.3 to 3.3.0. ([\#17413](https://github.com/element-hq/synapse/issues/17413)) +* Bump log from 0.4.21 to 0.4.22. ([\#17384](https://github.com/element-hq/synapse/issues/17384)) +* Bump mypy-zope from 1.0.4 to 1.0.5. ([\#17414](https://github.com/element-hq/synapse/issues/17414)) +* Bump pillow from 10.3.0 to 10.4.0. ([\#17412](https://github.com/element-hq/synapse/issues/17412)) +* Bump pydantic from 2.7.1 to 2.8.2. ([\#17415](https://github.com/element-hq/synapse/issues/17415)) +* Bump ruff from 0.3.7 to 0.5.0. ([\#17381](https://github.com/element-hq/synapse/issues/17381)) +* Bump serde from 1.0.203 to 1.0.204. ([\#17409](https://github.com/element-hq/synapse/issues/17409)) +* Bump serde_json from 1.0.117 to 1.0.120. ([\#17385](https://github.com/element-hq/synapse/issues/17385), [\#17408](https://github.com/element-hq/synapse/issues/17408)) +* Bump types-setuptools from 69.5.0.20240423 to 70.1.0.20240627. ([\#17380](https://github.com/element-hq/synapse/issues/17380)) + +# Synapse 1.110.0 (2024-07-03) + +No significant changes since 1.110.0rc3. + + + + +# Synapse 1.110.0rc3 (2024-07-02) + +### Bugfixes + +- Fix bug where `/sync` requests could get blocked indefinitely after an upgrade from Synapse versions before v1.109.0. ([\#17386](https://github.com/element-hq/synapse/issues/17386), [\#17391](https://github.com/element-hq/synapse/issues/17391)) + +### Internal Changes + +- Limit size of presence EDUs to 50 entries. ([\#17371](https://github.com/element-hq/synapse/issues/17371)) +- Fix building debian package for debian sid. ([\#17389](https://github.com/element-hq/synapse/issues/17389)) + + + + +# Synapse 1.110.0rc2 (2024-06-26) + +### Internal Changes + +- Fix uploading packages to PyPi. ([\#17363](https://github.com/element-hq/synapse/issues/17363)) + + + + +# Synapse 1.110.0rc1 (2024-06-26) + +### Features + +- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17187](https://github.com/element-hq/synapse/issues/17187)) +- Add experimental support for [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823) - Account suspension. ([\#17255](https://github.com/element-hq/synapse/issues/17255)) +- Improve ratelimiting in Synapse. ([\#17256](https://github.com/element-hq/synapse/issues/17256)) +- Add support for the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) report room API. ([\#17270](https://github.com/element-hq/synapse/issues/17270), [\#17296](https://github.com/element-hq/synapse/issues/17296)) +- Filter for public and empty rooms added to Admin-API [List Room API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#list-room-api). ([\#17276](https://github.com/element-hq/synapse/issues/17276)) +- Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17277](https://github.com/element-hq/synapse/issues/17277)) +- Add `is_encrypted` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17281](https://github.com/element-hq/synapse/issues/17281)) +- Include user membership in events served to clients, per [MSC4115](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17282](https://github.com/element-hq/synapse/issues/17282)) +- Do not require user-interactive authentication for uploading cross-signing keys for the first time, per [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967). ([\#17284](https://github.com/element-hq/synapse/issues/17284)) +- Add `stream_ordering` sort to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17293](https://github.com/element-hq/synapse/issues/17293)) +- `register_new_matrix_user` now supports a --password-file flag, which + is useful for scripting. ([\#17294](https://github.com/element-hq/synapse/issues/17294)) +- `register_new_matrix_user` now supports a --exists-ok flag to allow registration of users that already exist in the database. + This is useful for scripts that bootstrap user accounts with initial passwords. ([\#17304](https://github.com/element-hq/synapse/issues/17304)) +- Add support for via query parameter from [MSC4156](https://github.com/matrix-org/matrix-spec-proposals/pull/4156). ([\#17322](https://github.com/element-hq/synapse/issues/17322)) +- Add `is_invite` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17335](https://github.com/element-hq/synapse/issues/17335)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3916-authentication-for-media.md) by adding a federation /download endpoint. ([\#17350](https://github.com/element-hq/synapse/issues/17350)) + +### Bugfixes + +- Fix searching for users with their exact localpart whose ID includes a hyphen. ([\#17254](https://github.com/element-hq/synapse/issues/17254)) +- Fix wrong retention policy being used when filtering events. ([\#17272](https://github.com/element-hq/synapse/issues/17272)) +- Fix bug where OTKs were not always included in `/sync` response when using workers. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) +- Fix a long-standing bug where an invalid 'from' parameter to [`/notifications`](https://spec.matrix.org/v1.10/client-server-api/#get_matrixclientv3notifications) would result in an Internal Server Error. ([\#17283](https://github.com/element-hq/synapse/issues/17283)) +- Fix edge case in `/sync` returning the wrong the state when using sharded event persisters. ([\#17295](https://github.com/element-hq/synapse/issues/17295)) +- Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. ([\#17301](https://github.com/element-hq/synapse/issues/17301)) +- Fix email notification subject when invited to a space. ([\#17336](https://github.com/element-hq/synapse/issues/17336)) + +### Improved Documentation + +- Add missing quotes for example for `exclude_rooms_from_sync`. ([\#17308](https://github.com/element-hq/synapse/issues/17308)) +- Update header in the README to visually fix the the auto-generated table of contents. ([\#17329](https://github.com/element-hq/synapse/issues/17329)) +- Fix stale references to the Foundation's Security Disclosure Policy. ([\#17341](https://github.com/element-hq/synapse/issues/17341)) +- Add default values for `rc_invites.per_issuer` to docs. ([\#17347](https://github.com/element-hq/synapse/issues/17347)) +- Fix an error in the docs for `search_all_users` parameter under `user_directory`. ([\#17348](https://github.com/element-hq/synapse/issues/17348)) + +### Internal Changes + +- Remove unused `expire_access_token` option in the Synapse Docker config file. Contributed by @AaronDewes. ([\#17198](https://github.com/element-hq/synapse/issues/17198)) +- Use fully-qualified `PersistedEventPosition` when returning `RoomsForUser` to facilitate proper comparisons and `RoomStreamToken` generation. ([\#17265](https://github.com/element-hq/synapse/issues/17265)) +- Add debug logging for when room keys are uploaded, including whether they are replacing other room keys. ([\#17266](https://github.com/element-hq/synapse/issues/17266)) +- Handle OTK uploads off master. ([\#17271](https://github.com/element-hq/synapse/issues/17271)) +- Don't try and resync devices for remote users whose servers are marked as down. ([\#17273](https://github.com/element-hq/synapse/issues/17273)) +- Re-organize Pydantic models and types used in handlers. ([\#17279](https://github.com/element-hq/synapse/issues/17279)) +- Expose the worker instance that persisted the event on `event.internal_metadata.instance_name`. ([\#17300](https://github.com/element-hq/synapse/issues/17300)) +- Update the README with Element branding, improve headers and fix the #synapse:matrix.org support room link rendering. ([\#17324](https://github.com/element-hq/synapse/issues/17324)) +- Change path of the experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync implementation to `/org.matrix.simplified_msc3575/sync` since our simplified API is slightly incompatible with what's in the current MSC. ([\#17331](https://github.com/element-hq/synapse/issues/17331)) +- Handle device lists notifications for large accounts more efficiently in worker mode. ([\#17333](https://github.com/element-hq/synapse/issues/17333), [\#17358](https://github.com/element-hq/synapse/issues/17358)) +- Do not block event sending/receiving while calculating large event auth chains. ([\#17338](https://github.com/element-hq/synapse/issues/17338)) +- Tidy up `parse_integer` docs and call sites to reflect the fact that they require non-negative integers by default, and bring `parse_integer_from_args` default in alignment. Contributed by Denis Kasak (@dkasak). ([\#17339](https://github.com/element-hq/synapse/issues/17339)) + + + +### Updates to locked dependencies + +* Bump authlib from 1.3.0 to 1.3.1. ([\#17343](https://github.com/element-hq/synapse/issues/17343)) +* Bump dawidd6/action-download-artifact from 3.1.4 to 5. ([\#17289](https://github.com/element-hq/synapse/issues/17289)) +* Bump dawidd6/action-download-artifact from 5 to 6. ([\#17313](https://github.com/element-hq/synapse/issues/17313)) +* Bump docker/build-push-action from 5 to 6. ([\#17312](https://github.com/element-hq/synapse/issues/17312)) +* Bump jinja2 from 3.1.3 to 3.1.4. ([\#17287](https://github.com/element-hq/synapse/issues/17287)) +* Bump lazy_static from 1.4.0 to 1.5.0. ([\#17355](https://github.com/element-hq/synapse/issues/17355)) +* Bump msgpack from 1.0.7 to 1.0.8. ([\#17317](https://github.com/element-hq/synapse/issues/17317)) +* Bump netaddr from 1.2.1 to 1.3.0. ([\#17353](https://github.com/element-hq/synapse/issues/17353)) +* Bump packaging from 24.0 to 24.1. ([\#17352](https://github.com/element-hq/synapse/issues/17352)) +* Bump phonenumbers from 8.13.37 to 8.13.39. ([\#17315](https://github.com/element-hq/synapse/issues/17315)) +* Bump regex from 1.10.4 to 1.10.5. ([\#17290](https://github.com/element-hq/synapse/issues/17290)) +* Bump requests from 2.31.0 to 2.32.2. ([\#17345](https://github.com/element-hq/synapse/issues/17345)) +* Bump sentry-sdk from 2.1.1 to 2.3.1. ([\#17263](https://github.com/element-hq/synapse/issues/17263)) +* Bump sentry-sdk from 2.3.1 to 2.6.0. ([\#17351](https://github.com/element-hq/synapse/issues/17351)) +* Bump tornado from 6.4 to 6.4.1. ([\#17344](https://github.com/element-hq/synapse/issues/17344)) +* Bump mypy from 1.8.0 to 1.9.0. ([\#17297](https://github.com/element-hq/synapse/issues/17297)) +* Bump types-jsonschema from 4.21.0.20240311 to 4.22.0.20240610. ([\#17288](https://github.com/element-hq/synapse/issues/17288)) +* Bump types-netaddr from 1.2.0.20240219 to 1.3.0.20240530. ([\#17314](https://github.com/element-hq/synapse/issues/17314)) +* Bump types-pillow from 10.2.0.20240423 to 10.2.0.20240520. ([\#17285](https://github.com/element-hq/synapse/issues/17285)) +* Bump types-pyyaml from 6.0.12.12 to 6.0.12.20240311. ([\#17316](https://github.com/element-hq/synapse/issues/17316)) +* Bump typing-extensions from 4.11.0 to 4.12.2. ([\#17354](https://github.com/element-hq/synapse/issues/17354)) +* Bump urllib3 from 2.0.7 to 2.2.2. ([\#17346](https://github.com/element-hq/synapse/issues/17346)) + +# Synapse 1.109.0 (2024-06-18) + +### Internal Changes + +- Fix the building of binary wheels for macOS by switching to macOS 12 CI runners. ([\#17319](https://github.com/element-hq/synapse/issues/17319)) + + + + +# Synapse 1.109.0rc3 (2024-06-17) + +### Bugfixes + +- When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL. ([\#17305](https://github.com/element-hq/synapse/issues/17305), [\#17309](https://github.com/element-hq/synapse/issues/17309)) + +### Internal Changes + +- Use the release branch for sytest in release-branch PRs. ([\#17306](https://github.com/element-hq/synapse/issues/17306)) + + + + +# Synapse 1.109.0rc2 (2024-06-11) + +### Bugfixes + +- Fix bug where one-time-keys were not always included in `/sync` response when using workers. Introduced in v1.109.0rc1. ([\#17275](https://github.com/element-hq/synapse/issues/17275)) +- Fix bug where `/sync` could get stuck due to edge case in device lists handling. Introduced in v1.109.0rc1. ([\#17292](https://github.com/element-hq/synapse/issues/17292)) + + + + +# Synapse 1.109.0rc1 (2024-06-04) + +### Features + +- Add the ability to auto-accept invites on the behalf of users. See the [`auto_accept_invites`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#auto-accept-invites) config option for details. ([\#17147](https://github.com/element-hq/synapse/issues/17147)) +- Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for to-device messages and device encryption info. ([\#17167](https://github.com/element-hq/synapse/issues/17167)) +- Support [MSC3916](https://github.com/matrix-org/matrix-spec-proposals/issues/3916) by adding unstable media endpoints to `/_matrix/client`. ([\#17213](https://github.com/element-hq/synapse/issues/17213)) +- Add logging to tasks managed by the task scheduler, showing CPU and database usage. ([\#17219](https://github.com/element-hq/synapse/issues/17219)) + +### Bugfixes + +- Fix deduplicating of membership events to not create unused state groups. ([\#17164](https://github.com/element-hq/synapse/issues/17164)) +- Fix bug where duplicate events could be sent down sync when using workers that are overloaded. ([\#17215](https://github.com/element-hq/synapse/issues/17215)) +- Ignore attempts to send to-device messages to bad users, to avoid log spam when we try to connect to the bad server. ([\#17240](https://github.com/element-hq/synapse/issues/17240)) +- Fix handling of duplicate concurrent uploading of device one-time-keys. ([\#17241](https://github.com/element-hq/synapse/issues/17241)) +- Fix reporting of default tags to Sentry, such as worker name. Broke in v1.108.0. ([\#17251](https://github.com/element-hq/synapse/issues/17251)) +- Fix bug where typing updates would not be sent when using workers after a restart. ([\#17252](https://github.com/element-hq/synapse/issues/17252)) + +### Improved Documentation + +- Update the LemonLDAP documentation to say that claims should be explicitly included in the returned `id_token`, as Synapse won't request them. ([\#17204](https://github.com/element-hq/synapse/issues/17204)) + +### Internal Changes + +- Improve DB usage when fetching related events. ([\#17083](https://github.com/element-hq/synapse/issues/17083)) +- Log exceptions when failing to auto-join new user according to the `auto_join_rooms` option. ([\#17176](https://github.com/element-hq/synapse/issues/17176)) +- Reduce work of calculating outbound device lists updates. ([\#17211](https://github.com/element-hq/synapse/issues/17211)) +- Improve performance of calculating device lists changes in `/sync`. ([\#17216](https://github.com/element-hq/synapse/issues/17216)) +- Move towards using `MultiWriterIdGenerator` everywhere. ([\#17226](https://github.com/element-hq/synapse/issues/17226)) +- Replaces all usages of `StreamIdGenerator` with `MultiWriterIdGenerator`. ([\#17229](https://github.com/element-hq/synapse/issues/17229)) +- Change the `allow_unsafe_locale` config option to also apply when setting up new databases. ([\#17238](https://github.com/element-hq/synapse/issues/17238)) +- Fix errors in logs about closing incorrect logging contexts when media gets rejected by a module. ([\#17239](https://github.com/element-hq/synapse/issues/17239), [\#17246](https://github.com/element-hq/synapse/issues/17246)) +- Clean out invalid destinations from `device_federation_outbox` table. ([\#17242](https://github.com/element-hq/synapse/issues/17242)) +- Stop logging errors when receiving invalid User IDs in key querys requests. ([\#17250](https://github.com/element-hq/synapse/issues/17250)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.83 to 1.0.86. ([\#17220](https://github.com/element-hq/synapse/issues/17220)) +* Bump bcrypt from 4.1.2 to 4.1.3. ([\#17224](https://github.com/element-hq/synapse/issues/17224)) +* Bump lxml from 5.2.1 to 5.2.2. ([\#17261](https://github.com/element-hq/synapse/issues/17261)) +* Bump mypy-zope from 1.0.3 to 1.0.4. ([\#17262](https://github.com/element-hq/synapse/issues/17262)) +* Bump phonenumbers from 8.13.35 to 8.13.37. ([\#17235](https://github.com/element-hq/synapse/issues/17235)) +* Bump prometheus-client from 0.19.0 to 0.20.0. ([\#17233](https://github.com/element-hq/synapse/issues/17233)) +* Bump pyasn1 from 0.5.1 to 0.6.0. ([\#17223](https://github.com/element-hq/synapse/issues/17223)) +* Bump pyicu from 2.13 to 2.13.1. ([\#17236](https://github.com/element-hq/synapse/issues/17236)) +* Bump pyopenssl from 24.0.0 to 24.1.0. ([\#17234](https://github.com/element-hq/synapse/issues/17234)) +* Bump serde from 1.0.201 to 1.0.202. ([\#17221](https://github.com/element-hq/synapse/issues/17221)) +* Bump serde from 1.0.202 to 1.0.203. ([\#17232](https://github.com/element-hq/synapse/issues/17232)) +* Bump twine from 5.0.0 to 5.1.0. ([\#17225](https://github.com/element-hq/synapse/issues/17225)) +* Bump types-psycopg2 from 2.9.21.20240311 to 2.9.21.20240417. ([\#17222](https://github.com/element-hq/synapse/issues/17222)) +* Bump types-pyopenssl from 24.0.0.20240311 to 24.1.0.20240425. ([\#17260](https://github.com/element-hq/synapse/issues/17260)) + +# Synapse 1.108.0 (2024-05-28) + +No significant changes since 1.108.0rc1. + + + + +# Synapse 1.108.0rc1 (2024-05-21) + +### Features + +- Add a feature that allows clients to query the configured federation whitelist. Disabled by default. ([\#16848](https://github.com/element-hq/synapse/issues/16848), [\#17199](https://github.com/element-hq/synapse/issues/17199)) +- Add the ability to allow numeric user IDs with a specific prefix when in the CAS flow. Contributed by Aurélien Grimpard. ([\#17098](https://github.com/element-hq/synapse/issues/17098)) + +### Bugfixes + +- Fix bug where push rules would be empty in `/sync` for some accounts. Introduced in v1.93.0. ([\#17142](https://github.com/element-hq/synapse/issues/17142)) +- Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas. ([\#17145](https://github.com/element-hq/synapse/issues/17145)) +- Fix bug where disabling room publication prevented public rooms being created on workers. ([\#17177](https://github.com/element-hq/synapse/issues/17177), [\#17184](https://github.com/element-hq/synapse/issues/17184)) + +### Improved Documentation + +- Document [`/v1/make_knock`](https://spec.matrix.org/v1.10/server-server-api/#get_matrixfederationv1make_knockroomiduserid) and [`/v1/send_knock/`](https://spec.matrix.org/v1.10/server-server-api/#put_matrixfederationv1send_knockroomideventid) federation endpoints as worker-compatible. ([\#17058](https://github.com/element-hq/synapse/issues/17058)) +- Update User Admin API with note about prefixing OIDC external_id providers. ([\#17139](https://github.com/element-hq/synapse/issues/17139)) +- Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option. ([\#17150](https://github.com/element-hq/synapse/issues/17150)) +- Update the Admin FAQ with the current libjemalloc version for latest Debian stable. Additionally update the name of the "push_rules" stream in the Workers documentation. ([\#17171](https://github.com/element-hq/synapse/issues/17171)) + +### Internal Changes + +- Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will remain supported for some time. ([\#17151](https://github.com/element-hq/synapse/issues/17151)) +- Update dependency PyO3 to 0.21. ([\#17162](https://github.com/element-hq/synapse/issues/17162)) +- Fixes linter errors found in PR #17147. ([\#17166](https://github.com/element-hq/synapse/issues/17166)) +- Bump black from 24.2.0 to 24.4.2. ([\#17170](https://github.com/element-hq/synapse/issues/17170)) +- Cache literal sync filter validation for performance. ([\#17186](https://github.com/element-hq/synapse/issues/17186)) +- Improve performance by fixing a reactor pause. ([\#17192](https://github.com/element-hq/synapse/issues/17192)) +- Route `/make_knock` and `/send_knock` federation APIs to the federation reader worker in Complement test runs. ([\#17195](https://github.com/element-hq/synapse/issues/17195)) +- Prepare sync handler to be able to return different sync responses (`SyncVersion`). ([\#17200](https://github.com/element-hq/synapse/issues/17200)) +- Organize the sync cache key parameter outside of the sync config (separate concerns). ([\#17201](https://github.com/element-hq/synapse/issues/17201)) +- Refactor `SyncResultBuilder` assembly to its own function. ([\#17202](https://github.com/element-hq/synapse/issues/17202)) +- Rename to be obvious: `joined_rooms` -> `joined_room_ids`. ([\#17203](https://github.com/element-hq/synapse/issues/17203), [\#17208](https://github.com/element-hq/synapse/issues/17208)) +- Add a short pause when rate-limiting a request. ([\#17210](https://github.com/element-hq/synapse/issues/17210)) + + + +### Updates to locked dependencies + +* Bump cryptography from 42.0.5 to 42.0.7. ([\#17180](https://github.com/element-hq/synapse/issues/17180)) +* Bump gitpython from 3.1.41 to 3.1.43. ([\#17181](https://github.com/element-hq/synapse/issues/17181)) +* Bump immutabledict from 4.1.0 to 4.2.0. ([\#17179](https://github.com/element-hq/synapse/issues/17179)) +* Bump sentry-sdk from 1.40.3 to 2.1.1. ([\#17178](https://github.com/element-hq/synapse/issues/17178)) +* Bump serde from 1.0.200 to 1.0.201. ([\#17183](https://github.com/element-hq/synapse/issues/17183)) +* Bump serde_json from 1.0.116 to 1.0.117. ([\#17182](https://github.com/element-hq/synapse/issues/17182)) + +Synapse 1.107.0 (2024-05-14) +============================ + +No significant changes since 1.107.0rc1. + + +# Synapse 1.107.0rc1 (2024-05-07) + +### Features + +- Add preliminary support for [MSC3823: Account Suspension](https://github.com/matrix-org/matrix-spec-proposals/pull/3823). ([\#17051](https://github.com/element-hq/synapse/issues/17051)) +- Declare support for [Matrix v1.10](https://matrix.org/blog/2024/03/22/matrix-v1.10-release/). Contributed by @clokep. ([\#17082](https://github.com/element-hq/synapse/issues/17082)) +- Add support for [MSC4115: membership metadata on events](https://github.com/matrix-org/matrix-spec-proposals/pull/4115). ([\#17104](https://github.com/element-hq/synapse/issues/17104), [\#17137](https://github.com/element-hq/synapse/issues/17137)) + +### Bugfixes + +- Fixed search feature of Element Android on homesevers using SQLite by returning search terms as search highlights. ([\#17000](https://github.com/element-hq/synapse/issues/17000)) +- Fixes a bug introduced in v1.52.0 where the `destination` query parameter for the [Destination Rooms Admin API](https://element-hq.github.io/synapse/v1.105/usage/administration/admin_api/federation.html#destination-rooms) failed to actually filter returned rooms. ([\#17077](https://github.com/element-hq/synapse/issues/17077)) +- For MSC3266 room summaries, support queries at the recommended endpoint of `/_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}`. The existing endpoint of `/_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` is deprecated. ([\#17078](https://github.com/element-hq/synapse/issues/17078)) +- Apply user email & picture during OIDC registration if present & selected. ([\#17120](https://github.com/element-hq/synapse/issues/17120)) +- Improve error message for cross signing reset with [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) enabled. ([\#17121](https://github.com/element-hq/synapse/issues/17121)) +- Fix a bug which meant that to-device messages received over federation could be dropped when the server was under load or networking problems caused problems between Synapse processes or the database. ([\#17127](https://github.com/element-hq/synapse/issues/17127)) +- Fix bug where `StreamChangeCache` would not respect configured cache factors. ([\#17152](https://github.com/element-hq/synapse/issues/17152)) + +### Updates to the Docker image + +- Correct licensing metadata on Docker image. ([\#17141](https://github.com/element-hq/synapse/issues/17141)) + +### Improved Documentation + +- Update the `event_cache_size` and `global_factor` configuration options' documentation. ([\#17071](https://github.com/element-hq/synapse/issues/17071)) +- Remove broken sphinx docs. ([\#17073](https://github.com/element-hq/synapse/issues/17073), [\#17148](https://github.com/element-hq/synapse/issues/17148)) +- Add RuntimeDirectory to example matrix-synapse.service systemd unit. ([\#17084](https://github.com/element-hq/synapse/issues/17084)) +- Fix various small typos throughout the docs. ([\#17114](https://github.com/element-hq/synapse/issues/17114)) +- Update enable_notifs configuration documentation. ([\#17116](https://github.com/element-hq/synapse/issues/17116)) +- Update the Upgrade Notes with the latest minimum supported Rust version of 1.66.0. Contributed by @jahway603. ([\#17140](https://github.com/element-hq/synapse/issues/17140)) + +### Internal Changes + +- Enable [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) by default in the Synapse Complement image. ([\#17105](https://github.com/element-hq/synapse/issues/17105)) +- Add optimisation to `StreamChangeCache.get_entities_changed(..)`. ([\#17130](https://github.com/element-hq/synapse/issues/17130)) + + + +### Updates to locked dependencies + +* Bump furo from 2024.1.29 to 2024.4.27. ([\#17133](https://github.com/element-hq/synapse/issues/17133)) +* Bump idna from 3.6 to 3.7. ([\#17136](https://github.com/element-hq/synapse/issues/17136)) +* Bump jsonschema from 4.21.1 to 4.22.0. ([\#17157](https://github.com/element-hq/synapse/issues/17157)) +* Bump lxml from 5.1.0 to 5.2.1. ([\#17158](https://github.com/element-hq/synapse/issues/17158)) +* Bump phonenumbers from 8.13.29 to 8.13.35. ([\#17106](https://github.com/element-hq/synapse/issues/17106)) +- Bump pillow from 10.2.0 to 10.3.0. ([\#17146](https://github.com/element-hq/synapse/issues/17146)) +* Bump pydantic from 2.6.4 to 2.7.0. ([\#17107](https://github.com/element-hq/synapse/issues/17107)) +* Bump pydantic from 2.7.0 to 2.7.1. ([\#17160](https://github.com/element-hq/synapse/issues/17160)) +* Bump pyicu from 2.12 to 2.13. ([\#17109](https://github.com/element-hq/synapse/issues/17109)) +* Bump serde from 1.0.197 to 1.0.198. ([\#17111](https://github.com/element-hq/synapse/issues/17111)) +* Bump serde from 1.0.198 to 1.0.199. ([\#17132](https://github.com/element-hq/synapse/issues/17132)) +* Bump serde from 1.0.199 to 1.0.200. ([\#17161](https://github.com/element-hq/synapse/issues/17161)) +* Bump serde_json from 1.0.115 to 1.0.116. ([\#17112](https://github.com/element-hq/synapse/issues/17112)) +- Update `tornado` Python dependency from 6.2 to 6.4. ([\#17131](https://github.com/element-hq/synapse/issues/17131)) +* Bump twisted from 23.10.0 to 24.3.0. ([\#17135](https://github.com/element-hq/synapse/issues/17135)) +* Bump types-bleach from 6.1.0.1 to 6.1.0.20240331. ([\#17110](https://github.com/element-hq/synapse/issues/17110)) +* Bump types-pillow from 10.2.0.20240415 to 10.2.0.20240423. ([\#17159](https://github.com/element-hq/synapse/issues/17159)) +* Bump types-setuptools from 69.0.0.20240125 to 69.5.0.20240423. ([\#17134](https://github.com/element-hq/synapse/issues/17134)) + +# Synapse 1.106.0 (2024-04-30) + +No significant changes since 1.106.0rc1. + + + + +# Synapse 1.106.0rc1 (2024-04-25) + +### Features + +- Send an email if the address is already bound to an user account. ([\#16819](https://github.com/element-hq/synapse/issues/16819)) +- Implement the rendezvous mechanism described by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108). ([\#17056](https://github.com/element-hq/synapse/issues/17056)) +- Support delegating the rendezvous mechanism described [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/issues/4108) to an external implementation. ([\#17086](https://github.com/element-hq/synapse/issues/17086)) + +### Bugfixes + +- Add validation to ensure that the `limit` parameter on `/publicRooms` is non-negative. ([\#16920](https://github.com/element-hq/synapse/issues/16920)) +- Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error. ([\#16923](https://github.com/element-hq/synapse/issues/16923)) +- Make the CSAPI endpoint `/keys/device_signing/upload` idempotent. ([\#16943](https://github.com/element-hq/synapse/issues/16943)) +- Redact membership events if the user requested erasure upon deactivating. ([\#17076](https://github.com/element-hq/synapse/issues/17076)) + +### Improved Documentation + +- Add a prompt in the contributing guide to manually configure icu4c. ([\#17069](https://github.com/element-hq/synapse/issues/17069)) +- Clarify what part of message retention is still experimental. ([\#17099](https://github.com/element-hq/synapse/issues/17099)) + +### Internal Changes + +- Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar). ([\#17032](https://github.com/element-hq/synapse/issues/17032), [\#17096](https://github.com/element-hq/synapse/issues/17096)) +- Fix mypy with latest Twisted release. ([\#17036](https://github.com/element-hq/synapse/issues/17036)) +- Bump minimum supported Rust version to 1.66.0. ([\#17079](https://github.com/element-hq/synapse/issues/17079)) +- Add helpers to transform Twisted requests to Rust http Requests/Responses. ([\#17081](https://github.com/element-hq/synapse/issues/17081)) +- Fix type annotation for `visited_chains` after `mypy` upgrade. ([\#17125](https://github.com/element-hq/synapse/issues/17125)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.81 to 1.0.82. ([\#17095](https://github.com/element-hq/synapse/issues/17095)) +* Bump peaceiris/actions-gh-pages from 3.9.3 to 4.0.0. ([\#17087](https://github.com/element-hq/synapse/issues/17087)) +* Bump peaceiris/actions-mdbook from 1.2.0 to 2.0.0. ([\#17089](https://github.com/element-hq/synapse/issues/17089)) +* Bump pyasn1-modules from 0.3.0 to 0.4.0. ([\#17093](https://github.com/element-hq/synapse/issues/17093)) +* Bump pygithub from 2.2.0 to 2.3.0. ([\#17092](https://github.com/element-hq/synapse/issues/17092)) +* Bump ruff from 0.3.5 to 0.3.7. ([\#17094](https://github.com/element-hq/synapse/issues/17094)) +* Bump sigstore/cosign-installer from 3.4.0 to 3.5.0. ([\#17088](https://github.com/element-hq/synapse/issues/17088)) +* Bump twine from 4.0.2 to 5.0.0. ([\#17091](https://github.com/element-hq/synapse/issues/17091)) +* Bump types-pillow from 10.2.0.20240406 to 10.2.0.20240415. ([\#17090](https://github.com/element-hq/synapse/issues/17090)) + +# Synapse 1.105.1 (2024-04-23) + +## Security advisory + +The following issues are fixed in 1.105.1. + +- [GHSA-3h7q-rfh9-xm4v](https://github.com/element-hq/synapse/security/advisories/GHSA-3h7q-rfh9-xm4v) / [CVE-2024-31208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-31208) — High Severity + + Weakness in auth chain indexing allows DoS from remote room members through disk fill and high CPU usage. + +See the advisories for more details. If you have any questions, email security@element.io. + + + +# Synapse 1.105.0 (2024-04-16) + +No significant changes since 1.105.0rc1. + + + + +# Synapse 1.105.0rc1 (2024-04-11) + +### Features + +- Stabilize support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which clarifies the interaction of push rules and account data. Contributed by @clokep. ([\#17022](https://github.com/element-hq/synapse/issues/17022)) +- Stabilize support for [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981): `/relations` recursion. Contributed by @clokep. ([\#17023](https://github.com/element-hq/synapse/issues/17023)) +- Add support for moving `/pushrules` off of main process. ([\#17037](https://github.com/element-hq/synapse/issues/17037), [\#17038](https://github.com/element-hq/synapse/issues/17038)) + +### Bugfixes + +- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16930](https://github.com/element-hq/synapse/issues/16930), [\#16932](https://github.com/element-hq/synapse/issues/16932), [\#16942](https://github.com/element-hq/synapse/issues/16942), [\#17064](https://github.com/element-hq/synapse/issues/17064), [\#17065](https://github.com/element-hq/synapse/issues/17065), [\#17066](https://github.com/element-hq/synapse/issues/17066)) +- Fix server notice rooms not always being created as unencrypted rooms, even when `encryption_enabled_by_default_for_room_type` is in use (server notices are always unencrypted). ([\#17033](https://github.com/element-hq/synapse/issues/17033)) +- Fix the `.m.rule.encrypted_room_one_to_one` and `.m.rule.room_one_to_one` default underride push rules being in the wrong order. Contributed by @Sumpy1. ([\#17043](https://github.com/element-hq/synapse/issues/17043)) + +### Internal Changes + +- Refactor auth chain fetching to reduce duplication. ([\#17044](https://github.com/element-hq/synapse/issues/17044)) +- Improve database performance by adding a missing index to `access_tokens.refresh_token_id`. ([\#17045](https://github.com/element-hq/synapse/issues/17045), [\#17054](https://github.com/element-hq/synapse/issues/17054)) +- Improve database performance by reducing number of receipts fetched when sending push notifications. ([\#17049](https://github.com/element-hq/synapse/issues/17049)) + + + +### Updates to locked dependencies + +* Bump packaging from 23.2 to 24.0. ([\#17027](https://github.com/element-hq/synapse/issues/17027)) +* Bump regex from 1.10.3 to 1.10.4. ([\#17028](https://github.com/element-hq/synapse/issues/17028)) +* Bump ruff from 0.3.2 to 0.3.5. ([\#17060](https://github.com/element-hq/synapse/issues/17060)) +* Bump serde_json from 1.0.114 to 1.0.115. ([\#17041](https://github.com/element-hq/synapse/issues/17041)) +* Bump types-pillow from 10.2.0.20240125 to 10.2.0.20240406. ([\#17061](https://github.com/element-hq/synapse/issues/17061)) +* Bump types-requests from 2.31.0.20240125 to 2.31.0.20240406. ([\#17063](https://github.com/element-hq/synapse/issues/17063)) +* Bump typing-extensions from 4.9.0 to 4.11.0. ([\#17062](https://github.com/element-hq/synapse/issues/17062)) + +# Synapse 1.104.0 (2024-04-02) + +### Bugfixes + +- Fix regression when using OIDC provider. Introduced in v1.104.0rc1. ([\#17031](https://github.com/element-hq/synapse/issues/17031)) + + +# Synapse 1.104.0rc1 (2024-03-26) + +### Features + +- Add an OIDC config to specify extra parameters for the authorization grant URL. IT can be useful to pass an ACR value for example. ([\#16971](https://github.com/element-hq/synapse/issues/16971)) +- Add support for OIDC provider returning JWT. ([\#16972](https://github.com/element-hq/synapse/issues/16972), [\#17031](https://github.com/element-hq/synapse/issues/17031)) + +### Bugfixes + +- Fix a bug which meant that, under certain circumstances, we might never retry sending events or to-device messages over federation after a failure. ([\#16925](https://github.com/element-hq/synapse/issues/16925)) +- Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16949](https://github.com/element-hq/synapse/issues/16949)) +- Fix case in which `m.fully_read` marker would not get updated. Contributed by @SpiritCroc. ([\#16990](https://github.com/element-hq/synapse/issues/16990)) +- Fix bug which did not retract a user's pending knocks at rooms when their account was deactivated. Contributed by @hanadi92. ([\#17010](https://github.com/element-hq/synapse/issues/17010)) + +### Updates to the Docker image + +- Updated `start.py` to generate config using the correct user ID when running as root (fixes [\#16824](https://github.com/element-hq/synapse/issues/16824), [\#15202](https://github.com/element-hq/synapse/issues/15202)). ([\#16978](https://github.com/element-hq/synapse/issues/16978)) + +### Improved Documentation + +- Add a query to force a refresh of a remote user's device list to the "Useful SQL for Admins" documentation page. ([\#16892](https://github.com/element-hq/synapse/issues/16892)) +- Minor grammatical corrections to the upgrade documentation. ([\#16965](https://github.com/element-hq/synapse/issues/16965)) +- Fix the sort order for the documentation version picker, so that newer releases appear above older ones. ([\#16966](https://github.com/element-hq/synapse/issues/16966)) +- Remove recommendation for a specific poetry version from contributing guide. ([\#17002](https://github.com/element-hq/synapse/issues/17002)) + +### Internal Changes + +- Improve lock performance when a lot of locks are all waiting for a single lock to be released. ([\#16840](https://github.com/element-hq/synapse/issues/16840)) +- Update power level default for public rooms. ([\#16907](https://github.com/element-hq/synapse/issues/16907)) +- Improve event validation. ([\#16908](https://github.com/element-hq/synapse/issues/16908)) +- Multi-worker-docker-container: disable log buffering. ([\#16919](https://github.com/element-hq/synapse/issues/16919)) +- Refactor state delta calculation in `/sync` handler. ([\#16929](https://github.com/element-hq/synapse/issues/16929)) +- Clarify docs for some room state functions. ([\#16950](https://github.com/element-hq/synapse/issues/16950)) +- Specify IP subnets in canonical form. ([\#16953](https://github.com/element-hq/synapse/issues/16953)) +- As done for SAML mapping provider, let's pass the module API to the OIDC one so the mapper can do more logic in its code. ([\#16974](https://github.com/element-hq/synapse/issues/16974)) +- Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. ([\#16985](https://github.com/element-hq/synapse/issues/16985)) +- Raise poetry-core version cap to 1.9.0. ([\#16986](https://github.com/element-hq/synapse/issues/16986)) +- Patch the db conn pool sooner in tests. ([\#17017](https://github.com/element-hq/synapse/issues/17017)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.80 to 1.0.81. ([\#17009](https://github.com/element-hq/synapse/issues/17009)) +* Bump black from 23.10.1 to 24.2.0. ([\#16936](https://github.com/element-hq/synapse/issues/16936)) +* Bump cryptography from 41.0.7 to 42.0.5. ([\#16958](https://github.com/element-hq/synapse/issues/16958)) +* Bump dawidd6/action-download-artifact from 3.1.1 to 3.1.2. ([\#16960](https://github.com/element-hq/synapse/issues/16960)) +* Bump dawidd6/action-download-artifact from 3.1.2 to 3.1.4. ([\#17008](https://github.com/element-hq/synapse/issues/17008)) +* Bump jinja2 from 3.1.2 to 3.1.3. ([\#17005](https://github.com/element-hq/synapse/issues/17005)) +* Bump log from 0.4.20 to 0.4.21. ([\#16977](https://github.com/element-hq/synapse/issues/16977)) +* Bump mypy from 1.5.1 to 1.8.0. ([\#16901](https://github.com/element-hq/synapse/issues/16901)) +* Bump netaddr from 0.9.0 to 1.2.1. ([\#17006](https://github.com/element-hq/synapse/issues/17006)) +* Bump pydantic from 2.6.0 to 2.6.4. ([\#17004](https://github.com/element-hq/synapse/issues/17004)) +* Bump pyo3 from 0.20.2 to 0.20.3. ([\#16962](https://github.com/element-hq/synapse/issues/16962)) +* Bump ruff from 0.1.14 to 0.3.2. ([\#16994](https://github.com/element-hq/synapse/issues/16994)) +* Bump serde from 1.0.196 to 1.0.197. ([\#16963](https://github.com/element-hq/synapse/issues/16963)) +* Bump serde_json from 1.0.113 to 1.0.114. ([\#16961](https://github.com/element-hq/synapse/issues/16961)) +* Bump types-jsonschema from 4.21.0.20240118 to 4.21.0.20240311. ([\#17007](https://github.com/element-hq/synapse/issues/17007)) +* Bump types-psycopg2 from 2.9.21.16 to 2.9.21.20240311. ([\#16995](https://github.com/element-hq/synapse/issues/16995)) +* Bump types-pyopenssl from 23.3.0.0 to 24.0.0.20240311. ([\#17003](https://github.com/element-hq/synapse/issues/17003)) + +# Synapse 1.103.0 (2024-03-19) + +No significant changes since 1.103.0rc1. + + + + +# Synapse 1.103.0rc1 (2024-03-12) + +### Features + +- Add a new [List Accounts v3](https://element-hq.github.io/synapse/v1.103/admin_api/user_admin_api.html#list-accounts-v3) Admin API with improved deactivated user filtering capabilities. ([\#16874](https://github.com/element-hq/synapse/issues/16874)) +- Include `Retry-After` header by default per [MSC4041](https://github.com/matrix-org/matrix-spec-proposals/pull/4041). Contributed by @clokep. ([\#16947](https://github.com/element-hq/synapse/issues/16947)) + +### Bugfixes + +- Fix joining remote rooms when a module uses the `on_new_event` callback. This callback may now pass partial state events instead of the full state for remote rooms. Introduced in v1.76.0. ([\#16973](https://github.com/element-hq/synapse/issues/16973)) +- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. Contributed by @ggogel. ([\#16968](https://github.com/element-hq/synapse/issues/16968)) + +### Improved Documentation + +- Add HAProxy example for single port operation to reverse proxy documentation. Contributed by Georg Pfuetzenreuter (@tacerus). ([\#16768](https://github.com/element-hq/synapse/issues/16768)) +- Improve the documentation around running Complement tests with new configuration parameters. ([\#16946](https://github.com/element-hq/synapse/issues/16946)) +- Add docs on upgrading from a very old version. ([\#16951](https://github.com/element-hq/synapse/issues/16951)) + + +### Updates to locked dependencies + +* Bump JasonEtco/create-an-issue from 2.9.1 to 2.9.2. ([\#16934](https://github.com/element-hq/synapse/issues/16934)) +* Bump anyhow from 1.0.79 to 1.0.80. ([\#16935](https://github.com/element-hq/synapse/issues/16935)) +* Bump dawidd6/action-download-artifact from 3.0.0 to 3.1.1. ([\#16933](https://github.com/element-hq/synapse/issues/16933)) +* Bump furo from 2023.9.10 to 2024.1.29. ([\#16939](https://github.com/element-hq/synapse/issues/16939)) +* Bump pyopenssl from 23.3.0 to 24.0.0. ([\#16937](https://github.com/element-hq/synapse/issues/16937)) +* Bump types-netaddr from 0.10.0.20240106 to 1.2.0.20240219. ([\#16938](https://github.com/element-hq/synapse/issues/16938)) + + +# Synapse 1.102.0 (2024-03-05) + +### Bugfixes + +- Revert https://github.com/element-hq/synapse/pull/16756, which caused incorrect notification counts on mobile clients since v1.100.0. ([\#16979](https://github.com/element-hq/synapse/issues/16979)) + + +# Synapse 1.102.0rc1 (2024-02-20) + +### Features + +- A metric was added for emails sent by Synapse, broken down by type: `synapse_emails_sent_total`. Contributed by Remi Rampin. ([\#16881](https://github.com/element-hq/synapse/issues/16881)) + +### Bugfixes + +- Do not send multiple concurrent requests for keys for the same server. ([\#16894](https://github.com/element-hq/synapse/issues/16894)) +- Fix performance issue when joining very large rooms that can cause the server to lock up. Introduced in v1.100.0. ([\#16903](https://github.com/element-hq/synapse/issues/16903)) +- Always prefer unthreaded receipt when >1 exist ([MSC4102](https://github.com/matrix-org/matrix-spec-proposals/pull/4102)). ([\#16927](https://github.com/element-hq/synapse/issues/16927)) + +### Improved Documentation + +- Fix a small typo in the Rooms section of the Admin API documentation. Contributed by @RainerZufall187. ([\#16857](https://github.com/element-hq/synapse/issues/16857)) + +### Internal Changes + +- Don't invalidate the entire event cache when we purge history. ([\#16905](https://github.com/element-hq/synapse/issues/16905)) +- Add experimental config option to not send device list updates for specific users. ([\#16909](https://github.com/element-hq/synapse/issues/16909)) +- Fix incorrect docker hub link in release script. ([\#16910](https://github.com/element-hq/synapse/issues/16910)) + + + +### Updates to locked dependencies + +* Bump attrs from 23.1.0 to 23.2.0. ([\#16899](https://github.com/element-hq/synapse/issues/16899)) +* Bump bcrypt from 4.0.1 to 4.1.2. ([\#16900](https://github.com/element-hq/synapse/issues/16900)) +* Bump pygithub from 2.1.1 to 2.2.0. ([\#16902](https://github.com/element-hq/synapse/issues/16902)) +* Bump sentry-sdk from 1.40.0 to 1.40.3. ([\#16898](https://github.com/element-hq/synapse/issues/16898)) + +# Synapse 1.101.0 (2024-02-13) + +### Bugfixes + +- Fix performance regression when fetching auth chains from the DB. Introduced in v1.100.0. ([\#16893](https://github.com/element-hq/synapse/issues/16893)) + + + + +# Synapse 1.101.0rc1 (2024-02-06) + +### Improved Documentation + +- Fix broken links in the documentation. ([\#16853](https://github.com/element-hq/synapse/issues/16853)) +- Update MacOS installation instructions to mention that libicu is optional. ([\#16854](https://github.com/element-hq/synapse/issues/16854)) +- The version picker now correctly lists versions after `v1.98.0`. ([\#16880](https://github.com/element-hq/synapse/issues/16880)) + +### Internal Changes + +- Add support for stabilised [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981) that adds a `recurse` parameter on the `/relations` API. ([\#16842](https://github.com/element-hq/synapse/issues/16842)) + + + +### Updates to locked dependencies + +* Bump dorny/paths-filter from 2 to 3. ([\#16869](https://github.com/element-hq/synapse/issues/16869)) +* Bump gitpython from 3.1.40 to 3.1.41. ([\#16850](https://github.com/element-hq/synapse/issues/16850)) +* Bump hiredis from 2.2.3 to 2.3.2. ([\#16862](https://github.com/element-hq/synapse/issues/16862)) +* Bump jsonschema from 4.20.0 to 4.21.1. ([\#16887](https://github.com/element-hq/synapse/issues/16887)) +* Bump lxml-stubs from 0.4.0 to 0.5.1. ([\#16885](https://github.com/element-hq/synapse/issues/16885)) +* Bump mypy-zope from 1.0.1 to 1.0.3. ([\#16865](https://github.com/element-hq/synapse/issues/16865)) +* Bump phonenumbers from 8.13.26 to 8.13.29. ([\#16868](https://github.com/element-hq/synapse/issues/16868)) +* Bump pydantic from 2.5.3 to 2.6.0. ([\#16888](https://github.com/element-hq/synapse/issues/16888)) +* Bump sentry-sdk from 1.39.1 to 1.40.0. ([\#16889](https://github.com/element-hq/synapse/issues/16889)) +* Bump serde from 1.0.195 to 1.0.196. ([\#16867](https://github.com/element-hq/synapse/issues/16867)) +* Bump serde_json from 1.0.111 to 1.0.113. ([\#16866](https://github.com/element-hq/synapse/issues/16866)) +* Bump sigstore/cosign-installer from 3.3.0 to 3.4.0. ([\#16890](https://github.com/element-hq/synapse/issues/16890)) +* Bump types-pillow from 10.1.0.2 to 10.2.0.20240125. ([\#16864](https://github.com/element-hq/synapse/issues/16864)) +* Bump types-requests from 2.31.0.10 to 2.31.0.20240125. ([\#16886](https://github.com/element-hq/synapse/issues/16886)) +* Bump types-setuptools from 69.0.0.0 to 69.0.0.20240125. ([\#16863](https://github.com/element-hq/synapse/issues/16863)) + +# Synapse 1.100.0 (2024-01-30) + +No significant changes since 1.100.0rc3. + + + + +# Synapse 1.100.0rc3 (2024-01-24) + +### Bugfixes + +- Fix database performance regression due to changing Postgres table statistics. Introduced in v1.100.0rc1. ([\#16849](https://github.com/element-hq/synapse/issues/16849)) + + + + +# Synapse 1.100.0rc2 (2024-01-24) + +This version is the same as 1.100.0rc1 but with fixes to the release process. + +### Internal Changes + +- Downgrade the `download-artifact` and `upload-artifact` actions to v3 due to breaking changes. ([\#16847](https://github.com/element-hq/synapse/issues/16847)) + + +# Synapse 1.100.0rc1 (2024-01-23) + +*This version was never released to PyPI or the Debian repository due to failures in the automatic part of the release process.* + +### Features + +- Advertise experimental support for [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028) through `/_matrix/clients/versions` if enabled. Contributed by @hanadi92. ([\#16787](https://github.com/element-hq/synapse/issues/16787)) + +### Bugfixes + +- Handle wildcard type filters properly for room messages endpoint. Contributed by Mo Balaa. ([\#14984](https://github.com/element-hq/synapse/issues/14984)) + +### Improved Documentation + +- Add a link to the "Request log format" explainer on the "Logging sample config" documentation page. ([\#16778](https://github.com/element-hq/synapse/issues/16778)) +- Fix broken links in issue templates and documentation. ([\#16810](https://github.com/element-hq/synapse/issues/16810)) +- NGINX listen http2 deprecation in documentation template for reverse proxy. ([\#16831](https://github.com/element-hq/synapse/issues/16831)) + +### Internal Changes + +- Faster partial join to room with complex auth graph. ([\#7](https://github.com/element-hq/synapse/issues/7)) +- Improve DB performance of calculating badge counts for push. ([\#16756](https://github.com/element-hq/synapse/issues/16756)) +- Split up deleting devices into batches. ([\#16766](https://github.com/element-hq/synapse/issues/16766)) +- Remove CI check for sign-off as we require a CLA signature instead. ([\#16776](https://github.com/element-hq/synapse/issues/16776)) +- Ensure CI fails when linting fails to make sure auto-merge does the correct thing. ([\#16781](https://github.com/element-hq/synapse/issues/16781)) +- Faster load recents for sync by reducing amount of state pulled out. ([\#16783](https://github.com/element-hq/synapse/issues/16783)) +- Reduce amount of state pulled out when querying federation hierachy. ([\#16785](https://github.com/element-hq/synapse/issues/16785)) +- Pull less state out of the DB when we retry fetching old events during backfill. ([\#16788](https://github.com/element-hq/synapse/issues/16788)) +- Optimize query for fetching to-device messages in `/sync`. ([\#16805](https://github.com/element-hq/synapse/issues/16805)) +- Reject OIDC config when `client_secret` isn't specified, but the auth method requires one. ([\#16806](https://github.com/element-hq/synapse/issues/16806)) +- Allow room creation but not publishing to continue if room publication rules are violated when creating + a new room. ([\#16811](https://github.com/element-hq/synapse/issues/16811)) +- Bump minimum supported Rust version to 1.65.0. ([\#16818](https://github.com/element-hq/synapse/issues/16818)) +- Fixup copyright lines in file headers after the licensing change. ([\#16820](https://github.com/element-hq/synapse/issues/16820)) +- Add a `--generate-only` option to the internal configuration/launch script for Complement. ([\#16828](https://github.com/element-hq/synapse/issues/16828)) +- Preparatory work for tweaking performance of auth chain lookups. ([\#16833](https://github.com/element-hq/synapse/issues/16833)) +- Speed up e2e device keys queries for bot accounts. ([\#16841](https://github.com/element-hq/synapse/issues/16841)) + +### Updates to locked dependencies + +* Bump actions/cache from 3 to 4. ([\#16832](https://github.com/element-hq/synapse/issues/16832)) +* Bump actions/download-artifact from 3 to 4. ([\#16795](https://github.com/element-hq/synapse/issues/16795)) +* Bump actions/upload-artifact from 3 to 4. ([\#16796](https://github.com/element-hq/synapse/issues/16796)) +* Bump anyhow from 1.0.75 to 1.0.79. ([\#16789](https://github.com/element-hq/synapse/issues/16789)) +* Bump authlib from 1.2.1 to 1.3.0. ([\#16801](https://github.com/element-hq/synapse/issues/16801)) +* Bump dawidd6/action-download-artifact from 2.28.0 to 3.0.0. ([\#16794](https://github.com/element-hq/synapse/issues/16794)) +* Bump immutabledict from 4.0.0 to 4.1.0. ([\#16812](https://github.com/element-hq/synapse/issues/16812)) +* Bump isort from 5.13.1 to 5.13.2. ([\#16835](https://github.com/element-hq/synapse/issues/16835)) +* Bump lxml from 4.9.3 to 5.1.0. ([\#16813](https://github.com/element-hq/synapse/issues/16813)) +* Bump pillow from 10.1.0 to 10.2.0. ([\#16802](https://github.com/element-hq/synapse/issues/16802)) +* Bump pydantic from 2.5.2 to 2.5.3. ([\#16836](https://github.com/element-hq/synapse/issues/16836)) +* Bump pyo3 from 0.20.0 to 0.20.2. ([\#16791](https://github.com/element-hq/synapse/issues/16791)) +* Bump regex from 1.9.6 to 1.10.3. ([\#16837](https://github.com/element-hq/synapse/issues/16837)) +* Bump ruff from 0.1.13 to 0.1.14. ([\#16838](https://github.com/element-hq/synapse/issues/16838)) +* Bump ruff from 0.1.7 to 0.1.13. ([\#16814](https://github.com/element-hq/synapse/issues/16814)) +* Bump sentry-sdk from 1.35.0 to 1.39.1. ([\#16799](https://github.com/element-hq/synapse/issues/16799)) +* Bump serde_json from 1.0.108 to 1.0.111. ([\#16792](https://github.com/element-hq/synapse/issues/16792)) +* Bump service-identity from 23.1.0 to 24.1.0. ([\#16816](https://github.com/element-hq/synapse/issues/16816)) +* Bump types-commonmark from 0.9.2.4 to 0.9.2.20240106. ([\#16797](https://github.com/element-hq/synapse/issues/16797)) +* Bump types-jsonschema from 4.20.0.0 to 4.20.0.20240105. ([\#16800](https://github.com/element-hq/synapse/issues/16800)) +* Bump types-jsonschema from 4.20.0.20240105 to 4.21.0.20240118. ([\#16834](https://github.com/element-hq/synapse/issues/16834)) +* Bump types-netaddr from 0.9.0.1 to 0.10.0.20240106. ([\#16839](https://github.com/element-hq/synapse/issues/16839)) +* Bump typing-extensions from 4.8.0 to 4.9.0. ([\#16815](https://github.com/element-hq/synapse/issues/16815)) + + +# Synapse 1.99.0 (2024-01-16) + +Synapse 1.99.0 is the first Synapse release under an AGPLv3.0 licence (with CLA to enable Element to sell AGPL +exceptions). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + - https://element.io/blog/synapse-now-lives-at-github-com-element-hq-synapse/ + +No significant changes since 1.99.0rc1. + + +# Synapse 1.99.0rc1 (2024-01-09) + +### Features + +- Add [config options](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#server_notices) to set the avatar and the topic of the server notices room, as well as the avatar of the server notices user. ([\#16679](https://github.com/matrix-org/synapse/issues/16679)) +- Add config option [`email.notif_delay_before_mail`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#email) to tweak the delay before an email is sent following a notification. ([\#16696](https://github.com/matrix-org/synapse/issues/16696)) +- Add new configuration option [`sentry.environment`](https://element-hq.github.io/synapse/v1.99/usage/configuration/config_documentation.html#sentry) for improved system monitoring. Contributed by @zeeshanrafiqrana. ([\#16738](https://github.com/matrix-org/synapse/issues/16738)) +- Filter out rooms from the room directory being served to other homeservers when those rooms block that homeserver by their Access Control Lists. ([\#16759](https://github.com/element-hq/synapse/issues/16759)) + +### Bugfixes + +- Fix a long-standing bug where the signing keys generated by Synapse were world-readable. Contributed by Fabian Klemp. ([\#16740](https://github.com/matrix-org/synapse/issues/16740)) +- Fix email verification redirection. Contributed by Fadhlan Ridhwanallah. ([\#16761](https://github.com/element-hq/synapse/issues/16761)) +- Fixed a bug that prevented users from being queried by display name if it contains non-ASCII characters. ([\#16767](https://github.com/element-hq/synapse/issues/16767)) +- Allow reactivate user without password with Admin API in some edge cases. ([\#16770](https://github.com/element-hq/synapse/issues/16770)) +- Adds the `recursion_depth` parameter to the response of the /relations endpoint if MSC3981 recursion is being performed. ([\#16775](https://github.com/element-hq/synapse/issues/16775)) + +### Improved Documentation + +- Added version picker for Synapse documentation. Contributed by @Dmytro27Ind. ([\#16533](https://github.com/matrix-org/synapse/issues/16533)) +- Clarify that `password_config.enabled: "only_for_reauth"` does not allow new logins to be created using password auth. ([\#16737](https://github.com/matrix-org/synapse/issues/16737)) +- Remove value from header in configuration documentation for `refresh_token_lifetime`. ([\#16763](https://github.com/element-hq/synapse/issues/16763)) +- Add another custom statistics collection server to the documentation. Contributed by @loelkes. ([\#16769](https://github.com/element-hq/synapse/issues/16769)) + +### Internal Changes + +- Remove run-once workflow after adding the version picker to the documentation. ([\#9453](https://github.com/element-hq/synapse/issues/9453)) +- Update the implementation of [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965) (OIDC Provider discovery). ([\#16726](https://github.com/matrix-org/synapse/issues/16726)) +- Move the rust stubs inline for better IDE integration. ([\#16757](https://github.com/element-hq/synapse/issues/16757)) +- Fix sample config doc CI. ([\#16758](https://github.com/element-hq/synapse/issues/16758)) +- Simplify event internal metadata class. ([\#16762](https://github.com/element-hq/synapse/issues/16762), [\#16780](https://github.com/element-hq/synapse/issues/16780)) +- Sign the published docker image using [cosign](https://docs.sigstore.dev/). ([\#16774](https://github.com/element-hq/synapse/issues/16774)) +- Port `EventInternalMetadata` class to Rust. ([\#16782](https://github.com/element-hq/synapse/issues/16782)) + + + +### Updates to locked dependencies + +* Bump actions/setup-go from 4 to 5. ([\#16749](https://github.com/matrix-org/synapse/issues/16749)) +* Bump actions/setup-python from 4 to 5. ([\#16748](https://github.com/matrix-org/synapse/issues/16748)) +* Bump immutabledict from 3.0.0 to 4.0.0. ([\#16743](https://github.com/matrix-org/synapse/issues/16743)) +* Bump isort from 5.12.0 to 5.13.0. ([\#16745](https://github.com/matrix-org/synapse/issues/16745)) +* Bump isort from 5.13.0 to 5.13.1. ([\#16752](https://github.com/matrix-org/synapse/issues/16752)) +* Bump pydantic from 2.5.1 to 2.5.2. ([\#16747](https://github.com/matrix-org/synapse/issues/16747)) +* Bump ruff from 0.1.6 to 0.1.7. ([\#16746](https://github.com/matrix-org/synapse/issues/16746)) +* Bump types-setuptools from 68.2.0.2 to 69.0.0.0. ([\#16744](https://github.com/matrix-org/synapse/issues/16744)) From 5c736cd2af3eb9e1264d439c274d608f6ffc6a83 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 14 Jan 2025 14:23:35 +0000 Subject: [PATCH 088/691] move additional release missed in last commit --- CHANGES.md | 11 ----------- docs/changelogs/CHANGES-2024.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9613aefe4f..d1cdc259e9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -61,15 +61,4 @@ No significant changes since 1.122.0rc1. * Bump serde_json from 1.0.133 to 1.0.134. ([\#18044](https://github.com/element-hq/synapse/issues/18044)) * Bump twine from 5.1.1 to 6.0.1. ([\#18049](https://github.com/element-hq/synapse/issues/18049)) -# Synapse 1.121.1 (2024-12-11) - -This release contains a fix for our docker build CI. It is functionally identical to 1.121.0, whose changelog is below. - -### Internal Changes - -- Downgrade the Ubuntu GHA runner when building docker images. ([\#18026](https://github.com/element-hq/synapse/issues/18026)) - - - - **Changelogs for older versions can be found [here](docs/changelogs/).** diff --git a/docs/changelogs/CHANGES-2024.md b/docs/changelogs/CHANGES-2024.md index 42c151b6ea..ee354f1573 100644 --- a/docs/changelogs/CHANGES-2024.md +++ b/docs/changelogs/CHANGES-2024.md @@ -1,3 +1,13 @@ +# Synapse 1.121.1 (2024-12-11) + +This release contains a fix for our docker build CI. It is functionally identical to 1.121.0, whose changelog is below. + +### Internal Changes + +- Downgrade the Ubuntu GHA runner when building docker images. ([\#18026](https://github.com/element-hq/synapse/issues/18026)) + + + # Synapse 1.121.0 (2024-12-11) ### Internal Changes From fd3ec6435e2d52629ac8f2f1038bcd975f699639 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:17:46 +0000 Subject: [PATCH 089/691] Bump pillow from 11.0.0 to 11.1.0 (#18084) --- poetry.lock | 150 +++++++++++++++++++++++++--------------------------- 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4655d295c4..c1965506be 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1459,93 +1459,89 @@ files = [ [[package]] name = "pillow" -version = "11.0.0" +version = "11.1.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" files = [ - {file = "pillow-11.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6619654954dc4936fcff82db8eb6401d3159ec6be81e33c6000dfd76ae189947"}, - {file = "pillow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3c5ac4bed7519088103d9450a1107f76308ecf91d6dabc8a33a2fcfb18d0fba"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a65149d8ada1055029fcb665452b2814fe7d7082fcb0c5bed6db851cb69b2086"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88a58d8ac0cc0e7f3a014509f0455248a76629ca9b604eca7dc5927cc593c5e9"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c26845094b1af3c91852745ae78e3ea47abf3dbcd1cf962f16b9a5fbe3ee8488"}, - {file = "pillow-11.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1a61b54f87ab5786b8479f81c4b11f4d61702830354520837f8cc791ebba0f5f"}, - {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:674629ff60030d144b7bca2b8330225a9b11c482ed408813924619c6f302fdbb"}, - {file = "pillow-11.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:598b4e238f13276e0008299bd2482003f48158e2b11826862b1eb2ad7c768b97"}, - {file = "pillow-11.0.0-cp310-cp310-win32.whl", hash = "sha256:9a0f748eaa434a41fccf8e1ee7a3eed68af1b690e75328fd7a60af123c193b50"}, - {file = "pillow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a5629742881bcbc1f42e840af185fd4d83a5edeb96475a575f4da50d6ede337c"}, - {file = "pillow-11.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:ee217c198f2e41f184f3869f3e485557296d505b5195c513b2bfe0062dc537f1"}, - {file = "pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1c1d72714f429a521d8d2d018badc42414c3077eb187a59579f28e4270b4b0fc"}, - {file = "pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:499c3a1b0d6fc8213519e193796eb1a86a1be4b1877d678b30f83fd979811d1a"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8b2351c85d855293a299038e1f89db92a2f35e8d2f783489c6f0b2b5f3fe8a3"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4dba50cfa56f910241eb7f883c20f1e7b1d8f7d91c750cd0b318bad443f4d5"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5ddbfd761ee00c12ee1be86c9c0683ecf5bb14c9772ddbd782085779a63dd55b"}, - {file = "pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:45c566eb10b8967d71bf1ab8e4a525e5a93519e29ea071459ce517f6b903d7fa"}, - {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b4fd7bd29610a83a8c9b564d457cf5bd92b4e11e79a4ee4716a63c959699b306"}, - {file = "pillow-11.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb929ca942d0ec4fac404cbf520ee6cac37bf35be479b970c4ffadf2b6a1cad9"}, - {file = "pillow-11.0.0-cp311-cp311-win32.whl", hash = "sha256:006bcdd307cc47ba43e924099a038cbf9591062e6c50e570819743f5607404f5"}, - {file = "pillow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:52a2d8323a465f84faaba5236567d212c3668f2ab53e1c74c15583cf507a0291"}, - {file = "pillow-11.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:16095692a253047fe3ec028e951fa4221a1f3ed3d80c397e83541a3037ff67c9"}, - {file = "pillow-11.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923"}, - {file = "pillow-11.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8069c5179902dcdce0be9bfc8235347fdbac249d23bd90514b7a47a72d9fecf4"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f02541ef64077f22bf4924f225c0fd1248c168f86e4b7abdedd87d6ebaceab0f"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fcb4621042ac4b7865c179bb972ed0da0218a076dc1820ffc48b1d74c1e37fe9"}, - {file = "pillow-11.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7"}, - {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8853a3bf12afddfdf15f57c4b02d7ded92c7a75a5d7331d19f4f9572a89c17e6"}, - {file = "pillow-11.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3107c66e43bda25359d5ef446f59c497de2b5ed4c7fdba0894f8d6cf3822dafc"}, - {file = "pillow-11.0.0-cp312-cp312-win32.whl", hash = "sha256:86510e3f5eca0ab87429dd77fafc04693195eec7fd6a137c389c3eeb4cfb77c6"}, - {file = "pillow-11.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:8ec4a89295cd6cd4d1058a5e6aec6bf51e0eaaf9714774e1bfac7cfc9051db47"}, - {file = "pillow-11.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:27a7860107500d813fcd203b4ea19b04babe79448268403172782754870dac25"}, - {file = "pillow-11.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bcd1fb5bb7b07f64c15618c89efcc2cfa3e95f0e3bcdbaf4642509de1942a699"}, - {file = "pillow-11.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0e038b0745997c7dcaae350d35859c9715c71e92ffb7e0f4a8e8a16732150f38"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ae08bd8ffc41aebf578c2af2f9d8749d91f448b3bfd41d7d9ff573d74f2a6b2"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d69bfd8ec3219ae71bcde1f942b728903cad25fafe3100ba2258b973bd2bc1b2"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:61b887f9ddba63ddf62fd02a3ba7add935d053b6dd7d58998c630e6dbade8527"}, - {file = "pillow-11.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c6a660307ca9d4867caa8d9ca2c2658ab685de83792d1876274991adec7b93fa"}, - {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:73e3a0200cdda995c7e43dd47436c1548f87a30bb27fb871f352a22ab8dcf45f"}, - {file = "pillow-11.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fba162b8872d30fea8c52b258a542c5dfd7b235fb5cb352240c8d63b414013eb"}, - {file = "pillow-11.0.0-cp313-cp313-win32.whl", hash = "sha256:f1b82c27e89fffc6da125d5eb0ca6e68017faf5efc078128cfaa42cf5cb38798"}, - {file = "pillow-11.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ba470552b48e5835f1d23ecb936bb7f71d206f9dfeee64245f30c3270b994de"}, - {file = "pillow-11.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:846e193e103b41e984ac921b335df59195356ce3f71dcfd155aa79c603873b84"}, - {file = "pillow-11.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4ad70c4214f67d7466bea6a08061eba35c01b1b89eaa098040a35272a8efb22b"}, - {file = "pillow-11.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6ec0d5af64f2e3d64a165f490d96368bb5dea8b8f9ad04487f9ab60dc4bb6003"}, - {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c809a70e43c7977c4a42aefd62f0131823ebf7dd73556fa5d5950f5b354087e2"}, - {file = "pillow-11.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:4b60c9520f7207aaf2e1d94de026682fc227806c6e1f55bba7606d1c94dd623a"}, - {file = "pillow-11.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1e2688958a840c822279fda0086fec1fdab2f95bf2b717b66871c4ad9859d7e8"}, - {file = "pillow-11.0.0-cp313-cp313t-win32.whl", hash = "sha256:607bbe123c74e272e381a8d1957083a9463401f7bd01287f50521ecb05a313f8"}, - {file = "pillow-11.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c39ed17edea3bc69c743a8dd3e9853b7509625c2462532e62baa0732163a904"}, - {file = "pillow-11.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:75acbbeb05b86bc53cbe7b7e6fe00fbcf82ad7c684b3ad82e3d711da9ba287d3"}, - {file = "pillow-11.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2e46773dc9f35a1dd28bd6981332fd7f27bec001a918a72a79b4133cf5291dba"}, - {file = "pillow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2679d2258b7f1192b378e2893a8a0a0ca472234d4c2c0e6bdd3380e8dfa21b6a"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda2616eb2313cbb3eebbe51f19362eb434b18e3bb599466a1ffa76a033fb916"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ec184af98a121fb2da42642dea8a29ec80fc3efbaefb86d8fdd2606619045d"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:8594f42df584e5b4bb9281799698403f7af489fba84c34d53d1c4bfb71b7c4e7"}, - {file = "pillow-11.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:c12b5ae868897c7338519c03049a806af85b9b8c237b7d675b8c5e089e4a618e"}, - {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:70fbbdacd1d271b77b7721fe3cdd2d537bbbd75d29e6300c672ec6bb38d9672f"}, - {file = "pillow-11.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5178952973e588b3f1360868847334e9e3bf49d19e169bbbdfaf8398002419ae"}, - {file = "pillow-11.0.0-cp39-cp39-win32.whl", hash = "sha256:8c676b587da5673d3c75bd67dd2a8cdfeb282ca38a30f37950511766b26858c4"}, - {file = "pillow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:94f3e1780abb45062287b4614a5bc0874519c86a777d4a7ad34978e86428b8dd"}, - {file = "pillow-11.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:290f2cc809f9da7d6d622550bbf4c1e57518212da51b6a30fe8e0a270a5b78bd"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1187739620f2b365de756ce086fdb3604573337cc28a0d3ac4a01ab6b2d2a6d2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbbcb7b57dc9c794843e3d1258c0fbf0f48656d46ffe9e09b63bbd6e8cd5d0a2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d203af30149ae339ad1b4f710d9844ed8796e97fda23ffbc4cc472968a47d0b"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a0d3b115009ebb8ac3d2ebec5c2982cc693da935f4ab7bb5c8ebe2f47d36f2"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:73853108f56df97baf2bb8b522f3578221e56f646ba345a372c78326710d3830"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e58876c91f97b0952eb766123bfef372792ab3f4e3e1f1a2267834c2ab131734"}, - {file = "pillow-11.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:224aaa38177597bb179f3ec87eeefcce8e4f85e608025e9cfac60de237ba6316"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5bd2d3bdb846d757055910f0a59792d33b555800813c3b39ada1829c372ccb06"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:375b8dd15a1f5d2feafff536d47e22f69625c1aa92f12b339ec0b2ca40263273"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:daffdf51ee5db69a82dd127eabecce20729e21f7a3680cf7cbb23f0829189790"}, - {file = "pillow-11.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7326a1787e3c7b0429659e0a944725e1b03eeaa10edd945a86dead1913383944"}, - {file = "pillow-11.0.0.tar.gz", hash = "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, + {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, + {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, + {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, + {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, + {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, + {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, + {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, + {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, + {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, + {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, + {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, + {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, + {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, + {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, + {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, + {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, + {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, + {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, + {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, + {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, ] [package.extras] docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] typing = ["typing-extensions"] xmp = ["defusedxml"] From 6cefbc6852a8336548bfe81b13038dcba13f4da5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:17:58 +0000 Subject: [PATCH 090/691] Bump mypy from 1.12.1 to 1.13.0 (#18083) --- poetry.lock | 67 +++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/poetry.lock b/poetry.lock index c1965506be..9764904b81 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1314,43 +1314,43 @@ files = [ [[package]] name = "mypy" -version = "1.12.1" +version = "1.13.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3d7d4371829184e22fda4015278fbfdef0327a4b955a483012bd2d423a788801"}, - {file = "mypy-1.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f59f1dfbf497d473201356966e353ef09d4daec48caeacc0254db8ef633a28a5"}, - {file = "mypy-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b947097fae68004b8328c55161ac9db7d3566abfef72d9d41b47a021c2fba6b1"}, - {file = "mypy-1.12.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96af62050971c5241afb4701c15189ea9507db89ad07794a4ee7b4e092dc0627"}, - {file = "mypy-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:d90da248f4c2dba6c44ddcfea94bb361e491962f05f41990ff24dbd09969ce20"}, - {file = "mypy-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1230048fec1380faf240be6385e709c8570604d2d27ec6ca7e573e3bc09c3735"}, - {file = "mypy-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:02dcfe270c6ea13338210908f8cadc8d31af0f04cee8ca996438fe6a97b4ec66"}, - {file = "mypy-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5a437c9102a6a252d9e3a63edc191a3aed5f2fcb786d614722ee3f4472e33f6"}, - {file = "mypy-1.12.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:186e0c8346efc027ee1f9acf5ca734425fc4f7dc2b60144f0fbe27cc19dc7931"}, - {file = "mypy-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:673ba1140a478b50e6d265c03391702fa11a5c5aff3f54d69a62a48da32cb811"}, - {file = "mypy-1.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9fb83a7be97c498176fb7486cafbb81decccaef1ac339d837c377b0ce3743a7f"}, - {file = "mypy-1.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:389e307e333879c571029d5b93932cf838b811d3f5395ed1ad05086b52148fb0"}, - {file = "mypy-1.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b2048a95a21f7a9ebc9fbd075a4fcd310410d078aa0228dbbad7f71335e042"}, - {file = "mypy-1.12.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ee5932370ccf7ebf83f79d1c157a5929d7ea36313027b0d70a488493dc1b179"}, - {file = "mypy-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:19bf51f87a295e7ab2894f1d8167622b063492d754e69c3c2fed6563268cb42a"}, - {file = "mypy-1.12.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d34167d43613ffb1d6c6cdc0cc043bb106cac0aa5d6a4171f77ab92a3c758bcc"}, - {file = "mypy-1.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:427878aa54f2e2c5d8db31fa9010c599ed9f994b3b49e64ae9cd9990c40bd635"}, - {file = "mypy-1.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5fcde63ea2c9f69d6be859a1e6dd35955e87fa81de95bc240143cf00de1f7f81"}, - {file = "mypy-1.12.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d54d840f6c052929f4a3d2aab2066af0f45a020b085fe0e40d4583db52aab4e4"}, - {file = "mypy-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:20db6eb1ca3d1de8ece00033b12f793f1ea9da767334b7e8c626a4872090cf02"}, - {file = "mypy-1.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b16fe09f9c741d85a2e3b14a5257a27a4f4886c171d562bc5a5e90d8591906b8"}, - {file = "mypy-1.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dcc1e843d58f444fce19da4cce5bd35c282d4bde232acdeca8279523087088a"}, - {file = "mypy-1.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e10ba7de5c616e44ad21005fa13450cd0de7caaa303a626147d45307492e4f2d"}, - {file = "mypy-1.12.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e6fe449223fa59fbee351db32283838a8fee8059e0028e9e6494a03802b4004"}, - {file = "mypy-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:dc6e2a2195a290a7fd5bac3e60b586d77fc88e986eba7feced8b778c373f9afe"}, - {file = "mypy-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:de5b2a8988b4e1269a98beaf0e7cc71b510d050dce80c343b53b4955fff45f19"}, - {file = "mypy-1.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843826966f1d65925e8b50d2b483065c51fc16dc5d72647e0236aae51dc8d77e"}, - {file = "mypy-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fe20f89da41a95e14c34b1ddb09c80262edcc295ad891f22cc4b60013e8f78d"}, - {file = "mypy-1.12.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8135ffec02121a75f75dc97c81af7c14aa4ae0dda277132cfcd6abcd21551bfd"}, - {file = "mypy-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:a7b76fa83260824300cc4834a3ab93180db19876bce59af921467fd03e692810"}, - {file = "mypy-1.12.1-py3-none-any.whl", hash = "sha256:ce561a09e3bb9863ab77edf29ae3a50e65685ad74bba1431278185b7e5d5486e"}, - {file = "mypy-1.12.1.tar.gz", hash = "sha256:f5b3936f7a6d0e8280c9bdef94c7ce4847f5cdfc258fbb2c29a8c1711e8bb96d"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, ] [package.dependencies] @@ -1360,6 +1360,7 @@ typing-extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] install-types = ["pip"] mypyc = ["setuptools (>=50)"] reports = ["lxml"] From 5f15a549d787206eb331ae3799a689489e0949d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:14:46 +0000 Subject: [PATCH 091/691] Bump ulid from 1.1.3 to 1.1.4 (#18080) --- Cargo.lock | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 859dd2086d..f0e06bb491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,10 +124,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] @@ -538,11 +536,10 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ulid" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" +checksum = "f294bff79170ed1c5633812aff1e565c35d993a36e757f9bc0accf5eec4e6045" dependencies = [ - "getrandom", "rand", "web-time", ] From 3fda8d3b67def7a4b734707b838fe4353ab1f8fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:15:01 +0000 Subject: [PATCH 092/691] Bump serde_json from 1.0.134 to 1.0.135 (#18081) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0e06bb491..6a7ecd4b44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,9 +449,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", From 24c4d82aeb1bd5ac15cc0614e243595404fb2009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:18:06 +0000 Subject: [PATCH 093/691] Bump pyo3 from 0.23.3 to 0.23.4 (#18079) --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a7ecd4b44..3cec1ea4a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e484fd2c8b4cb67ab05a318f1fd6fa8f199fcc30819f08f07d200809dba26c15" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "anyhow", "cfg-if", @@ -289,9 +289,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e0469a84f208e20044b98965e1561028180219e35352a2afaf2b942beff3b" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -299,9 +299,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1547a7f9966f6f1a0f0227564a9945fe36b90da5a93b3933fc3dc03fae372d" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -320,9 +320,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb6da8ec6fa5cedd1626c886fc8749bdcbb09424a86461eb8cdf096b7c33257" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -332,9 +332,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a385202ff5a92791168b1136afae5059d3ac118457bb7bc304c197c2d33e7d" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", From 48db0c2d6c0a8b05405d0074f9b0298659233be8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Jan 2025 17:14:06 +0000 Subject: [PATCH 094/691] Drop indices concurrently on background updates (#18091) Otherwise these can race with other long running queries and lock out all other queries. This caused problems in v1.22.0 as we added an index to `events` table in #17948, but that got interrupted and so next time we ran the background update we needed to delete the half-finished index. However, that got blocked behind some long running queries and then locked other queries out (stopping workers from even starting). --- changelog.d/18091.bugfix | 1 + synapse/storage/background_updates.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18091.bugfix diff --git a/changelog.d/18091.bugfix b/changelog.d/18091.bugfix new file mode 100644 index 0000000000..c163b4f9a2 --- /dev/null +++ b/changelog.d/18091.bugfix @@ -0,0 +1 @@ +Fix rare race where on upgrade to v1.22.0 a long running database upgrade could lock out new events from being received or sent. diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 34139f580d..a02b4cc9ce 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -789,7 +789,7 @@ class BackgroundUpdater: # we may already have a half-built index. Let's just drop it # before trying to create it again. - sql = "DROP INDEX IF EXISTS %s" % (index_name,) + sql = "DROP INDEX CONCURRENTLY IF EXISTS %s" % (index_name,) logger.debug("[SQL] %s", sql) c.execute(sql) @@ -814,7 +814,7 @@ class BackgroundUpdater: if replaces_index is not None: # We drop the old index as the new index has now been created. - sql = f"DROP INDEX IF EXISTS {replaces_index}" + sql = f"DROP INDEX CONCURRENTLY IF EXISTS {replaces_index}" logger.debug("[SQL] %s", sql) c.execute(sql) finally: From 0a31cf18cd12003ec2ab2bd41d4e225cd1739e17 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 Jan 2025 12:40:05 -0600 Subject: [PATCH 095/691] Document possibility of configuring `tls` for a worker instance in `instance_map` (#18064) --- changelog.d/18064.doc | 1 + docs/usage/configuration/config_documentation.md | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/18064.doc diff --git a/changelog.d/18064.doc b/changelog.d/18064.doc new file mode 100644 index 0000000000..09f73c9b9b --- /dev/null +++ b/changelog.d/18064.doc @@ -0,0 +1 @@ +Document `tls` option for a worker instance in `instance_map`. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 74522a1e4e..64392366ca 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4465,6 +4465,10 @@ instance_map: worker1: host: localhost port: 8034 + other: + host: localhost + port: 8035 + tls: true ``` Example configuration(#2, for UNIX sockets): ```yaml From ca290d325c762968e66842470475f05ae71a2aea Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 21 Jan 2025 06:11:04 -0500 Subject: [PATCH 096/691] Implement MSC4133 to support custom profile fields. (#17488) Implementation of [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) to support custom profile fields. It is behind an experimental flag and includes tests. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/17488.feature | 1 + synapse/api/errors.py | 4 + synapse/config/experimental.py | 3 + synapse/handlers/profile.py | 138 +++++++- synapse/rest/client/capabilities.py | 17 + synapse/rest/client/profile.py | 193 ++++++++++- synapse/rest/client/versions.py | 2 + synapse/storage/databases/main/profile.py | 312 +++++++++++++++++- .../delta/88/01_custom_profile_fields.sql | 15 + synapse/util/stringutils.py | 12 + tests/rest/client/test_capabilities.py | 44 +++ tests/rest/client/test_profile.py | 296 +++++++++++++++++ tests/util/test_stringutils.py | 28 +- 13 files changed, 1039 insertions(+), 26 deletions(-) create mode 100644 changelog.d/17488.feature create mode 100644 synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql diff --git a/changelog.d/17488.feature b/changelog.d/17488.feature new file mode 100644 index 0000000000..15cccf3ac2 --- /dev/null +++ b/changelog.d/17488.feature @@ -0,0 +1 @@ +Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 21989b6e0e..5dd6e84289 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -132,6 +132,10 @@ class Codes(str, Enum): # connection. UNKNOWN_POS = "M_UNKNOWN_POS" + # Part of MSC4133 + PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" + KEY_TOO_LARGE = "M_KEY_TOO_LARGE" + class CodeMessageException(RuntimeError): """An exception with integer code, a message string attributes and optional headers. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 90d19849ff..94a25c7ee8 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -436,6 +436,9 @@ class ExperimentalConfig(Config): ("experimental", "msc4108_delegation_endpoint"), ) + # MSC4133: Custom profile fields + self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False) + # MSC4210: Remove legacy mentions self.msc4210_enabled: bool = experimental.get("msc4210_enabled", False) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 22eedcb54f..cdc388b4ab 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -32,7 +32,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia -from synapse.types import JsonDict, Requester, UserID, create_requester +from synapse.types import JsonDict, JsonValue, Requester, UserID, create_requester from synapse.util.caches.descriptors import cached from synapse.util.stringutils import parse_and_validate_mxc_uri @@ -43,6 +43,8 @@ logger = logging.getLogger(__name__) MAX_DISPLAYNAME_LEN = 256 MAX_AVATAR_URL_LEN = 1000 +# Field name length is specced at 255 bytes. +MAX_CUSTOM_FIELD_LEN = 255 class ProfileHandler: @@ -90,7 +92,15 @@ class ProfileHandler: if self.hs.is_mine(target_user): profileinfo = await self.store.get_profileinfo(target_user) - if profileinfo.display_name is None and profileinfo.avatar_url is None: + extra_fields = {} + if self.hs.config.experimental.msc4133_enabled: + extra_fields = await self.store.get_profile_fields(target_user) + + if ( + profileinfo.display_name is None + and profileinfo.avatar_url is None + and not extra_fields + ): raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) # Do not include display name or avatar if unset. @@ -99,6 +109,9 @@ class ProfileHandler: ret[ProfileFields.DISPLAYNAME] = profileinfo.display_name if profileinfo.avatar_url is not None: ret[ProfileFields.AVATAR_URL] = profileinfo.avatar_url + if extra_fields: + ret.update(extra_fields) + return ret else: try: @@ -403,6 +416,110 @@ class ProfileHandler: return True + async def get_profile_field( + self, target_user: UserID, field_name: str + ) -> JsonValue: + """ + Fetch a user's profile from the database for local users and over federation + for remote users. + + Args: + target_user: The user ID to fetch the profile for. + field_name: The field to fetch the profile for. + + Returns: + The value for the profile field or None if the field does not exist. + """ + if self.hs.is_mine(target_user): + try: + field_value = await self.store.get_profile_field( + target_user, field_name + ) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) + raise + + return field_value + else: + try: + result = await self.federation.make_query( + destination=target_user.domain, + query_type="profile", + args={"user_id": target_user.to_string(), "field": field_name}, + ignore_backoff=True, + ) + except RequestSendFailed as e: + raise SynapseError(502, "Failed to fetch profile") from e + except HttpResponseException as e: + raise e.to_synapse_error() + + return result.get(field_name) + + async def set_profile_field( + self, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue, + by_admin: bool = False, + deactivation: bool = False, + ) -> None: + """Set a new profile field for a user. + + Args: + target_user: the user whose profile is to be changed. + requester: The user attempting to make this change. + field_name: The name of the profile field to update. + new_value: The new field value for this user. + by_admin: Whether this change was made by an administrator. + deactivation: Whether this change was made while deactivating the user. + """ + if not self.hs.is_mine(target_user): + raise SynapseError(400, "User is not hosted on this homeserver") + + if not by_admin and target_user != requester.user: + raise AuthError(403, "Cannot set another user's profile") + + await self.store.set_profile_field(target_user, field_name, new_value) + + # Custom fields do not propagate into the user directory *or* rooms. + profile = await self.store.get_profileinfo(target_user) + await self._third_party_rules.on_profile_update( + target_user.to_string(), profile, by_admin, deactivation + ) + + async def delete_profile_field( + self, + target_user: UserID, + requester: Requester, + field_name: str, + by_admin: bool = False, + deactivation: bool = False, + ) -> None: + """Delete a field from a user's profile. + + Args: + target_user: the user whose profile is to be changed. + requester: The user attempting to make this change. + field_name: The name of the profile field to remove. + by_admin: Whether this change was made by an administrator. + deactivation: Whether this change was made while deactivating the user. + """ + if not self.hs.is_mine(target_user): + raise SynapseError(400, "User is not hosted on this homeserver") + + if not by_admin and target_user != requester.user: + raise AuthError(400, "Cannot set another user's profile") + + await self.store.delete_profile_field(target_user, field_name) + + # Custom fields do not propagate into the user directory *or* rooms. + profile = await self.store.get_profileinfo(target_user) + await self._third_party_rules.on_profile_update( + target_user.to_string(), profile, by_admin, deactivation + ) + async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" @@ -419,13 +536,24 @@ class ProfileHandler: just_field = args.get("field", None) - response = {} + response: JsonDict = {} try: - if just_field is None or just_field == "displayname": + if just_field is None or just_field == ProfileFields.DISPLAYNAME: response["displayname"] = await self.store.get_profile_displayname(user) - if just_field is None or just_field == "avatar_url": + if just_field is None or just_field == ProfileFields.AVATAR_URL: response["avatar_url"] = await self.store.get_profile_avatar_url(user) + + if self.hs.config.experimental.msc4133_enabled: + if just_field is None: + response.update(await self.store.get_profile_fields(user)) + elif just_field not in ( + ProfileFields.DISPLAYNAME, + ProfileFields.AVATAR_URL, + ): + response[just_field] = await self.store.get_profile_field( + user, just_field + ) except StoreError as e: if e.code == 404: raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 63b8a9364a..ebd5a33ea5 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -92,6 +92,23 @@ class CapabilitiesRestServlet(RestServlet): "enabled": self.config.experimental.msc3664_enabled, } + if self.config.experimental.msc4133_enabled: + response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = { + "enabled": True, + } + + # Ensure this is consistent with the legacy m.set_displayname and + # m.set_avatar_url. + disallowed = [] + if not self.config.registration.enable_set_displayname: + disallowed.append("displayname") + if not self.config.registration.enable_set_avatar_url: + disallowed.append("avatar_url") + if disallowed: + response["capabilities"]["uk.tcpip.msc4133.profile_fields"][ + "disallowed" + ] = disallowed + return HTTPStatus.OK, response diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index ef59582865..8326d8017c 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -21,10 +21,13 @@ """This module contains REST servlets to do with profile: /profile/""" +import re from http import HTTPStatus from typing import TYPE_CHECKING, Tuple +from synapse.api.constants import ProfileFields from synapse.api.errors import Codes, SynapseError +from synapse.handlers.profile import MAX_CUSTOM_FIELD_LEN from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -33,7 +36,8 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, JsonValue, UserID +from synapse.util.stringutils import is_namedspaced_grammar if TYPE_CHECKING: from synapse.server import HomeServer @@ -91,6 +95,11 @@ class ProfileDisplaynameRestServlet(RestServlet): async def on_PUT( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request, allow_guest=True) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -101,9 +110,7 @@ class ProfileDisplaynameRestServlet(RestServlet): new_name = content["displayname"] except Exception: raise SynapseError( - code=400, - msg="Unable to parse name", - errcode=Codes.BAD_JSON, + 400, "Missing key 'displayname'", errcode=Codes.MISSING_PARAM ) propagate = _read_propagate(self.hs, request) @@ -166,6 +173,11 @@ class ProfileAvatarURLRestServlet(RestServlet): async def on_PUT( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -232,7 +244,180 @@ class ProfileRestServlet(RestServlet): return 200, ret +class UnstableProfileFieldRestServlet(RestServlet): + PATTERNS = [ + re.compile( + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + ) + ] + CATEGORY = "Event sending requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.hs = hs + self.profile_handler = hs.get_profile_handler() + self.auth = hs.get_auth() + + async def on_GET( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + requester_user = None + + if self.hs.config.server.require_auth_for_profile_requests: + requester = await self.auth.get_user_by_req(request) + requester_user = requester.user + + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + user = UserID.from_string(user_id) + await self.profile_handler.check_profile_query_allowed(user, requester_user) + + if field_name == ProfileFields.DISPLAYNAME: + field_value: JsonValue = await self.profile_handler.get_displayname(user) + elif field_name == ProfileFields.AVATAR_URL: + field_value = await self.profile_handler.get_avatar_url(user) + else: + field_value = await self.profile_handler.get_profile_field(user, field_name) + + return 200, {field_name: field_value} + + async def on_PUT( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + requester = await self.auth.get_user_by_req(request) + user = UserID.from_string(user_id) + is_admin = await self.auth.is_server_admin(requester) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + content = parse_json_object_from_request(request) + try: + new_value = content[field_name] + except KeyError: + raise SynapseError( + 400, f"Missing key '{field_name}'", errcode=Codes.MISSING_PARAM + ) + + propagate = _read_propagate(self.hs, request) + + requester_suspended = ( + await self.hs.get_datastores().main.get_user_suspended_status( + requester.user.to_string() + ) + ) + + if requester_suspended: + raise SynapseError( + 403, + "Updating profile while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + + if field_name == ProfileFields.DISPLAYNAME: + await self.profile_handler.set_displayname( + user, requester, new_value, is_admin, propagate=propagate + ) + elif field_name == ProfileFields.AVATAR_URL: + await self.profile_handler.set_avatar_url( + user, requester, new_value, is_admin, propagate=propagate + ) + else: + await self.profile_handler.set_profile_field( + user, requester, field_name, new_value, is_admin + ) + + return 200, {} + + async def on_DELETE( + self, request: SynapseRequest, user_id: str, field_name: str + ) -> Tuple[int, JsonDict]: + if not UserID.is_valid(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM + ) + + requester = await self.auth.get_user_by_req(request) + user = UserID.from_string(user_id) + is_admin = await self.auth.is_server_admin(requester) + + if not field_name: + raise SynapseError(400, "Field name too short", errcode=Codes.INVALID_PARAM) + + if len(field_name.encode("utf-8")) > MAX_CUSTOM_FIELD_LEN: + raise SynapseError(400, "Field name too long", errcode=Codes.KEY_TOO_LARGE) + if not is_namedspaced_grammar(field_name): + raise SynapseError( + 400, + "Field name does not follow Common Namespaced Identifier Grammar", + errcode=Codes.INVALID_PARAM, + ) + + propagate = _read_propagate(self.hs, request) + + requester_suspended = ( + await self.hs.get_datastores().main.get_user_suspended_status( + requester.user.to_string() + ) + ) + + if requester_suspended: + raise SynapseError( + 403, + "Updating profile while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + + if field_name == ProfileFields.DISPLAYNAME: + await self.profile_handler.set_displayname( + user, requester, "", is_admin, propagate=propagate + ) + elif field_name == ProfileFields.AVATAR_URL: + await self.profile_handler.set_avatar_url( + user, requester, "", is_admin, propagate=propagate + ) + else: + await self.profile_handler.delete_profile_field( + user, requester, field_name, is_admin + ) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + # The specific displayname / avatar URL / custom field endpoints *must* appear + # before their corresponding generic profile endpoint. ProfileDisplaynameRestServlet(hs).register(http_server) ProfileAvatarURLRestServlet(hs).register(http_server) ProfileRestServlet(hs).register(http_server) + if hs.config.experimental.msc4133_enabled: + UnstableProfileFieldRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index a1d089ebac..266a0b835b 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -172,6 +172,8 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), # Simplified sliding sync "org.matrix.simplified_msc3575": msc3575_enabled, + # Arbitrary key-value profile fields. + "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, }, }, ) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 41cf08211f..30d8a58d96 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -18,8 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Optional +import json +from typing import TYPE_CHECKING, Dict, Optional, Tuple, cast +from canonicaljson import encode_canonical_json + +from synapse.api.constants import ProfileFields +from synapse.api.errors import Codes, StoreError from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -27,13 +32,17 @@ from synapse.storage.database import ( LoggingTransaction, ) from synapse.storage.databases.main.roommember import ProfileInfo -from synapse.storage.engines import PostgresEngine -from synapse.types import JsonDict, UserID +from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.types import JsonDict, JsonValue, UserID if TYPE_CHECKING: from synapse.server import HomeServer +# The number of bytes that the serialized profile can have. +MAX_PROFILE_SIZE = 65536 + + class ProfileWorkerStore(SQLBaseStore): def __init__( self, @@ -201,6 +210,89 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) + async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue: + """ + Get a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The custom profile field name. + + Returns: + The string value if the field exists, otherwise raises 404. + """ + + def get_profile_field(txn: LoggingTransaction) -> JsonValue: + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + field_path = f'$."{field_name}"' + + if isinstance(self.database_engine, PostgresEngine): + sql = """ + SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) + FROM profiles + WHERE user_id = ? + """ + txn.execute( + sql, + (field_path, field_name, user_id.localpart), + ) + + # Test exists first since value being None is used for both + # missing and a null JSON value. + exists, value = cast(Tuple[bool, JsonValue], txn.fetchone()) + if not exists: + raise StoreError(404, "No row found") + return value + + else: + sql = """ + SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) + FROM profiles + WHERE user_id = ? + """ + txn.execute( + sql, + (field_path, field_path, user_id.localpart), + ) + + # If value_type is None, then the value did not exist. + value_type, value = cast( + Tuple[Optional[str], JsonValue], txn.fetchone() + ) + if not value_type: + raise StoreError(404, "No row found") + # If value_type is object or array, then need to deserialize the JSON. + # Scalar values are properly returned directly. + if value_type in ("object", "array"): + assert isinstance(value, str) + return json.loads(value) + return value + + return await self.db_pool.runInteraction("get_profile_field", get_profile_field) + + async def get_profile_fields(self, user_id: UserID) -> Dict[str, str]: + """ + Get all custom profile fields for a user. + + Args: + user_id: The user's ID. + + Returns: + A dictionary of custom profile fields. + """ + result = await self.db_pool.simple_select_one_onecol( + table="profiles", + keyvalues={"full_user_id": user_id.to_string()}, + retcol="fields", + desc="get_profile_fields", + ) + # The SQLite driver doesn't automatically convert JSON to + # Python objects + if isinstance(self.database_engine, Sqlite3Engine) and result: + result = json.loads(result) + return result or {} + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user. @@ -215,6 +307,71 @@ class ProfileWorkerStore(SQLBaseStore): desc="create_profile", ) + def _check_profile_size( + self, + txn: LoggingTransaction, + user_id: UserID, + new_field_name: str, + new_value: JsonValue, + ) -> None: + # For each entry there are 4 quotes (2 each for key and value), 1 colon, + # and 1 comma. + PER_VALUE_EXTRA = 6 + + # Add the size of the current custom profile fields, ignoring the entry + # which will be overwritten. + if isinstance(txn.database_engine, PostgresEngine): + size_sql = """ + SELECT + OCTET_LENGTH((fields - ?)::text), OCTET_LENGTH(displayname), OCTET_LENGTH(avatar_url) + FROM profiles + WHERE + user_id = ? + """ + txn.execute( + size_sql, + (new_field_name, user_id.localpart), + ) + else: + size_sql = """ + SELECT + LENGTH(json_remove(fields, ?)), LENGTH(displayname), LENGTH(avatar_url) + FROM profiles + WHERE + user_id = ? + """ + txn.execute( + size_sql, + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + (f'$."{new_field_name}"', user_id.localpart), + ) + row = cast(Tuple[Optional[int], Optional[int], Optional[int]], txn.fetchone()) + + # The values return null if the column is null. + total_bytes = ( + # Discount the opening and closing braces to avoid double counting, + # but add one for a comma. + # -2 + 1 = -1 + (row[0] - 1 if row[0] else 0) + + ( + row[1] + len("displayname") + PER_VALUE_EXTRA + if new_field_name != ProfileFields.DISPLAYNAME and row[1] + else 0 + ) + + ( + row[2] + len("avatar_url") + PER_VALUE_EXTRA + if new_field_name != ProfileFields.AVATAR_URL and row[2] + else 0 + ) + ) + + # Add the length of the field being added + the braces. + total_bytes += len(encode_canonical_json({new_field_name: new_value})) + + if total_bytes > MAX_PROFILE_SIZE: + raise StoreError(400, "Profile too large", Codes.PROFILE_TOO_LARGE) + async def set_profile_displayname( self, user_id: UserID, new_displayname: Optional[str] ) -> None: @@ -227,14 +384,25 @@ class ProfileWorkerStore(SQLBaseStore): name is removed. """ user_localpart = user_id.localpart - await self.db_pool.simple_upsert( - table="profiles", - keyvalues={"user_id": user_localpart}, - values={ - "displayname": new_displayname, - "full_user_id": user_id.to_string(), - }, - desc="set_profile_displayname", + + def set_profile_displayname(txn: LoggingTransaction) -> None: + if new_displayname is not None: + self._check_profile_size( + txn, user_id, ProfileFields.DISPLAYNAME, new_displayname + ) + + self.db_pool.simple_upsert_txn( + txn, + table="profiles", + keyvalues={"user_id": user_localpart}, + values={ + "displayname": new_displayname, + "full_user_id": user_id.to_string(), + }, + ) + + await self.db_pool.runInteraction( + "set_profile_displayname", set_profile_displayname ) async def set_profile_avatar_url( @@ -249,13 +417,125 @@ class ProfileWorkerStore(SQLBaseStore): removed. """ user_localpart = user_id.localpart - await self.db_pool.simple_upsert( - table="profiles", - keyvalues={"user_id": user_localpart}, - values={"avatar_url": new_avatar_url, "full_user_id": user_id.to_string()}, - desc="set_profile_avatar_url", + + def set_profile_avatar_url(txn: LoggingTransaction) -> None: + if new_avatar_url is not None: + self._check_profile_size( + txn, user_id, ProfileFields.AVATAR_URL, new_avatar_url + ) + + self.db_pool.simple_upsert_txn( + txn, + table="profiles", + keyvalues={"user_id": user_localpart}, + values={ + "avatar_url": new_avatar_url, + "full_user_id": user_id.to_string(), + }, + ) + + await self.db_pool.runInteraction( + "set_profile_avatar_url", set_profile_avatar_url ) + async def set_profile_field( + self, user_id: UserID, field_name: str, new_value: JsonValue + ) -> None: + """ + Set a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + new_value: The value of the custom profile field. + """ + + # Encode to canonical JSON. + canonical_value = encode_canonical_json(new_value) + + def set_profile_field(txn: LoggingTransaction) -> None: + self._check_profile_size(txn, user_id, field_name, new_value) + + if isinstance(self.database_engine, PostgresEngine): + from psycopg2.extras import Json + + # Note that the || jsonb operator is not recursive, any duplicate + # keys will be taken from the second value. + sql = """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields + """ + + txn.execute( + sql, + ( + user_id.localpart, + user_id.to_string(), + field_name, + # Pass as a JSON object since we have passing bytes disabled + # at the database driver. + Json(json.loads(canonical_value)), + ), + ) + else: + # You may be tempted to use json_patch instead of providing the parameters + # twice, but that recursively merges objects instead of replacing. + sql = """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) + """ + # This will error if field_name has double quotes in it, but that's not + # possible due to the grammar. + json_field_name = f'$."{field_name}"' + + txn.execute( + sql, + ( + user_id.localpart, + user_id.to_string(), + json_field_name, + canonical_value, + json_field_name, + canonical_value, + ), + ) + + await self.db_pool.runInteraction("set_profile_field", set_profile_field) + + async def delete_profile_field(self, user_id: UserID, field_name: str) -> None: + """ + Remove a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + """ + + def delete_profile_field(txn: LoggingTransaction) -> None: + if isinstance(self.database_engine, PostgresEngine): + sql = """ + UPDATE profiles SET fields = fields - ? + WHERE user_id = ? + """ + txn.execute( + sql, + (field_name, user_id.localpart), + ) + else: + sql = """ + UPDATE profiles SET fields = json_remove(fields, ?) + WHERE user_id = ? + """ + txn.execute( + sql, + # This will error if field_name has double quotes in it. + (f'$."{field_name}"', user_id.localpart), + ) + + await self.db_pool.runInteraction("delete_profile_field", delete_profile_field) + class ProfileStore(ProfileWorkerStore): pass diff --git a/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql b/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql new file mode 100644 index 0000000000..63cbd7ffa9 --- /dev/null +++ b/synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 Patrick Cloke +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Custom profile fields. +ALTER TABLE profiles ADD COLUMN fields JSONB; diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 13ff54b669..32b5bc00c9 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -43,6 +43,14 @@ CLIENT_SECRET_REGEX = re.compile(r"^[0-9a-zA-Z\.=_\-]+$") # MXC_REGEX = re.compile("^mxc://([^/]+)/([^/#?]+)$") +# https://spec.matrix.org/v1.13/appendices/#common-namespaced-identifier-grammar +# +# At least one character, less than or equal to 255 characters. Must start with +# a-z, the rest is a-z, 0-9, -, _, or .. +# +# This doesn't check anything about validity of namespaces. +NAMESPACED_GRAMMAR = re.compile(r"^[a-z][a-z0-9_.-]{0,254}$") + def random_string(length: int) -> str: """Generate a cryptographically secure string of random letters. @@ -68,6 +76,10 @@ def is_ascii(s: bytes) -> bool: return True +def is_namedspaced_grammar(s: str) -> bool: + return bool(NAMESPACED_GRAMMAR.match(s)) + + def assert_valid_client_secret(client_secret: str) -> None: """Validate that a given string matches the client_secret defined by the spec""" if ( diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index bbe8ab1a7c..8af00221c2 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -142,6 +142,50 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + @override_config( + { + "enable_set_displayname": False, + "experimental_features": {"msc4133_enabled": True}, + } + ) + def test_get_set_displayname_capabilities_displayname_disabled_msc4133( + self, + ) -> None: + """Test if set displayname is disabled that the server responds it.""" + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], + ["displayname"], + ) + + @override_config( + { + "enable_set_avatar_url": False, + "experimental_features": {"msc4133_enabled": True}, + } + ) + def test_get_set_avatar_url_capabilities_avatar_url_disabled_msc4133(self) -> None: + """Test if set avatar_url is disabled that the server responds it.""" + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], + ["avatar_url"], + ) + @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: """Test if change 3pid is disabled that the server responds it.""" diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index a92713d220..708402b792 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -25,16 +25,20 @@ import urllib.parse from http import HTTPStatus from typing import Any, Dict, Optional +from canonicaljson import encode_canonical_json + from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import Codes from synapse.rest import admin from synapse.rest.client import login, profile, room from synapse.server import HomeServer +from synapse.storage.databases.main.profile import MAX_PROFILE_SIZE from synapse.types import UserID from synapse.util import Clock from tests import unittest +from tests.utils import USE_POSTGRES_FOR_TESTS class ProfileTestCase(unittest.HomeserverTestCase): @@ -480,6 +484,298 @@ class ProfileTestCase(unittest.HomeserverTestCase): # The client requested ?propagate=true, so it should have happened. self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_get_missing_custom_field(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_get_missing_custom_field_invalid_field_name(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/[custom_field]", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_get_custom_field_rejects_bad_username(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{urllib.parse.quote('@alice:')}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + content={"custom_field": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"custom_field": "test"}) + + # Overwriting the field should work. + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + content={"custom_field": "new_Value"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"custom_field": "new_Value"}) + + # Deleting the field should work. + channel = self.make_request( + "DELETE", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + content={}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_non_string(self) -> None: + """Non-string fields are supported for custom fields.""" + fields = { + "bool_field": True, + "array_field": ["test"], + "object_field": {"test": "test"}, + "numeric_field": 1, + "null_field": None, + } + + for key, value in fields.items(): + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: value}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"displayname": "owner", **fields}) + + # Check getting individual fields works. + for key, value in fields.items(): + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {key: value}) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_noauth(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + content={"custom_field": "test"}, + ) + self.assertEqual(channel.code, 401, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.MISSING_TOKEN) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_size(self) -> None: + """ + Attempts to set a custom field name that is too long should get a 400 error. + """ + # Key is missing. + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/", + content={"": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + # Single key is too large. + key = "c" * 500 + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.KEY_TOO_LARGE) + + channel = self.make_request( + "DELETE", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.KEY_TOO_LARGE) + + # Key doesn't match body. + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + content={"diff_key": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.MISSING_PARAM) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_profile_too_long(self) -> None: + """ + Attempts to set a custom field that would push the overall profile too large. + """ + # Get right to the boundary: + # len("displayname") + len("owner") + 5 = 21 for the displayname + # 1 + 65498 + 5 for key "a" = 65504 + # 2 braces, 1 comma + # 3 + 21 + 65498 = 65522 < 65536. + key = "a" + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: "a" * 65498}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Get the entire profile. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/profile/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + canonical_json = encode_canonical_json(channel.json_body) + # 6 is the minimum bytes to store a value: 4 quotes, 1 colon, 1 comma, an empty key. + # Be one below that so we can prove we're at the boundary. + self.assertEqual(len(canonical_json), MAX_PROFILE_SIZE - 8) + + # Postgres stores JSONB with whitespace, while SQLite doesn't. + if USE_POSTGRES_FOR_TESTS: + ADDITIONAL_CHARS = 0 + else: + ADDITIONAL_CHARS = 1 + + # The next one should fail, note the value has a (JSON) length of 2. + key = "b" + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: "1" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + # Setting an avatar or (longer) display name should not work. + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/displayname", + content={"displayname": "owner12345678" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/avatar_url", + content={"avatar_url": "mxc://foo/bar"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 400, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.PROFILE_TOO_LARGE) + + # Removing a single byte should work. + key = "b" + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: "" + "a" * ADDITIONAL_CHARS}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Finally, setting a field that already exists to a value that is <= in length should work. + key = "a" + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + content={key: ""}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_displayname(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/displayname", + content={"displayname": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + displayname = self._get_displayname() + self.assertEqual(displayname, "test") + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_avatar_url(self) -> None: + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/avatar_url", + content={"avatar_url": "mxc://test/good"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + avatar_url = self._get_avatar_url() + self.assertEqual(avatar_url, "mxc://test/good") + + @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) + def test_set_custom_field_other(self) -> None: + """Setting someone else's profile field should fail""" + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.other}/custom_field", + content={"custom_field": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + def _setup_local_files(self, names_and_props: Dict[str, Dict[str, Any]]) -> None: """Stores metadata about files in the database. diff --git a/tests/util/test_stringutils.py b/tests/util/test_stringutils.py index 646fd2163e..34c2395ecf 100644 --- a/tests/util/test_stringutils.py +++ b/tests/util/test_stringutils.py @@ -20,7 +20,11 @@ # from synapse.api.errors import SynapseError -from synapse.util.stringutils import assert_valid_client_secret, base62_encode +from synapse.util.stringutils import ( + assert_valid_client_secret, + base62_encode, + is_namedspaced_grammar, +) from .. import unittest @@ -58,3 +62,25 @@ class StringUtilsTestCase(unittest.TestCase): self.assertEqual("10", base62_encode(62)) self.assertEqual("1c", base62_encode(100)) self.assertEqual("001c", base62_encode(100, minwidth=4)) + + def test_namespaced_identifier(self) -> None: + self.assertTrue(is_namedspaced_grammar("test")) + self.assertTrue(is_namedspaced_grammar("m.test")) + self.assertTrue(is_namedspaced_grammar("org.matrix.test")) + self.assertTrue(is_namedspaced_grammar("org.matrix.msc1234")) + self.assertTrue(is_namedspaced_grammar("test")) + self.assertTrue(is_namedspaced_grammar("t-e_s.t")) + + # Must start with letter. + self.assertFalse(is_namedspaced_grammar("1test")) + self.assertFalse(is_namedspaced_grammar("-test")) + self.assertFalse(is_namedspaced_grammar("_test")) + self.assertFalse(is_namedspaced_grammar(".test")) + + # Must contain only a-z, 0-9, -, _, .. + self.assertFalse(is_namedspaced_grammar("test/")) + self.assertFalse(is_namedspaced_grammar('test"')) + self.assertFalse(is_namedspaced_grammar("testö")) + + # Must be < 255 characters. + self.assertFalse(is_namedspaced_grammar("t" * 256)) From 048c1ac7f6cb3c05ac71fcbb73db49d9a32b1984 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 21 Jan 2025 13:48:49 +0100 Subject: [PATCH 097/691] Support the new `/auth_metadata` endpoint defined in MSC2965. (#18093) See the updated MSC2965 --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18093.feature | 1 + synapse/api/auth/msc3861_delegated.py | 6 + synapse/rest/__init__.py | 6 +- .../{auth_issuer.py => auth_metadata.py} | 37 +++++ tests/rest/client/test_auth_issuer.py | 79 ---------- tests/rest/client/test_auth_metadata.py | 140 ++++++++++++++++++ 6 files changed, 187 insertions(+), 82 deletions(-) create mode 100644 changelog.d/18093.feature rename synapse/rest/client/{auth_issuer.py => auth_metadata.py} (64%) delete mode 100644 tests/rest/client/test_auth_issuer.py create mode 100644 tests/rest/client/test_auth_metadata.py diff --git a/changelog.d/18093.feature b/changelog.d/18093.feature new file mode 100644 index 0000000000..689766ec0a --- /dev/null +++ b/changelog.d/18093.feature @@ -0,0 +1 @@ +Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). \ No newline at end of file diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 53907c01d4..802ea51d18 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -174,6 +174,12 @@ class MSC3861DelegatedAuth(BaseAuth): logger.warning("Failed to load metadata:", exc_info=True) return None + async def auth_metadata(self) -> Dict[str, Any]: + """ + Returns the auth metadata dict + """ + return await self._issuer_metadata.get() + async def _introspection_endpoint(self) -> str: """ Returns the introspection endpoint of the issuer diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 4e594e6595..2f1ef84e26 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -29,7 +29,7 @@ from synapse.rest.client import ( account_validity, appservice_ping, auth, - auth_issuer, + auth_metadata, capabilities, delayed_events, devices, @@ -121,7 +121,7 @@ CLIENT_SERVLET_FUNCTIONS: Tuple[RegisterServletsFunc, ...] = ( mutual_rooms.register_servlets, login_token_request.register_servlets, rendezvous.register_servlets, - auth_issuer.register_servlets, + auth_metadata.register_servlets, ) SERVLET_GROUPS: Dict[str, Iterable[RegisterServletsFunc]] = { @@ -187,7 +187,7 @@ class ClientRestResource(JsonResource): mutual_rooms.register_servlets, login_token_request.register_servlets, rendezvous.register_servlets, - auth_issuer.register_servlets, + auth_metadata.register_servlets, ]: continue diff --git a/synapse/rest/client/auth_issuer.py b/synapse/rest/client/auth_metadata.py similarity index 64% rename from synapse/rest/client/auth_issuer.py rename to synapse/rest/client/auth_metadata.py index acd0399d85..5444a89be6 100644 --- a/synapse/rest/client/auth_issuer.py +++ b/synapse/rest/client/auth_metadata.py @@ -32,6 +32,8 @@ logger = logging.getLogger(__name__) class AuthIssuerServlet(RestServlet): """ Advertises what OpenID Connect issuer clients should use to authorise users. + This endpoint was defined in a previous iteration of MSC2965, and is still + used by some clients. """ PATTERNS = client_patterns( @@ -63,7 +65,42 @@ class AuthIssuerServlet(RestServlet): ) +class AuthMetadataServlet(RestServlet): + """ + Advertises the OAuth 2.0 server metadata for the homeserver. + """ + + PATTERNS = client_patterns( + "/org.matrix.msc2965/auth_metadata$", + unstable=True, + releases=(), + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self._config = hs.config + self._auth = hs.get_auth() + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + if self._config.experimental.msc3861.enabled: + # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth + # We import lazily here because of the authlib requirement + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + auth = cast(MSC3861DelegatedAuth, self._auth) + return 200, await auth.auth_metadata() + else: + # Wouldn't expect this to be reached: the servlet shouldn't have been + # registered. Still, fail gracefully if we are registered for some reason. + raise SynapseError( + 404, + "OIDC discovery has not been configured on this homeserver", + Codes.NOT_FOUND, + ) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: # We use the MSC3861 values as they are used by multiple MSCs if hs.config.experimental.msc3861.enabled: AuthIssuerServlet(hs).register(http_server) + AuthMetadataServlet(hs).register(http_server) diff --git a/tests/rest/client/test_auth_issuer.py b/tests/rest/client/test_auth_issuer.py deleted file mode 100644 index d6f334a7ab..0000000000 --- a/tests/rest/client/test_auth_issuer.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2023 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from http import HTTPStatus -from unittest.mock import AsyncMock - -from synapse.rest.client import auth_issuer - -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - -ISSUER = "https://account.example.com/" - - -class AuthIssuerTestCase(HomeserverTestCase): - servlets = [ - auth_issuer.register_servlets, - ] - - def test_returns_404_when_msc3861_disabled(self) -> None: - # Make an unauthenticated request for the discovery info. - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock(return_value={"issuer": ISSUER}) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - - req_mock.assert_called_with( - "https://account.example.com/.well-known/openid-configuration" - ) - req_mock.reset_mock() - - # Second call it should use the cached value - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - req_mock.assert_not_called() diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py new file mode 100644 index 0000000000..a935533b09 --- /dev/null +++ b/tests/rest/client/test_auth_metadata.py @@ -0,0 +1,140 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2023 The Matrix.org Foundation C.I.C +# Copyright (C) 2023-2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +from http import HTTPStatus +from unittest.mock import AsyncMock + +from synapse.rest.client import auth_metadata + +from tests.unittest import HomeserverTestCase, override_config, skip_unless +from tests.utils import HAS_AUTHLIB + +ISSUER = "https://account.example.com/" + + +class AuthIssuerTestCase(HomeserverTestCase): + servlets = [ + auth_metadata.register_servlets, + ] + + def test_returns_404_when_msc3861_disabled(self) -> None: + # Make an unauthenticated request for the discovery info. + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + + @skip_unless(HAS_AUTHLIB, "requires authlib") + @override_config( + { + "disable_registration": True, + "experimental_features": { + "msc3861": { + "enabled": True, + "issuer": ISSUER, + "client_id": "David Lister", + "client_auth_method": "client_secret_post", + "client_secret": "Who shot Mister Burns?", + } + }, + } + ) + def test_returns_issuer_when_oidc_enabled(self) -> None: + # Patch the HTTP client to return the issuer metadata + req_mock = AsyncMock(return_value={"issuer": ISSUER}) + self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] + + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"issuer": ISSUER}) + + req_mock.assert_called_with( + "https://account.example.com/.well-known/openid-configuration" + ) + req_mock.reset_mock() + + # Second call it should use the cached value + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"issuer": ISSUER}) + req_mock.assert_not_called() + + +class AuthMetadataTestCase(HomeserverTestCase): + servlets = [ + auth_metadata.register_servlets, + ] + + def test_returns_404_when_msc3861_disabled(self) -> None: + # Make an unauthenticated request for the discovery info. + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_metadata", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + + @skip_unless(HAS_AUTHLIB, "requires authlib") + @override_config( + { + "disable_registration": True, + "experimental_features": { + "msc3861": { + "enabled": True, + "issuer": ISSUER, + "client_id": "David Lister", + "client_auth_method": "client_secret_post", + "client_secret": "Who shot Mister Burns?", + } + }, + } + ) + def test_returns_issuer_when_oidc_enabled(self) -> None: + # Patch the HTTP client to return the issuer metadata + req_mock = AsyncMock( + return_value={ + "issuer": ISSUER, + "authorization_endpoint": "https://example.com/auth", + "token_endpoint": "https://example.com/token", + } + ) + self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] + + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc2965/auth_metadata", + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + channel.json_body, + { + "issuer": ISSUER, + "authorization_endpoint": "https://example.com/auth", + "token_endpoint": "https://example.com/token", + }, + ) From e27808f30677971792aee08f8ebcce26e920faea Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:46:40 +0100 Subject: [PATCH 098/691] 1.123.0rc1 --- CHANGES.md | 37 +++++++++++++++++++++++++++++++++++++ changelog.d/17488.feature | 1 - changelog.d/17732.bugfix | 1 - changelog.d/18035.feature | 1 - changelog.d/18052.removal | 1 - changelog.d/18064.doc | 1 - changelog.d/18072.misc | 1 - changelog.d/18091.bugfix | 1 - changelog.d/18093.feature | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 11 files changed, 44 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/17488.feature delete mode 100644 changelog.d/17732.bugfix delete mode 100644 changelog.d/18035.feature delete mode 100644 changelog.d/18052.removal delete mode 100644 changelog.d/18064.doc delete mode 100644 changelog.d/18072.misc delete mode 100644 changelog.d/18091.bugfix delete mode 100644 changelog.d/18093.feature diff --git a/CHANGES.md b/CHANGES.md index d1cdc259e9..2c7763fbc0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,40 @@ +# Synapse 1.123.0rc1 (2025-01-21) + +### Features + +- Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. ([\#17488](https://github.com/element-hq/synapse/issues/17488)) +- Add a unit test for the `type` parameter of the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api). ([\#18035](https://github.com/element-hq/synapse/issues/18035)) +- Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). ([\#18093](https://github.com/element-hq/synapse/issues/18093)) + +### Bugfixes + +- Fix membership caches not updating in state reset scenarios. ([\#17732](https://github.com/element-hq/synapse/issues/17732)) +- Fix rare race where on upgrade to v1.122.0 a long running database upgrade could lock out new events from being received or sent. ([\#18091](https://github.com/element-hq/synapse/issues/18091)) + +### Improved Documentation + +- Document `tls` option for a worker instance in `instance_map`. ([\#18064](https://github.com/element-hq/synapse/issues/18064)) + +### Deprecations and Removals + +- Remove the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) implementation. The stable support remains, per [Matrix 1.13](https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv3roomsroomidreport). ([\#18052](https://github.com/element-hq/synapse/issues/18052)) + +### Internal Changes + +- Increase invite rate limits (`rc_invites.per_issuer`) for Complement. ([\#18072](https://github.com/element-hq/synapse/issues/18072)) + + + +### Updates to locked dependencies + +* Bump jinja2 from 3.1.4 to 3.1.5. ([\#18067](https://github.com/element-hq/synapse/issues/18067)) +* Bump mypy from 1.12.1 to 1.13.0. ([\#18083](https://github.com/element-hq/synapse/issues/18083)) +* Bump pillow from 11.0.0 to 11.1.0. ([\#18084](https://github.com/element-hq/synapse/issues/18084)) +* Bump pyo3 from 0.23.3 to 0.23.4. ([\#18079](https://github.com/element-hq/synapse/issues/18079)) +* Bump pyopenssl from 24.2.1 to 24.3.0. ([\#18062](https://github.com/element-hq/synapse/issues/18062)) +* Bump serde_json from 1.0.134 to 1.0.135. ([\#18081](https://github.com/element-hq/synapse/issues/18081)) +* Bump ulid from 1.1.3 to 1.1.4. ([\#18080](https://github.com/element-hq/synapse/issues/18080)) + # Synapse 1.122.0 (2025-01-14) Please note that this version of Synapse drops support for PostgreSQL 11 and 12. The minimum version of PostgreSQL supported is now version 13. diff --git a/changelog.d/17488.feature b/changelog.d/17488.feature deleted file mode 100644 index 15cccf3ac2..0000000000 --- a/changelog.d/17488.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. diff --git a/changelog.d/17732.bugfix b/changelog.d/17732.bugfix deleted file mode 100644 index 572c13fc57..0000000000 --- a/changelog.d/17732.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix membership caches not updating in state reset scenarios. diff --git a/changelog.d/18035.feature b/changelog.d/18035.feature deleted file mode 100644 index 99b68a9e45..0000000000 --- a/changelog.d/18035.feature +++ /dev/null @@ -1 +0,0 @@ -Add a unit test for the `type` parameter of the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api). \ No newline at end of file diff --git a/changelog.d/18052.removal b/changelog.d/18052.removal deleted file mode 100644 index f173b9ba4d..0000000000 --- a/changelog.d/18052.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the unstable [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151) implementation. The stable support remains, per [Matrix 1.13](https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv3roomsroomidreport). \ No newline at end of file diff --git a/changelog.d/18064.doc b/changelog.d/18064.doc deleted file mode 100644 index 09f73c9b9b..0000000000 --- a/changelog.d/18064.doc +++ /dev/null @@ -1 +0,0 @@ -Document `tls` option for a worker instance in `instance_map`. diff --git a/changelog.d/18072.misc b/changelog.d/18072.misc deleted file mode 100644 index d701768727..0000000000 --- a/changelog.d/18072.misc +++ /dev/null @@ -1 +0,0 @@ -Increase invite rate limits (`rc_invites.per_issuer`) for Complement. diff --git a/changelog.d/18091.bugfix b/changelog.d/18091.bugfix deleted file mode 100644 index c163b4f9a2..0000000000 --- a/changelog.d/18091.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rare race where on upgrade to v1.22.0 a long running database upgrade could lock out new events from being received or sent. diff --git a/changelog.d/18093.feature b/changelog.d/18093.feature deleted file mode 100644 index 689766ec0a..0000000000 --- a/changelog.d/18093.feature +++ /dev/null @@ -1 +0,0 @@ -Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 7e50f3ce94..2a18b776f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.123.0~rc1) stable; urgency=medium + + * New Synapse release 1.123.0rc1. + + -- Synapse Packaging team Tue, 21 Jan 2025 14:39:57 +0100 + matrix-synapse-py3 (1.122.0) stable; urgency=medium * New Synapse release 1.122.0. diff --git a/pyproject.toml b/pyproject.toml index c947a89f55..e6899207da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.122.0" +version = "1.123.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 8364c01a2bb97b75df1a63f80f1cd157fee0b96f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:58:20 +0100 Subject: [PATCH 099/691] Update changelog --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2c7763fbc0..599eff1dd7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,8 @@ ### Features -- Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. ([\#17488](https://github.com/element-hq/synapse/issues/17488)) -- Add a unit test for the `type` parameter of the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api). ([\#18035](https://github.com/element-hq/synapse/issues/18035)) +- Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. Contributed by @clokep. ([\#17488](https://github.com/element-hq/synapse/issues/17488)) +- Add a query param `type` to the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api) that filters the state event. ([\#18035](https://github.com/element-hq/synapse/issues/18035)) - Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). ([\#18093](https://github.com/element-hq/synapse/issues/18093)) ### Bugfixes From aa07a01452d737f650ca2d1424d1bad054e7c616 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:01:16 +0100 Subject: [PATCH 100/691] One more tiny change --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 599eff1dd7..f3ad507de8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ ### Features - Implement [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) for custom profile fields. Contributed by @clokep. ([\#17488](https://github.com/element-hq/synapse/issues/17488)) -- Add a query param `type` to the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api) that filters the state event. ([\#18035](https://github.com/element-hq/synapse/issues/18035)) +- Add a query parameter `type` to the [Room State Admin API](https://element-hq.github.io/synapse/develop/admin_api/rooms.html#room-state-api) that filters the state event. ([\#18035](https://github.com/element-hq/synapse/issues/18035)) - Support the new `/auth_metadata` endpoint defined in [MSC2965](https://github.com/matrix-org/matrix-spec-proposals/pull/2965). ([\#18093](https://github.com/element-hq/synapse/issues/18093)) ### Bugfixes From 90a6bd01c23fcaba1941ccd427793bd2a0130a64 Mon Sep 17 00:00:00 2001 From: Max Kratz Date: Tue, 21 Jan 2025 19:54:31 +0100 Subject: [PATCH 101/691] Contrib: Docker: updates PostgreSQL version in `docker-compose.yml` (#18089) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18089.bugfix | 2 ++ contrib/docker/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18089.bugfix diff --git a/changelog.d/18089.bugfix b/changelog.d/18089.bugfix new file mode 100644 index 0000000000..607fab7112 --- /dev/null +++ b/changelog.d/18089.bugfix @@ -0,0 +1,2 @@ +Updates contributed `docker-compose.yml` file to PostgreSQL v15, as v12 is no longer supported by Synapse. +Contributed by @maxkratz. \ No newline at end of file diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 36d5fd5309..9dffc852fd 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -51,7 +51,7 @@ services: - traefik.http.routers.https-synapse.tls.certResolver=le-ssl db: - image: docker.io/postgres:12-alpine + image: docker.io/postgres:15-alpine # Change that password, of course! environment: - POSTGRES_USER=synapse From 9c5d08fff8d66a7cc0e2ecfeeb783f933a778c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Fri, 24 Jan 2025 20:58:01 +0100 Subject: [PATCH 102/691] Ratelimit presence updates (#18000) --- changelog.d/18000.bugfix | 1 + .../conf/workers-shared-extra.yaml.j2 | 5 + .../configuration/config_documentation.md | 21 ++++ synapse/api/ratelimiting.py | 8 +- synapse/config/ratelimiting.py | 6 + synapse/rest/client/presence.py | 22 +++- synapse/rest/client/sync.py | 19 +++- tests/handlers/test_presence.py | 105 +++++++++++++++++- tests/rest/client/test_presence.py | 52 +++++++++ tests/utils.py | 1 + 10 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18000.bugfix diff --git a/changelog.d/18000.bugfix b/changelog.d/18000.bugfix new file mode 100644 index 0000000000..a8f1545bf5 --- /dev/null +++ b/changelog.d/18000.bugfix @@ -0,0 +1 @@ +Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 4c8fa65e62..797d58e9b3 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -89,6 +89,11 @@ rc_invites: per_second: 1000 burst_count: 1000 +rc_presence: + per_user: + per_second: 9999 + burst_count: 9999 + federation_rr_transactions_per_room_per_second: 9999 allow_device_name_lookup_over_federation: true diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 64392366ca..a1e671ab8e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1868,6 +1868,27 @@ rc_federation: concurrent: 5 ``` --- +### `rc_presence` + +This option sets ratelimiting for presence. + +The `rc_presence.per_user` option sets rate limits on how often a specific +users' presence updates are evaluated. Ratelimited presence updates sent via sync are +ignored, and no error is returned to the client. +This option also sets the rate limit for the +[`PUT /_matrix/client/v3/presence/{userId}/status`](https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3presenceuseridstatus) +endpoint. + +`per_user` defaults to `per_second: 0.1`, `burst_count: 1`. + +Example configuration: +```yaml +rc_presence: + per_user: + per_second: 0.05 + burst_count: 0.5 +``` +--- ### `federation_rr_transactions_per_room_per_second` Sets outgoing federation transaction frequency for sending read-receipts, diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index b80630c5d3..229329a5ae 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -275,6 +275,7 @@ class Ratelimiter: update: bool = True, n_actions: int = 1, _time_now_s: Optional[float] = None, + pause: Optional[float] = 0.5, ) -> None: """Checks if an action can be performed. If not, raises a LimitExceededError @@ -298,6 +299,8 @@ class Ratelimiter: at all. _time_now_s: The current time. Optional, defaults to the current time according to self.clock. Only used by tests. + pause: Time in seconds to pause when an action is being limited. Defaults to 0.5 + to stop clients from "tight-looping" on retrying their request. Raises: LimitExceededError: If an action could not be performed, along with the time in @@ -316,9 +319,8 @@ class Ratelimiter: ) if not allowed: - # We pause for a bit here to stop clients from "tight-looping" on - # retrying their request. - await self.clock.sleep(0.5) + if pause: + await self.clock.sleep(pause) raise LimitExceededError( limiter_name=self._limiter_name, diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 3fa33f5373..06af4da3c5 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -228,3 +228,9 @@ class RatelimitConfig(Config): config.get("remote_media_download_burst_count", "500M") ), ) + + self.rc_presence_per_user = RatelimitSettings.parse( + config, + "rc_presence.per_user", + defaults={"per_second": 0.1, "burst_count": 1}, + ) diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index ecc52956e4..104d54cd89 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -24,7 +24,8 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.api.errors import AuthError, SynapseError +from synapse.api.errors import AuthError, Codes, LimitExceededError, SynapseError +from synapse.api.ratelimiting import Ratelimiter from synapse.handlers.presence import format_user_presence_state from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request @@ -48,6 +49,14 @@ class PresenceStatusRestServlet(RestServlet): self.presence_handler = hs.get_presence_handler() self.clock = hs.get_clock() self.auth = hs.get_auth() + self.store = hs.get_datastores().main + + # Ratelimiter for presence updates, keyed by requester. + self._presence_per_user_limiter = Ratelimiter( + store=self.store, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_presence_per_user, + ) async def on_GET( self, request: SynapseRequest, user_id: str @@ -82,6 +91,17 @@ class PresenceStatusRestServlet(RestServlet): if requester.user != user: raise AuthError(403, "Can only set your own presence state") + # ignore the presence update if the ratelimit is exceeded + try: + await self._presence_per_user_limiter.ratelimit(requester) + except LimitExceededError as e: + logger.debug("User presence ratelimit exceeded; ignoring it.") + return 429, { + "errcode": Codes.LIMIT_EXCEEDED, + "error": "Too many requests", + "retry_after_ms": e.retry_after_ms, + } + state = {} content = parse_json_object_from_request(request) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index f4ef84a038..4fb9c0c8e7 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -24,9 +24,10 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState -from synapse.api.errors import Codes, StoreError, SynapseError +from synapse.api.errors import Codes, LimitExceededError, StoreError, SynapseError from synapse.api.filtering import FilterCollection from synapse.api.presence import UserPresenceState +from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( SerializeEventConfig, format_event_for_client_v2_without_room_id, @@ -126,6 +127,13 @@ class SyncRestServlet(RestServlet): cache_name="sync_valid_filter", ) + # Ratelimiter for presence updates, keyed by requester. + self._presence_per_user_limiter = Ratelimiter( + store=self.store, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_presence_per_user, + ) + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: # This will always be set by the time Twisted calls us. assert request.args is not None @@ -239,7 +247,14 @@ class SyncRestServlet(RestServlet): # send any outstanding server notices to the user. await self._server_notices_sender.on_user_syncing(user.to_string()) - affect_presence = set_presence != PresenceState.OFFLINE + # ignore the presence update if the ratelimit is exceeded but do not pause the request + try: + await self._presence_per_user_limiter.ratelimit(requester, pause=0.0) + except LimitExceededError: + affect_presence = False + logger.debug("User set_presence ratelimit exceeded; ignoring it.") + else: + affect_presence = set_presence != PresenceState.OFFLINE context = await self.presence_handler.user_syncing( user.to_string(), diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 598d6c13cd..4cf048f0df 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -45,7 +45,7 @@ from synapse.handlers.presence import ( handle_update, ) from synapse.rest import admin -from synapse.rest.client import room +from synapse.rest.client import login, room, sync from synapse.server import HomeServer from synapse.storage.database import LoggingDatabaseConnection from synapse.types import JsonDict, UserID, get_domain_from_id @@ -53,10 +53,15 @@ from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.unittest import override_config class PresenceUpdateTestCase(unittest.HomeserverTestCase): - servlets = [admin.register_servlets] + servlets = [ + admin.register_servlets, + login.register_servlets, + sync.register_servlets, + ] def prepare( self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer @@ -425,6 +430,102 @@ class PresenceUpdateTestCase(unittest.HomeserverTestCase): wheel_timer.insert.assert_not_called() + # `rc_presence` is set very high during unit tests to avoid ratelimiting + # subtly impacting unrelated tests. We set the ratelimiting back to a + # reasonable value for the tests specific to presence ratelimiting. + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_over_ratelimit_offline_to_online_to_unavailable(self) -> None: + """ + Send a presence update, check that it went through, immediately send another one and + check that it was ignored. + """ + self._test_ratelimit_offline_to_online_to_unavailable(ratelimited=True) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_within_ratelimit_offline_to_online_to_unavailable(self) -> None: + """ + Send a presence update, check that it went through, advancing time a sufficient amount, + send another presence update and check that it also worked. + """ + self._test_ratelimit_offline_to_online_to_unavailable(ratelimited=False) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def _test_ratelimit_offline_to_online_to_unavailable( + self, ratelimited: bool + ) -> None: + """Test rate limit for presence updates sent with sync requests. + + Args: + ratelimited: Test rate limited case. + """ + wheel_timer = Mock() + user_id = "@user:pass" + now = 5000000 + sync_url = "/sync?access_token=%s&set_presence=%s" + + # Register the user who syncs presence + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Get the handler (which kicks off a bunch of timers). + presence_handler = self.hs.get_presence_handler() + + # Ensure the user is initially offline. + prev_state = UserPresenceState.default(user_id) + new_state = prev_state.copy_and_replace( + state=PresenceState.OFFLINE, last_active_ts=now + ) + + state, persist_and_notify, federation_ping = handle_update( + prev_state, + new_state, + is_mine=True, + wheel_timer=wheel_timer, + now=now, + persist=False, + ) + + # Check that the user is offline. + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + # Send sync request with set_presence=online. + channel = self.make_request("GET", sync_url % (access_token, "online")) + self.assertEqual(200, channel.code) + + # Assert the user is now online. + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + self.assertEqual(state.state, PresenceState.ONLINE) + + if not ratelimited: + # Advance time a sufficient amount to avoid rate limiting. + self.reactor.advance(30) + + # Send another sync request with set_presence=unavailable. + channel = self.make_request("GET", sync_url % (access_token, "unavailable")) + self.assertEqual(200, channel.code) + + state = self.get_success( + presence_handler.get_state(UserID.from_string(user_id)) + ) + + if ratelimited: + # Assert the user is still online and presence update was ignored. + self.assertEqual(state.state, PresenceState.ONLINE) + else: + # Assert the user is now unavailable. + self.assertEqual(state.state, PresenceState.UNAVAILABLE) + class PresenceTimeoutTestCase(unittest.TestCase): """Tests different timers and that the timer does not change `status_msg` of user.""" diff --git a/tests/rest/client/test_presence.py b/tests/rest/client/test_presence.py index 5ced8319e1..6b9c70974a 100644 --- a/tests/rest/client/test_presence.py +++ b/tests/rest/client/test_presence.py @@ -29,6 +29,7 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest +from tests.unittest import override_config class PresenceTestCase(unittest.HomeserverTestCase): @@ -95,3 +96,54 @@ class PresenceTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertEqual(self.presence_handler.set_state.call_count, 0) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_put_presence_over_ratelimit(self) -> None: + """ + Multiple PUTs to the status endpoint without sufficient delay will be rate limited. + """ + self.hs.config.server.presence_enabled = True + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.TOO_MANY_REQUESTS) + self.assertEqual(self.presence_handler.set_state.call_count, 1) + + @override_config( + {"rc_presence": {"per_user": {"per_second": 0.1, "burst_count": 1}}} + ) + def test_put_presence_within_ratelimit(self) -> None: + """ + Multiple PUTs to the status endpoint with sufficient delay should all call set_state. + """ + self.hs.config.server.presence_enabled = True + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + + # Advance time a sufficient amount to avoid rate limiting. + self.reactor.advance(30) + + body = {"presence": "here", "status_msg": "beep boop"} + channel = self.make_request( + "PUT", "/presence/%s/status" % (self.user_id,), body + ) + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(self.presence_handler.set_state.call_count, 2) diff --git a/tests/utils.py b/tests/utils.py index 9fd26ef348..3f4a7bb560 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -200,6 +200,7 @@ def default_config( "per_user": {"per_second": 10000, "burst_count": 10000}, }, "rc_3pid_validation": {"per_second": 10000, "burst_count": 10000}, + "rc_presence": {"per_user": {"per_second": 10000, "burst_count": 10000}}, "saml2_enabled": False, "public_baseurl": None, "default_identity_server": None, From 56ed412839e23a8ee79142db5a7d679192851c96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:20:41 +0000 Subject: [PATCH 103/691] Bump dawidd6/action-download-artifact from 7 to 8 (#18108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [dawidd6/action-download-artifact](https://github.com/dawidd6/action-download-artifact) from 7 to 8.
Release notes

Sourced from dawidd6/action-download-artifact's releases.

v8

New features

  • use_unzip boolean input (defaulting to false) - if set to true, the action will use system provided unzip utility for unpacking downloaded artifact(s) (note that the action will first download the .zip artifact file, then unpack it and remove the .zip file)

What's Changed

New Contributors

Full Changelog: https://github.com/dawidd6/action-download-artifact/compare/v7...v8

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=dawidd6/action-download-artifact&package-manager=github_actions&previous-version=7&new-version=8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr-netlify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 3962f75055..0fbf6e02b7 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 with: workflow: docs-pr.yaml run_id: ${{ github.event.workflow_run.id }} From 148e93576e6a8dc588ce95f1d01888c391601b05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:23:28 +0000 Subject: [PATCH 104/691] Bump log from 0.4.22 to 0.4.25 (#18098) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [log](https://github.com/rust-lang/log) from 0.4.22 to 0.4.25.
Release notes

Sourced from log's releases.

0.4.25

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.24...0.4.25

0.4.24 (yanked)

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.23...0.4.24

0.4.23 (yanked)

What's Changed

New Contributors

Full Changelog: https://github.com/rust-lang/log/compare/0.4.22...0.4.23

Changelog

Sourced from log's changelog.

[0.4.25] - 2025-01-14

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.24...0.4.25

[0.4.24] - 2025-01-11

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.23...0.4.24

[0.4.23] - 2025-01-10 (yanked)

What's Changed

New Contributors

Full Changelog: https://github.com/rust-lang/log/compare/0.4.22...0.4.23

Commits
  • 22be810 Merge pull request #663 from rust-lang/cargo/0.4.25
  • 0279730 prepare for 0.4.25 release
  • 4099bcb Merge pull request #662 from rust-lang/fix/cargo-features
  • 36e7e3f revert loosening of kv cargo features
  • 2282191 Merge pull request #660 from rust-lang/cargo/0.4.24
  • 2994f0a prepare for 0.4.24 release
  • 5fcb50e Merge pull request #659 from rust-lang/fix/feature-builds
  • 29fe9e6 fix up feature activation
  • b1824f2 use cargo hack in CI to test all feature combinations
  • e6b643d Merge pull request #656 from rust-lang/cargo/0.4.23
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=log&package-manager=cargo&previous-version=0.4.22&new-version=0.4.25)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3cec1ea4a6..d7312c0125 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "memchr" From 6ec5e13ec94c484a573bad11e3d80fc6b4b2b943 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 27 Jan 2025 11:21:10 -0600 Subject: [PATCH 105/691] Fix join being denied after being invited over federation (#18075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also happens for rejecting an invite. Basically, any out-of-band membership transition where we first get the membership as an `outlier` and then rely on federation filling us in to de-outlier it. This PR mainly addresses automated test flakiness, bots/scripts, and options within Synapse like [`auto_accept_invites`](https://element-hq.github.io/synapse/v1.122/usage/configuration/config_documentation.html#auto_accept_invites) that are able to react quickly (before federation is able to push us events), but also helps in generic scenarios where federation is lagging. I initially thought this might be a Synapse consistency issue (see issues labeled with [`Z-Read-After-Write`](https://github.com/matrix-org/synapse/labels/Z-Read-After-Write)) but it seems to be an event auth logic problem. Workers probably do increase the number of possible race condition scenarios that make this visible though (replication and cache invalidation lag). Fix https://github.com/element-hq/synapse/issues/15012 (probably fixes https://github.com/matrix-org/synapse/issues/15012 (https://github.com/element-hq/synapse/issues/15012)) Related to https://github.com/matrix-org/matrix-spec/issues/2062 Problems: 1. We don't consider [out-of-band membership](https://github.com/element-hq/synapse/blob/develop/docs/development/room-dag-concepts.md#out-of-band-membership-events) (outliers) in our `event_auth` logic even though we expose them in `/sync`. 1. (This PR doesn't address this point) Perhaps we should consider authing events in the persistence queue as events already in the queue could allow subsequent events to be allowed (events come through many channels: federation transaction, remote invite, remote join, local send). But this doesn't save us in the case where the event is more delayed over federation. ### What happened before? I wrote some Complement test that stresses this exact scenario and reproduces the problem: https://github.com/matrix-org/complement/pull/757 ``` COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS=1 COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestSynapseConsistency ``` We have `hs1` and `hs2` running in monolith mode (no workers): 1. `@charlie1:hs2` is invited and joins the room: 1. `hs1` invites `@charlie1:hs2` to a room which we receive on `hs2` as `PUT /_matrix/federation/v1/invite/{roomId}/{eventId}` (`on_invite_request(...)`) and the invite membership is persisted as an outlier. The `room_memberships` and `local_current_membership` database tables are also updated which means they are visible down `/sync` at this point. 1. `@charlie1:hs2` decides to join because it saw the invite down `/sync`. Because `hs2` is not yet in the room, this happens as a remote join `make_join`/`send_join` which comes back with all of the auth events needed to auth successfully and now `@charlie1:hs2` is successfully joined to the room. 1. `@charlie2:hs2` is invited and and tries to join the room: 1. `hs1` invites `@charlie2:hs2` to the room which we receive on `hs2` as `PUT /_matrix/federation/v1/invite/{roomId}/{eventId}` (`on_invite_request(...)`) and the invite membership is persisted as an outlier. The `room_memberships` and `local_current_membership` database tables are also updated which means they are visible down `/sync` at this point. 1. Because `hs2` is already participating in the room, we also see the invite come over federation in a transaction and we start processing it (not done yet, see below) 1. `@charlie2:hs2` decides to join because it saw the invite down `/sync`. Because `hs2`, is already in the room, this happens as a local join but we deny the event because our `event_auth` logic thinks that we have no membership in the room :x: (expected to be able to join because we saw the invite down `/sync`) 1. We finally finish processing the `@charlie2:hs2` invite event from and de-outlier it. - If this finished before we tried to join we would have been fine but this is the race condition that makes this situation visible. Logs for `hs2`: ``` 🗳️ on_invite_request: handling event 🔦 _store_room_members_txn update room_memberships: 🔦 _store_room_members_txn update local_current_membership: 📨 Notifying about new event ✅ on_invite_request: handled event 🧲 do_invite_join for @user-2-charlie1:hs2 in !sfZVBdLUezpPWetrol:hs1 🔦 _store_room_members_txn update room_memberships: 🔦 _store_room_members_txn update room_memberships: 📨 Notifying about new event ... 🗳️ on_invite_request: handling event 🔦 _store_room_members_txn update room_memberships: 🔦 _store_room_members_txn update local_current_membership: 📨 Notifying about new event ✅ on_invite_request: handled event 📬 handling received PDU in room !sfZVBdLUezpPWetrol:hs1: 📮 handle_new_client_event: handling ❌ Denying new event because 403: You are not invited to this room. synapse.http.server - 130 - INFO - POST-16 - SynapseError: 403 - You are not invited to this room. 📨 Notifying about new event ✅ handled received PDU in room !sfZVBdLUezpPWetrol:hs1: ``` --- changelog.d/18075.bugfix | 1 + synapse/event_auth.py | 4 +- synapse/events/__init__.py | 7 +- synapse/events/builder.py | 55 +- synapse/handlers/federation_event.py | 5 +- synapse/server.py | 4 +- tests/federation/test_federation_devices.py | 161 +++++ .../test_federation_out_of_band_membership.py | 671 ++++++++++++++++++ tests/federation/test_federation_server.py | 247 ++++++- tests/handlers/test_federation_event.py | 2 +- tests/handlers/test_presence.py | 104 ++- tests/handlers/test_sync.py | 11 +- .../test_federation_sender_shard.py | 110 ++- tests/rest/client/test_rooms.py | 4 +- tests/test_federation.py | 378 ---------- tests/utils.py | 19 +- 16 files changed, 1341 insertions(+), 442 deletions(-) create mode 100644 changelog.d/18075.bugfix create mode 100644 tests/federation/test_federation_devices.py create mode 100644 tests/federation/test_federation_out_of_band_membership.py delete mode 100644 tests/test_federation.py diff --git a/changelog.d/18075.bugfix b/changelog.d/18075.bugfix new file mode 100644 index 0000000000..95b486bed1 --- /dev/null +++ b/changelog.d/18075.bugfix @@ -0,0 +1 @@ +Fix join being denied after being invited over federation. Also fixes other out-of-band membership transitions. diff --git a/synapse/event_auth.py b/synapse/event_auth.py index c208b900c5..3fe344ac93 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -566,6 +566,7 @@ def _is_membership_change_allowed( logger.debug( "_is_membership_change_allowed: %s", { + "caller_membership": caller.membership if caller else None, "caller_in_room": caller_in_room, "caller_invited": caller_invited, "caller_knocked": caller_knocked, @@ -677,7 +678,8 @@ def _is_membership_change_allowed( and join_rule == JoinRules.KNOCK_RESTRICTED ) ): - if not caller_in_room and not caller_invited: + # You can only join the room if you are invited or are already in the room. + if not (caller_in_room or caller_invited): raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 2e56b671f0..8e9d27138c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -42,7 +42,7 @@ import attr from typing_extensions import Literal from unpaddedbase64 import encode_base64 -from synapse.api.constants import RelationTypes +from synapse.api.constants import EventTypes, RelationTypes from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.synapse_rust.events import EventInternalMetadata from synapse.types import JsonDict, StrCollection @@ -325,12 +325,17 @@ class EventBase(metaclass=abc.ABCMeta): def __repr__(self) -> str: rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" + conditional_membership_string = "" + if self.get("type") == EventTypes.Member: + conditional_membership_string = f"membership={self.membership}, " + return ( f"<{self.__class__.__name__} " f"{rejection}" f"event_id={self.event_id}, " f"type={self.get('type')}, " f"state_key={self.get('state_key')}, " + f"{conditional_membership_string}" f"outlier={self.internal_metadata.is_outlier()}" ">" ) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 10ef01131b..76df083d69 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import attr from signedjson.types import SigningKey -from synapse.api.constants import MAX_DEPTH +from synapse.api.constants import MAX_DEPTH, EventTypes from synapse.api.room_versions import ( KNOWN_EVENT_FORMAT_VERSIONS, EventFormatVersions, @@ -109,6 +109,19 @@ class EventBuilder: def is_state(self) -> bool: return self._state_key is not None + def is_mine_id(self, user_id: str) -> bool: + """Determines whether a user ID or room alias originates from this homeserver. + + Returns: + `True` if the hostname part of the user ID or room alias matches this + homeserver. + `False` otherwise, or if the user ID or room alias is malformed. + """ + localpart_hostname = user_id.split(":", 1) + if len(localpart_hostname) < 2: + return False + return localpart_hostname[1] == self._hostname + async def build( self, prev_event_ids: List[str], @@ -142,6 +155,46 @@ class EventBuilder: self, state_ids ) + # Check for out-of-band membership that may have been exposed on `/sync` but + # the events have not been de-outliered yet so they won't be part of the + # room state yet. + # + # This helps in situations where a remote homeserver invites a local user to + # a room that we're already participating in; and we've persisted the invite + # as an out-of-band membership (outlier), but it hasn't been pushed to us as + # part of a `/send` transaction yet and de-outliered. This also helps for + # any of the other out-of-band membership transitions. + # + # As an optimization, we could check if the room state already includes a + # non-`leave` membership event, then we can assume the membership event has + # been de-outliered and we don't need to check for an out-of-band + # membership. But we don't have the necessary information from a + # `StateMap[str]` and we'll just have to take the hit of this extra lookup + # for any membership event for now. + if self.type == EventTypes.Member and self.is_mine_id(self.state_key): + ( + _membership, + member_event_id, + ) = await self._store.get_local_current_membership_for_user_in_room( + user_id=self.state_key, + room_id=self.room_id, + ) + # There is no need to check if the membership is actually an + # out-of-band membership (`outlier`) as we would end up with the + # same result either way (adding the member event to the + # `auth_event_ids`). + if ( + member_event_id is not None + # We only need to be careful about duplicating the event in the + # `auth_event_ids` list (duplicate `type`/`state_key` is part of the + # authorization rules) + and member_event_id not in auth_event_ids + ): + auth_event_ids.append(member_event_id) + # Also make sure to point to the previous membership event that will + # allow this one to happen so the computed state works out. + prev_event_ids.append(member_event_id) + format_version = self.room_version.event_format # The types of auth/prev events changes between event versions. prev_events: Union[StrCollection, List[Tuple[str, Dict[str, str]]]] diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index c85deaed56..1b535ea2cb 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -2272,8 +2272,9 @@ class FederationEventHandler: event_and_contexts, backfilled=backfilled ) - # After persistence we always need to notify replication there may - # be new data. + # After persistence, we never notify clients (wake up `/sync` streams) about + # backfilled events but it's important to let all the workers know about any + # new event (backfilled or not) because TODO self._notifier.notify_replication() if self._ephemeral_messages_enabled: diff --git a/synapse/server.py b/synapse/server.py index 462e15cc2f..bd2faa61b9 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -391,7 +391,7 @@ class HomeServer(metaclass=abc.ABCMeta): def is_mine(self, domain_specific_string: DomainSpecificString) -> bool: return domain_specific_string.domain == self.hostname - def is_mine_id(self, string: str) -> bool: + def is_mine_id(self, user_id: str) -> bool: """Determines whether a user ID or room alias originates from this homeserver. Returns: @@ -399,7 +399,7 @@ class HomeServer(metaclass=abc.ABCMeta): homeserver. `False` otherwise, or if the user ID or room alias is malformed. """ - localpart_hostname = string.split(":", 1) + localpart_hostname = user_id.split(":", 1) if len(localpart_hostname) < 2: return False return localpart_hostname[1] == self.hostname diff --git a/tests/federation/test_federation_devices.py b/tests/federation/test_federation_devices.py new file mode 100644 index 0000000000..ba27e69479 --- /dev/null +++ b/tests/federation/test_federation_devices.py @@ -0,0 +1,161 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +from unittest.mock import AsyncMock, Mock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.handlers.device import DeviceListUpdater +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock +from synapse.util.retryutils import NotRetryingDestination + +from tests import unittest + +logger = logging.getLogger(__name__) + + +class DeviceListResyncTestCase(unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + + def test_retry_device_list_resync(self) -> None: + """Tests that device lists are marked as stale if they couldn't be synced, and + that stale device lists are retried periodically. + """ + remote_user_id = "@john:test_remote" + remote_origin = "test_remote" + + # Track the number of attempts to resync the user's device list. + self.resync_attempts = 0 + + # When this function is called, increment the number of resync attempts (only if + # we're querying devices for the right user ID), then raise a + # NotRetryingDestination error to fail the resync gracefully. + def query_user_devices( + destination: str, user_id: str, timeout: int = 30000 + ) -> JsonDict: + if user_id == remote_user_id: + self.resync_attempts += 1 + + raise NotRetryingDestination(0, 0, destination) + + # Register the mock on the federation client. + federation_client = self.hs.get_federation_client() + federation_client.query_user_devices = Mock(side_effect=query_user_devices) # type: ignore[method-assign] + + # Register a mock on the store so that the incoming update doesn't fail because + # we don't share a room with the user. + self.store.get_rooms_for_user = AsyncMock(return_value=["!someroom:test"]) + + # Manually inject a fake device list update. We need this update to include at + # least one prev_id so that the user's device list will need to be retried. + device_list_updater = self.hs.get_device_handler().device_list_updater + assert isinstance(device_list_updater, DeviceListUpdater) + self.get_success( + device_list_updater.incoming_device_list_update( + origin=remote_origin, + edu_content={ + "deleted": False, + "device_display_name": "Mobile", + "device_id": "QBUAZIFURK", + "prev_id": [5], + "stream_id": 6, + "user_id": remote_user_id, + }, + ) + ) + + # Check that there was one resync attempt. + self.assertEqual(self.resync_attempts, 1) + + # Check that the resync attempt failed and caused the user's device list to be + # marked as stale. + need_resync = self.get_success( + self.store.get_user_ids_requiring_device_list_resync() + ) + self.assertIn(remote_user_id, need_resync) + + # Check that waiting for 30 seconds caused Synapse to retry resyncing the device + # list. + self.reactor.advance(30) + self.assertEqual(self.resync_attempts, 2) + + def test_cross_signing_keys_retry(self) -> None: + """Tests that resyncing a device list correctly processes cross-signing keys from + the remote server. + """ + remote_user_id = "@john:test_remote" + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + # Register mock device list retrieval on the federation client. + federation_client = self.hs.get_federation_client() + federation_client.query_user_devices = AsyncMock( # type: ignore[method-assign] + return_value={ + "user_id": remote_user_id, + "stream_id": 1, + "devices": [], + "master_key": { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + "self_signing_key": { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + }, + } + ) + + # Resync the device list. + device_handler = self.hs.get_device_handler() + self.get_success( + device_handler.device_list_updater.multi_user_device_resync( + [remote_user_id] + ), + ) + + # Retrieve the cross-signing keys for this user. + keys = self.get_success( + self.store.get_e2e_cross_signing_keys_bulk(user_ids=[remote_user_id]), + ) + self.assertIn(remote_user_id, keys) + key = keys[remote_user_id] + assert key is not None + + # Check that the master key is the one returned by the mock. + master_key = key["master"] + self.assertEqual(len(master_key["keys"]), 1) + self.assertTrue("ed25519:" + remote_master_key in master_key["keys"].keys()) + self.assertTrue(remote_master_key in master_key["keys"].values()) + + # Check that the self-signing key is the one returned by the mock. + self_signing_key = key["self_signing"] + self.assertEqual(len(self_signing_key["keys"]), 1) + self.assertTrue( + "ed25519:" + remote_self_signing_key in self_signing_key["keys"].keys(), + ) + self.assertTrue(remote_self_signing_key in self_signing_key["keys"].values()) diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py new file mode 100644 index 0000000000..a4a266cf06 --- /dev/null +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -0,0 +1,671 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2020 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +import time +import urllib.parse +from http import HTTPStatus +from typing import Any, Callable, Optional, Set, Tuple, TypeVar, Union +from unittest.mock import Mock + +import attr +from parameterized import parameterized + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import EventContentFields, EventTypes, Membership +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase, make_event_from_dict +from synapse.events.utils import strip_event +from synapse.federation.federation_base import ( + event_from_pdu_json, +) +from synapse.federation.transport.client import SendJoinResponse +from synapse.http.matrixfederationclient import ( + ByteParser, +) +from synapse.http.types import QueryParams +from synapse.rest import admin +from synapse.rest.client import login, room, sync +from synapse.server import HomeServer +from synapse.types import JsonDict, MutableStateMap, StateMap +from synapse.types.handlers.sliding_sync import ( + StateValues, +) +from synapse.util import Clock + +from tests import unittest +from tests.utils import test_timeout + +logger = logging.getLogger(__name__) + + +def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase]: + state_map: MutableStateMap[EventBase] = {} + + # Scrutinize JSON values to ensure it's in the expected format + if isinstance(required_state, list): + for state_event_dict in required_state: + # Yell because we're in a test and this is unexpected + assert isinstance( + state_event_dict, dict + ), "`required_state` should be a list of event dicts" + + event_type = state_event_dict["type"] + event_state_key = state_event_dict["state_key"] + + # Yell because we're in a test and this is unexpected + assert isinstance( + event_type, str + ), "Each event in `required_state` should have a string `type`" + assert isinstance( + event_state_key, str + ), "Each event in `required_state` should have a string `state_key`" + + state_map[(event_type, event_state_key)] = make_event_from_dict( + state_event_dict + ) + else: + # Yell because we're in a test and this is unexpected + raise AssertionError("`required_state` should be a list of event dicts") + + return state_map + + +@attr.s(slots=True, auto_attribs=True) +class RemoteRoomJoinResult: + remote_room_id: str + room_version: RoomVersion + remote_room_creator_user_id: str + local_user1_id: str + local_user1_tok: str + state_map: StateMap[EventBase] + + +class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): + """ + Tests to make sure that interactions with out-of-band membership (outliers) works as + expected. + + - invites received over federation, before we join the room + - *rejections* for said invites + + See the "Out-of-band membership events" section in + `docs/development/room-dag-concepts.md` for more information. + """ + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + sync_endpoint = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync" + + def default_config(self) -> JsonDict: + conf = super().default_config() + # Federation sending is disabled by default in the test environment + # so we need to enable it like this. + conf["federation_sender_instances"] = ["master"] + + return conf + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.federation_http_client = Mock( + # The problem with using `spec=MatrixFederationHttpClient` here is that it + # requires everything to be mocked which is a lot of work that I don't want + # to do when the code only uses a few methods (`get_json` and `put_json`). + ) + return self.setup_test_homeserver( + federation_http_client=self.federation_http_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + + def do_sync( + self, sync_body: JsonDict, *, since: Optional[str] = None, tok: str + ) -> Tuple[JsonDict, str]: + """Do a sliding sync request with given body. + + Asserts the request was successful. + + Attributes: + sync_body: The full request body to use + since: Optional since token + tok: Access token to use + + Returns: + A tuple of the response body and the `pos` field. + """ + + sync_path = self.sync_endpoint + if since: + sync_path += f"?pos={since}" + + channel = self.make_request( + method="POST", + path=sync_path, + content=sync_body, + access_token=tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + return channel.json_body, channel.json_body["pos"] + + def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: + """ + Helper to reproduce this scenario: + + 1. The remote user invites our local user to a room on their remote server (which + creates an out-of-band invite membership for user1 on our local server). + 2. The local user notices the invite from `/sync`. + 3. The local user joins the room. + 4. The local user can see that they are now joined to the room from `/sync`. + """ + + # Create a local user + local_user1_id = self.register_user("user1", "pass") + local_user1_tok = self.login(local_user1_id, "pass") + + # Create a remote room + room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" + remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" + room_version = RoomVersions.V10 + + room_create_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 1, + "origin_server_ts": 1, + "type": EventTypes.Create, + "state_key": "", + "content": { + # The `ROOM_CREATOR` field could be removed if we used a room + # version > 10 (in favor of relying on `sender`) + EventContentFields.ROOM_CREATOR: room_creator_user_id, + EventContentFields.ROOM_VERSION: room_version.identifier, + }, + "auth_events": [], + "prev_events": [], + } + ), + room_version=room_version, + ) + + creator_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 2, + "origin_server_ts": 2, + "type": EventTypes.Member, + "state_key": room_creator_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [room_create_event.event_id], + "prev_events": [room_create_event.event_id], + } + ), + room_version=room_version, + ) + + # From the remote homeserver, invite user1 on the local homserver + user1_invite_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": room_creator_user_id, + "depth": 3, + "origin_server_ts": 3, + "type": EventTypes.Member, + "state_key": local_user1_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ], + "prev_events": [creator_membership_event.event_id], + } + ), + room_version=room_version, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{user1_invite_membership_event.event_id}", + content={ + "event": user1_invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event(room_create_event), + ], + "room_version": room_version.identifier, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.WILDCARD)], + "timeline_limit": 0, + } + } + } + + # Sync until the local user1 can see the invite + with test_timeout( + 3, + "Unable to find user1's invite event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user1_tok) + if ( + remote_room_id in response_body["rooms"].keys() + # If they have `invite_state` for the room, they are invited + and len( + response_body["rooms"][remote_room_id].get("invite_state", []) + ) + > 0 + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + user1_join_membership_event_template = make_event_from_dict( + { + "room_id": remote_room_id, + "sender": local_user1_id, + "depth": 4, + "origin_server_ts": 4, + "type": EventTypes.Member, + "state_key": local_user1_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + room_create_event.event_id, + user1_invite_membership_event.event_id, + ], + "prev_events": [user1_invite_membership_event.event_id], + }, + room_version=room_version, + ) + + T = TypeVar("T") + + # Mock the remote homeserver responding to our HTTP requests + # + # We're going to mock the following endpoints so that user1 can join the remote room: + # - GET /_matrix/federation/v1/make_join/{room_id}/{user_id} + # - PUT /_matrix/federation/v2/send_join/{room_id}/{user_id} + # + async def get_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + retry_on_dns_fail: bool = True, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + ) -> Union[JsonDict, T]: + if ( + path + == f"/_matrix/federation/v1/make_join/{urllib.parse.quote_plus(remote_room_id)}/{urllib.parse.quote_plus(local_user1_id)}" + ): + return { + "event": user1_join_membership_event_template.get_pdu_json(), + "room_version": room_version.identifier, + } + + raise NotImplementedError( + "We have not mocked a response for `get_json(...)` for the following endpoint yet: " + + f"{destination}{path}" + ) + + self.federation_http_client.get_json.side_effect = get_json + + # PDU's that hs1 sent to hs2 + collected_pdus_from_hs1_federation_send: Set[str] = set() + + async def put_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + data: Optional[JsonDict] = None, + json_data_callback: Optional[Callable[[], JsonDict]] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + backoff_on_all_error_codes: bool = False, + ) -> Union[JsonDict, T, SendJoinResponse]: + if ( + path.startswith( + f"/_matrix/federation/v2/send_join/{urllib.parse.quote_plus(remote_room_id)}/" + ) + and data is not None + and data.get("type") == EventTypes.Member + and data.get("state_key") == local_user1_id + # We're assuming this is a `ByteParser[SendJoinResponse]` + and parser is not None + ): + # As the remote server, we need to sign the event before sending it back + user1_join_membership_event_signed = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server(data), + room_version=room_version, + ) + + # Since they passed in a `parser`, we need to return the type that + # they're expecting instead of just a `JsonDict` + return SendJoinResponse( + auth_events=[ + room_create_event, + user1_invite_membership_event, + ], + state=[ + room_create_event, + creator_membership_event, + user1_invite_membership_event, + ], + event_dict=user1_join_membership_event_signed.get_pdu_json(), + event=user1_join_membership_event_signed, + members_omitted=False, + servers_in_room=[ + self.OTHER_SERVER_NAME, + ], + ) + + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + for pdu in data.get("pdus", []): + event = event_from_pdu_json(pdu, room_version) + collected_pdus_from_hs1_federation_send.add(event.event_id) + + # Just acknowledge everything hs1 is trying to send hs2 + return { + event_from_pdu_json(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self.federation_http_client.put_json.side_effect = put_json + + # User1 joins the room + self.helper.join(remote_room_id, local_user1_id, tok=local_user1_tok) + + # Reset the mocks now that user1 has joined the room + self.federation_http_client.get_json.side_effect = None + self.federation_http_client.put_json.side_effect = None + + # Sync until the local user1 can see that they are now joined to the room + with test_timeout( + 3, + "Unable to find user1's join event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user1_tok) + if remote_room_id in response_body["rooms"].keys(): + required_state_map = required_state_json_to_state_map( + response_body["rooms"][remote_room_id]["required_state"] + ) + if ( + required_state_map.get((EventTypes.Member, local_user1_id)) + is not None + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + # Nothing needs to be sent from hs1 to hs2 since we already let the other + # homeserver know by doing the `/make_join` and `/send_join` dance. + self.assertIncludes( + collected_pdus_from_hs1_federation_send, + set(), + exact=True, + message="Didn't expect any events to be sent from hs1 over federation to hs2", + ) + + return RemoteRoomJoinResult( + remote_room_id=remote_room_id, + room_version=room_version, + remote_room_creator_user_id=room_creator_user_id, + local_user1_id=local_user1_id, + local_user1_tok=local_user1_tok, + state_map=self.get_success( + self.storage_controllers.state.get_current_state(remote_room_id) + ), + ) + + def test_can_join_from_out_of_band_invite(self) -> None: + """ + Test to make sure that we can join a room that we were invited to over + federation; even if our server has never participated in the room before. + """ + self._invite_local_user_to_remote_room_and_join() + + @parameterized.expand( + [("accept invite", Membership.JOIN), ("reject invite", Membership.LEAVE)] + ) + def test_can_x_from_out_of_band_invite_after_we_are_already_participating_in_the_room( + self, _test_description: str, membership_action: str + ) -> None: + """ + Test to make sure that we can do either a) join the room (accept the invite) or + b) reject the invite after being invited to over federation; even if we are + already participating in the room. + + This is a regression test to make sure we stress the scenario where even though + we are already participating in the room, local users can still react to invites + regardless of whether the remote server has told us about the invite event (via + a federation `/send` transaction) and we have de-outliered the invite event. + Previously, we would mistakenly throw an error saying the user wasn't in the + room when they tried to join or reject the invite. + """ + remote_room_join_result = self._invite_local_user_to_remote_room_and_join() + remote_room_id = remote_room_join_result.remote_room_id + room_version = remote_room_join_result.room_version + + # Create another local user + local_user2_id = self.register_user("user2", "pass") + local_user2_tok = self.login(local_user2_id, "pass") + + T = TypeVar("T") + + # PDU's that hs1 sent to hs2 + collected_pdus_from_hs1_federation_send: Set[str] = set() + + async def put_json( + destination: str, + path: str, + args: Optional[QueryParams] = None, + data: Optional[JsonDict] = None, + json_data_callback: Optional[Callable[[], JsonDict]] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: Optional[ByteParser[T]] = None, + backoff_on_all_error_codes: bool = False, + ) -> Union[JsonDict, T]: + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + for pdu in data.get("pdus", []): + event = event_from_pdu_json(pdu, room_version) + collected_pdus_from_hs1_federation_send.add(event.event_id) + + # Just acknowledge everything hs1 is trying to send hs2 + return { + event_from_pdu_json(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self.federation_http_client.put_json.side_effect = put_json + + # From the remote homeserver, invite user2 on the local homserver + user2_invite_membership_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": remote_room_join_result.remote_room_creator_user_id, + "depth": 5, + "origin_server_ts": 5, + "type": EventTypes.Member, + "state_key": local_user2_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + remote_room_join_result.state_map[ + (EventTypes.Create, "") + ].event_id, + remote_room_join_result.state_map[ + ( + EventTypes.Member, + remote_room_join_result.remote_room_creator_user_id, + ) + ].event_id, + ], + "prev_events": [ + remote_room_join_result.state_map[ + (EventTypes.Member, remote_room_join_result.local_user1_id) + ].event_id + ], + } + ), + room_version=room_version, + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{user2_invite_membership_event.event_id}", + content={ + "event": user2_invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event( + remote_room_join_result.state_map[(EventTypes.Create, "")] + ), + ], + "room_version": room_version.identifier, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.WILDCARD)], + "timeline_limit": 0, + } + } + } + + # Sync until the local user2 can see the invite + with test_timeout( + 3, + "Unable to find user2's invite event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user2_tok) + if ( + remote_room_id in response_body["rooms"].keys() + # If they have `invite_state` for the room, they are invited + and len( + response_body["rooms"][remote_room_id].get("invite_state", []) + ) + > 0 + ): + break + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + if membership_action == Membership.JOIN: + # User2 joins the room + join_event = self.helper.join( + remote_room_join_result.remote_room_id, + local_user2_id, + tok=local_user2_tok, + ) + expected_pdu_event_id = join_event["event_id"] + elif membership_action == Membership.LEAVE: + # User2 rejects the invite + leave_event = self.helper.leave( + remote_room_join_result.remote_room_id, + local_user2_id, + tok=local_user2_tok, + ) + expected_pdu_event_id = leave_event["event_id"] + else: + raise NotImplementedError( + "This test does not support this membership action yet" + ) + + # Sync until the local user2 can see their new membership in the room + with test_timeout( + 3, + "Unable to find user2's new membership event in the room", + ): + while True: + response_body, _ = self.do_sync(sync_body, tok=local_user2_tok) + if membership_action == Membership.JOIN: + if remote_room_id in response_body["rooms"].keys(): + required_state_map = required_state_json_to_state_map( + response_body["rooms"][remote_room_id]["required_state"] + ) + if ( + required_state_map.get((EventTypes.Member, local_user2_id)) + is not None + ): + break + elif membership_action == Membership.LEAVE: + if remote_room_id not in response_body["rooms"].keys(): + break + else: + raise NotImplementedError( + "This test does not support this membership action yet" + ) + + # Prevent tight-looping to allow the `test_timeout` to work + time.sleep(0.1) + + # Make sure that we let hs2 know about the new membership event + self.assertIncludes( + collected_pdus_from_hs1_federation_send, + {expected_pdu_event_id}, + exact=True, + message="Expected to find the event ID of the user2 membership to be sent from hs1 over federation to hs2", + ) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 88261450b1..42dc844734 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -20,14 +20,21 @@ # import logging from http import HTTPStatus +from typing import Optional, Union +from unittest.mock import Mock from parameterized import parameterized from twisted.test.proto_helpers import MemoryReactor -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.api.constants import EventTypes, Membership +from synapse.api.errors import FederationError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.events import EventBase, make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json +from synapse.http.types import QueryParams +from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer @@ -85,6 +92,163 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(500, channel.code, channel.result) +def _create_acl_event(content: JsonDict) -> EventBase: + return make_event_from_dict( + { + "room_id": "!a:b", + "event_id": "$a:b", + "type": "m.room.server_acls", + "sender": "@a:b", + "content": content, + } + ) + + +class MessageAcceptTests(unittest.FederatingHomeserverTestCase): + """ + Tests to make sure that we don't accept flawed events from federation (incoming). + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.http_client = Mock() + return self.setup_test_homeserver(federation_http_client=self.http_client) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + self.federation_event_handler = self.hs.get_federation_event_handler() + + # Create a local room + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + self.room_id = self.helper.create_room_as( + user1_id, tok=user1_tok, is_public=True + ) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(self.room_id) + ) + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + + # Join a remote user to the room that will attempt to send bad events + self.remote_bad_user_id = f"@baduser:{self.OTHER_SERVER_NAME}" + self.remote_bad_user_join_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_bad_user_id, + "state_key": self.remote_bad_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Send the join, it should return None (which is not an error) + self.assertEqual( + self.get_success( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, self.remote_bad_user_join_event + ) + ), + None, + ) + + # Make sure we actually joined the room + self.assertEqual( + self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)), + {self.remote_bad_user_join_event.event_id}, + ) + + def test_cant_hide_direct_ancestors(self) -> None: + """ + If you send a message, you must be able to provide the direct + prev_events that said event references. + """ + + async def post_json( + destination: str, + path: str, + data: Optional[JsonDict] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + args: Optional[QueryParams] = None, + ) -> Union[JsonDict, list]: + # If it asks us for new missing events, give them NOTHING + if path.startswith("/_matrix/federation/v1/get_missing_events/"): + return {"events": []} + return {} + + self.http_client.post_json = post_json + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + + # Now lie about an event's prev_events + lying_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_bad_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.message", + "content": {"body": "hewwo?"}, + "auth_events": [], + "prev_events": ["$missing_prev_event"] + + list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + with LoggingContext("test-context"): + failure = self.get_failure( + self.federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, lying_event + ), + FederationError, + ) + + # on_receive_pdu should throw an error + self.assertEqual( + failure.value.args[0], + ( + "ERROR 403: Your server isn't divulging details about prev_events " + "referenced in this event." + ), + ) + + # Make sure the invalid event isn't there + extrem = self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)) + self.assertEqual(extrem, {self.remote_bad_user_join_event.event_id}) + + class ServerACLsTestCase(unittest.TestCase): def test_blocked_server(self) -> None: e = _create_acl_event({"allow": ["*"], "deny": ["evil.com"]}) @@ -355,13 +519,76 @@ class SendJoinFederationTests(unittest.FederatingHomeserverTestCase): # is probably sufficient to reassure that the bucket is updated. -def _create_acl_event(content: JsonDict) -> EventBase: - return make_event_from_dict( - { - "room_id": "!a:b", - "event_id": "$a:b", - "type": "m.room.server_acls", - "sender": "@a:b", - "content": content, +class StripUnsignedFromEventsTestCase(unittest.TestCase): + """ + Test to make sure that we handle the raw JSON events from federation carefully and + strip anything that shouldn't be there. + """ + + def test_strip_unauthorized_unsigned_values(self) -> None: + event1 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event1:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "origin": "test.servx", + "content": {"membership": "join"}, + "auth_events": [], + "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, } - ) + filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + # Make sure unauthorized fields are stripped from unsigned + self.assertNotIn("more warez", filtered_event.unsigned) + + def test_strip_event_maintains_allowed_fields(self) -> None: + event2 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event2:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "origin": "test.servx", + "auth_events": [], + "content": {"membership": "join"}, + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) + self.assertIn("age", filtered_event2.unsigned) + self.assertEqual(14, filtered_event2.unsigned["age"]) + self.assertNotIn("more warez", filtered_event2.unsigned) + # Invite_room_state is allowed in events of type m.room.member + self.assertIn("invite_room_state", filtered_event2.unsigned) + self.assertEqual([], filtered_event2.unsigned["invite_room_state"]) + + def test_strip_event_removes_fields_based_on_event_type(self) -> None: + event3 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event3:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.power_levels", + "origin": "test.servx", + "content": {}, + "auth_events": [], + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) + self.assertIn("age", filtered_event3.unsigned) + # Invite_room_state field is only permitted in event type m.room.member + self.assertNotIn("invite_room_state", filtered_event3.unsigned) + self.assertNotIn("more warez", filtered_event3.unsigned) diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 5db10fa74c..61b0efb87e 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -375,7 +375,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): In this test, we pretend we are processing a "pulled" event via backfill. The pulled event succesfully processes and the backward - extremeties are updated along with clearing out any failed pull attempts + extremities are updated along with clearing out any failed pull attempts for those old extremities. We check that we correctly cleared failed pull attempts of the diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 4cf048f0df..6b7bf112c2 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -23,14 +23,21 @@ from typing import Optional, cast from unittest.mock import Mock, call from parameterized import parameterized -from signedjson.key import generate_signing_key +from signedjson.key import ( + encode_verify_key_base64, + generate_signing_key, + get_verify_key, +) from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes, Membership, PresenceState from synapse.api.presence import UserDevicePresenceState, UserPresenceState -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events.builder import EventBuilder +from synapse.api.room_versions import ( + RoomVersion, +) +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, make_event_from_dict from synapse.federation.sender import FederationSender from synapse.handlers.presence import ( BUSY_ONLINE_TIMEOUT, @@ -48,6 +55,7 @@ from synapse.rest import admin from synapse.rest.client import login, room, sync from synapse.server import HomeServer from synapse.storage.database import LoggingDatabaseConnection +from synapse.storage.keys import FetchKeyResult from synapse.types import JsonDict, UserID, get_domain_from_id from synapse.util import Clock @@ -1926,6 +1934,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): # self.event_builder_for_2.hostname = "test2" self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -2041,29 +2050,35 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): hostname = get_domain_from_id(user_id) - room_version = self.get_success(self.store.get_room_version_id(room_id)) + room_version = self.get_success(self.store.get_room_version(room_id)) - builder = EventBuilder( - state=self.state, - event_auth_handler=self._event_auth_handler, - store=self.store, - clock=self.clock, - hostname=hostname, - signing_key=self.random_signing_key, - room_version=KNOWN_ROOM_VERSIONS[room_version], - room_id=room_id, - type=EventTypes.Member, - sender=user_id, - state_key=user_id, - content={"membership": Membership.JOIN}, + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) ) - prev_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(room_id) + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(room_id) ) - event = self.get_success( - builder.build(prev_event_ids=list(prev_event_ids), auth_event_ids=None) + event = self.create_fake_event_from_remote_server( + remote_server_name=hostname, + event_dict={ + "room_id": room_id, + "sender": user_id, + "type": EventTypes.Member, + "state_key": user_id, + "depth": 1000, + "origin_server_ts": 1, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + }, + room_version=room_version, ) self.get_success(self.federation_event_handler.on_receive_pdu(hostname, event)) @@ -2071,3 +2086,50 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): # Check that it was successfully persisted. self.get_success(self.store.get_event(event.event_id)) self.get_success(self.store.get_event(event.event_id)) + + def create_fake_event_from_remote_server( + self, remote_server_name: str, event_dict: JsonDict, room_version: RoomVersion + ) -> EventBase: + """ + This is similar to what `FederatingHomeserverTestCase` is doing but we don't + need all of the extra baggage and we want to be able to create an event from + many remote servers. + """ + + # poke the other server's signing key into the key store, so that we don't + # make requests for it + other_server_signature_key = generate_signing_key("test") + verify_key = get_verify_key(other_server_signature_key) + verify_key_id = "%s:%s" % (verify_key.alg, verify_key.version) + + self.get_success( + self.hs.get_datastores().main.store_server_keys_response( + remote_server_name, + from_server=remote_server_name, + ts_added_ms=self.clock.time_msec(), + verify_keys={ + verify_key_id: FetchKeyResult( + verify_key=verify_key, + valid_until_ts=self.clock.time_msec() + 10000, + ), + }, + response_json={ + "verify_keys": { + verify_key_id: {"key": encode_verify_key_base64(verify_key)} + } + }, + ) + ) + + add_hashes_and_signatures( + room_version=room_version, + event_dict=event_dict, + signature_name=remote_server_name, + signing_key=other_server_signature_key, + ) + event = make_event_from_dict( + event_dict, + room_version=room_version, + ) + + return event diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 9dd0e98971..6b202dfbd5 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -17,6 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # +from http import HTTPStatus from typing import Collection, ContextManager, List, Optional from unittest.mock import AsyncMock, Mock, patch @@ -347,7 +348,15 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): # the prev_events used when creating the join event, such that the ban does not # precede the join. with self._patch_get_latest_events([last_room_creation_event_id]): - self.helper.join(room_id, eve, tok=eve_token) + self.helper.join( + room_id, + eve, + tok=eve_token, + # Previously, this join would succeed but now we expect it to fail at + # this point. The rest of the test is for the case when this used to + # succeed. + expect_code=HTTPStatus.FORBIDDEN, + ) # Eve makes a second, incremental sync. eve_incremental_sync_after_join: SyncResult = self.get_success( diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 4429d0f4e2..58a7a9dc72 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -22,14 +22,26 @@ import logging from unittest.mock import AsyncMock, Mock from netaddr import IPSet +from signedjson.key import ( + encode_verify_key_base64, + generate_signing_key, + get_verify_key, +) + +from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes, Membership -from synapse.events.builder import EventBuilderFactory +from synapse.api.room_versions import RoomVersion +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, make_event_from_dict from synapse.handlers.typing import TypingWriterHandler from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent from synapse.rest.admin import register_servlets_for_client_rest_resource from synapse.rest.client import login, room -from synapse.types import UserID, create_requester +from synapse.server import HomeServer +from synapse.storage.keys import FetchKeyResult +from synapse.types import JsonDict, UserID, create_requester +from synapse.util import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import get_clock @@ -63,6 +75,9 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): ip_blocklist=IPSet(), ) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.storage_controllers = hs.get_storage_controllers() + def test_send_event_single_sender(self) -> None: """Test that using a single federation sender worker correctly sends a new event. @@ -243,35 +258,92 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): self.assertTrue(sent_on_1) self.assertTrue(sent_on_2) + def create_fake_event_from_remote_server( + self, remote_server_name: str, event_dict: JsonDict, room_version: RoomVersion + ) -> EventBase: + """ + This is similar to what `FederatingHomeserverTestCase` is doing but we don't + need all of the extra baggage and we want to be able to create an event from + many remote servers. + """ + + # poke the other server's signing key into the key store, so that we don't + # make requests for it + other_server_signature_key = generate_signing_key("test") + verify_key = get_verify_key(other_server_signature_key) + verify_key_id = "%s:%s" % (verify_key.alg, verify_key.version) + + self.get_success( + self.hs.get_datastores().main.store_server_keys_response( + remote_server_name, + from_server=remote_server_name, + ts_added_ms=self.clock.time_msec(), + verify_keys={ + verify_key_id: FetchKeyResult( + verify_key=verify_key, + valid_until_ts=self.clock.time_msec() + 10000, + ), + }, + response_json={ + "verify_keys": { + verify_key_id: {"key": encode_verify_key_base64(verify_key)} + } + }, + ) + ) + + add_hashes_and_signatures( + room_version=room_version, + event_dict=event_dict, + signature_name=remote_server_name, + signing_key=other_server_signature_key, + ) + event = make_event_from_dict( + event_dict, + room_version=room_version, + ) + + return event + def create_room_with_remote_server( self, user: str, token: str, remote_server: str = "other_server" ) -> str: - room = self.helper.create_room_as(user, tok=token) + room_id = self.helper.create_room_as(user, tok=token) store = self.hs.get_datastores().main federation = self.hs.get_federation_event_handler() - prev_event_ids = self.get_success(store.get_latest_event_ids_in_room(room)) - room_version = self.get_success(store.get_room_version(room)) + room_version = self.get_success(store.get_room_version(room_id)) - factory = EventBuilderFactory(self.hs) - factory.hostname = remote_server + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + + # Figure out what the forward extremities in the room are (the most recent + # events that aren't tied into the DAG) + prev_event_ids = self.get_success(store.get_latest_event_ids_in_room(room_id)) user_id = UserID("user", remote_server).to_string() - event_dict = { - "type": EventTypes.Member, - "state_key": user_id, - "content": {"membership": Membership.JOIN}, - "sender": user_id, - "room_id": room, - } - - builder = factory.for_room_version(room_version, event_dict) - join_event = self.get_success( - builder.build(prev_event_ids=list(prev_event_ids), auth_event_ids=None) + join_event = self.create_fake_event_from_remote_server( + remote_server_name=remote_server, + event_dict={ + "room_id": room_id, + "sender": user_id, + "type": EventTypes.Member, + "state_key": user_id, + "depth": 1000, + "origin_server_ts": 1, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(prev_event_ids), + }, + room_version=room_version, ) self.get_success(federation.on_send_membership_event(remote_server, join_event)) self.replicate() - return room + return room_id diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 4cf1a3dc51..833bd6fff8 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -742,7 +742,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(33, channel.resource_usage.db_txn_count) + self.assertEqual(34, channel.resource_usage.db_txn_count) def test_post_room_initial_state(self) -> None: # POST with initial_state config key, expect new room id @@ -755,7 +755,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(35, channel.resource_usage.db_txn_count) + self.assertEqual(36, channel.resource_usage.db_txn_count) def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id diff --git a/tests/test_federation.py b/tests/test_federation.py deleted file mode 100644 index 94b0fa9856..0000000000 --- a/tests/test_federation.py +++ /dev/null @@ -1,378 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2020 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from typing import Collection, List, Optional, Union -from unittest.mock import AsyncMock, Mock - -from twisted.test.proto_helpers import MemoryReactor - -from synapse.api.errors import FederationError -from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict -from synapse.events.snapshot import EventContext -from synapse.federation.federation_base import event_from_pdu_json -from synapse.handlers.device import DeviceListUpdater -from synapse.http.types import QueryParams -from synapse.logging.context import LoggingContext -from synapse.server import HomeServer -from synapse.types import JsonDict, UserID, create_requester -from synapse.util import Clock -from synapse.util.retryutils import NotRetryingDestination - -from tests import unittest - - -class MessageAcceptTests(unittest.HomeserverTestCase): - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - self.http_client = Mock() - return self.setup_test_homeserver(federation_http_client=self.http_client) - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - user_id = UserID("us", "test") - our_user = create_requester(user_id) - room_creator = self.hs.get_room_creation_handler() - self.room_id = self.get_success( - room_creator.create_room( - our_user, room_creator._presets_dict["public_chat"], ratelimit=False - ) - )[0] - - self.store = self.hs.get_datastores().main - - # Figure out what the most recent event is - most_recent = next( - iter( - self.get_success( - self.hs.get_datastores().main.get_latest_event_ids_in_room( - self.room_id - ) - ) - ) - ) - - join_event = make_event_from_dict( - { - "room_id": self.room_id, - "sender": "@baduser:test.serv", - "state_key": "@baduser:test.serv", - "event_id": "$join:test.serv", - "depth": 1000, - "origin_server_ts": 1, - "type": "m.room.member", - "origin": "test.servx", - "content": {"membership": "join"}, - "auth_events": [], - "prev_state": [(most_recent, {})], - "prev_events": [(most_recent, {})], - } - ) - - self.handler = self.hs.get_federation_handler() - federation_event_handler = self.hs.get_federation_event_handler() - - async def _check_event_auth( - origin: Optional[str], event: EventBase, context: EventContext - ) -> None: - pass - - federation_event_handler._check_event_auth = _check_event_auth # type: ignore[method-assign] - self.client = self.hs.get_federation_client() - - async def _check_sigs_and_hash_for_pulled_events_and_fetch( - dest: str, pdus: Collection[EventBase], room_version: RoomVersion - ) -> List[EventBase]: - return list(pdus) - - self.client._check_sigs_and_hash_for_pulled_events_and_fetch = ( # type: ignore[method-assign] - _check_sigs_and_hash_for_pulled_events_and_fetch # type: ignore[assignment] - ) - - # Send the join, it should return None (which is not an error) - self.assertEqual( - self.get_success( - federation_event_handler.on_receive_pdu("test.serv", join_event) - ), - None, - ) - - # Make sure we actually joined the room - self.assertEqual( - self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)), - {"$join:test.serv"}, - ) - - def test_cant_hide_direct_ancestors(self) -> None: - """ - If you send a message, you must be able to provide the direct - prev_events that said event references. - """ - - async def post_json( - destination: str, - path: str, - data: Optional[JsonDict] = None, - long_retries: bool = False, - timeout: Optional[int] = None, - ignore_backoff: bool = False, - args: Optional[QueryParams] = None, - ) -> Union[JsonDict, list]: - # If it asks us for new missing events, give them NOTHING - if path.startswith("/_matrix/federation/v1/get_missing_events/"): - return {"events": []} - return {} - - self.http_client.post_json = post_json - - # Figure out what the most recent event is - most_recent = next( - iter( - self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)) - ) - ) - - # Now lie about an event - lying_event = make_event_from_dict( - { - "room_id": self.room_id, - "sender": "@baduser:test.serv", - "event_id": "one:test.serv", - "depth": 1000, - "origin_server_ts": 1, - "type": "m.room.message", - "origin": "test.serv", - "content": {"body": "hewwo?"}, - "auth_events": [], - "prev_events": [("two:test.serv", {}), (most_recent, {})], - } - ) - - federation_event_handler = self.hs.get_federation_event_handler() - with LoggingContext("test-context"): - failure = self.get_failure( - federation_event_handler.on_receive_pdu("test.serv", lying_event), - FederationError, - ) - - # on_receive_pdu should throw an error - self.assertEqual( - failure.value.args[0], - ( - "ERROR 403: Your server isn't divulging details about prev_events " - "referenced in this event." - ), - ) - - # Make sure the invalid event isn't there - extrem = self.get_success(self.store.get_latest_event_ids_in_room(self.room_id)) - self.assertEqual(extrem, {"$join:test.serv"}) - - def test_retry_device_list_resync(self) -> None: - """Tests that device lists are marked as stale if they couldn't be synced, and - that stale device lists are retried periodically. - """ - remote_user_id = "@john:test_remote" - remote_origin = "test_remote" - - # Track the number of attempts to resync the user's device list. - self.resync_attempts = 0 - - # When this function is called, increment the number of resync attempts (only if - # we're querying devices for the right user ID), then raise a - # NotRetryingDestination error to fail the resync gracefully. - def query_user_devices( - destination: str, user_id: str, timeout: int = 30000 - ) -> JsonDict: - if user_id == remote_user_id: - self.resync_attempts += 1 - - raise NotRetryingDestination(0, 0, destination) - - # Register the mock on the federation client. - federation_client = self.hs.get_federation_client() - federation_client.query_user_devices = Mock(side_effect=query_user_devices) # type: ignore[method-assign] - - # Register a mock on the store so that the incoming update doesn't fail because - # we don't share a room with the user. - store = self.hs.get_datastores().main - store.get_rooms_for_user = AsyncMock(return_value=["!someroom:test"]) - - # Manually inject a fake device list update. We need this update to include at - # least one prev_id so that the user's device list will need to be retried. - device_list_updater = self.hs.get_device_handler().device_list_updater - assert isinstance(device_list_updater, DeviceListUpdater) - self.get_success( - device_list_updater.incoming_device_list_update( - origin=remote_origin, - edu_content={ - "deleted": False, - "device_display_name": "Mobile", - "device_id": "QBUAZIFURK", - "prev_id": [5], - "stream_id": 6, - "user_id": remote_user_id, - }, - ) - ) - - # Check that there was one resync attempt. - self.assertEqual(self.resync_attempts, 1) - - # Check that the resync attempt failed and caused the user's device list to be - # marked as stale. - need_resync = self.get_success( - store.get_user_ids_requiring_device_list_resync() - ) - self.assertIn(remote_user_id, need_resync) - - # Check that waiting for 30 seconds caused Synapse to retry resyncing the device - # list. - self.reactor.advance(30) - self.assertEqual(self.resync_attempts, 2) - - def test_cross_signing_keys_retry(self) -> None: - """Tests that resyncing a device list correctly processes cross-signing keys from - the remote server. - """ - remote_user_id = "@john:test_remote" - remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" - remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" - - # Register mock device list retrieval on the federation client. - federation_client = self.hs.get_federation_client() - federation_client.query_user_devices = AsyncMock( # type: ignore[method-assign] - return_value={ - "user_id": remote_user_id, - "stream_id": 1, - "devices": [], - "master_key": { - "user_id": remote_user_id, - "usage": ["master"], - "keys": {"ed25519:" + remote_master_key: remote_master_key}, - }, - "self_signing_key": { - "user_id": remote_user_id, - "usage": ["self_signing"], - "keys": { - "ed25519:" + remote_self_signing_key: remote_self_signing_key - }, - }, - } - ) - - # Resync the device list. - device_handler = self.hs.get_device_handler() - self.get_success( - device_handler.device_list_updater.multi_user_device_resync( - [remote_user_id] - ), - ) - - # Retrieve the cross-signing keys for this user. - keys = self.get_success( - self.store.get_e2e_cross_signing_keys_bulk(user_ids=[remote_user_id]), - ) - self.assertIn(remote_user_id, keys) - key = keys[remote_user_id] - assert key is not None - - # Check that the master key is the one returned by the mock. - master_key = key["master"] - self.assertEqual(len(master_key["keys"]), 1) - self.assertTrue("ed25519:" + remote_master_key in master_key["keys"].keys()) - self.assertTrue(remote_master_key in master_key["keys"].values()) - - # Check that the self-signing key is the one returned by the mock. - self_signing_key = key["self_signing"] - self.assertEqual(len(self_signing_key["keys"]), 1) - self.assertTrue( - "ed25519:" + remote_self_signing_key in self_signing_key["keys"].keys(), - ) - self.assertTrue(remote_self_signing_key in self_signing_key["keys"].values()) - - -class StripUnsignedFromEventsTestCase(unittest.TestCase): - def test_strip_unauthorized_unsigned_values(self) -> None: - event1 = { - "sender": "@baduser:test.serv", - "state_key": "@baduser:test.serv", - "event_id": "$event1:test.serv", - "depth": 1000, - "origin_server_ts": 1, - "type": "m.room.member", - "origin": "test.servx", - "content": {"membership": "join"}, - "auth_events": [], - "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, - } - filtered_event = event_from_pdu_json(event1, RoomVersions.V1) - # Make sure unauthorized fields are stripped from unsigned - self.assertNotIn("more warez", filtered_event.unsigned) - - def test_strip_event_maintains_allowed_fields(self) -> None: - event2 = { - "sender": "@baduser:test.serv", - "state_key": "@baduser:test.serv", - "event_id": "$event2:test.serv", - "depth": 1000, - "origin_server_ts": 1, - "type": "m.room.member", - "origin": "test.servx", - "auth_events": [], - "content": {"membership": "join"}, - "unsigned": { - "malicious garbage": "hackz", - "more warez": "more hackz", - "age": 14, - "invite_room_state": [], - }, - } - - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) - self.assertIn("age", filtered_event2.unsigned) - self.assertEqual(14, filtered_event2.unsigned["age"]) - self.assertNotIn("more warez", filtered_event2.unsigned) - # Invite_room_state is allowed in events of type m.room.member - self.assertIn("invite_room_state", filtered_event2.unsigned) - self.assertEqual([], filtered_event2.unsigned["invite_room_state"]) - - def test_strip_event_removes_fields_based_on_event_type(self) -> None: - event3 = { - "sender": "@baduser:test.serv", - "state_key": "@baduser:test.serv", - "event_id": "$event3:test.serv", - "depth": 1000, - "origin_server_ts": 1, - "type": "m.room.power_levels", - "origin": "test.servx", - "content": {}, - "auth_events": [], - "unsigned": { - "malicious garbage": "hackz", - "more warez": "more hackz", - "age": 14, - "invite_room_state": [], - }, - } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) - self.assertIn("age", filtered_event3.unsigned) - # Invite_room_state field is only permitted in event type m.room.member - self.assertNotIn("invite_room_state", filtered_event3.unsigned) - self.assertNotIn("more warez", filtered_event3.unsigned) diff --git a/tests/utils.py b/tests/utils.py index 3f4a7bb560..d4aebc3069 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -400,11 +400,24 @@ class TestTimeout(Exception): class test_timeout: + """ + FIXME: This implementation is not robust against other code tight-looping and + preventing the signals propagating and timing out the test. You may need to add + `time.sleep(0.1)` to your code in order to allow this timeout to work correctly. + + ```py + with test_timeout(3): + while True: + my_checking_func() + time.sleep(0.1) + ``` + """ + def __init__(self, seconds: int, error_message: Optional[str] = None) -> None: - if error_message is None: - error_message = "test timed out after {}s.".format(seconds) + self.error_message = f"Test timed out after {seconds}s" + if error_message is not None: + self.error_message += f": {error_message}" self.seconds = seconds - self.error_message = error_message def handle_timeout(self, signum: int, frame: Optional[FrameType]) -> None: raise TestTimeout(self.error_message) From b41a9ebb38163f0bd9b08ba411d31a2bb515d9ef Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:39:51 +0000 Subject: [PATCH 106/691] OIDC: increase length of generated `nonce` parameter from 30->32 chars (#18109) --- changelog.d/18109.misc | 1 + synapse/handlers/oidc.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18109.misc diff --git a/changelog.d/18109.misc b/changelog.d/18109.misc new file mode 100644 index 0000000000..c310e76f78 --- /dev/null +++ b/changelog.d/18109.misc @@ -0,0 +1 @@ +Increase the length of the generated `nonce` parameter when perfoming OIDC logins to comply with the TI-Messenger spec. \ No newline at end of file diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 22b59829fa..701e828081 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -1002,7 +1002,21 @@ class OidcProvider: """ state = generate_token() - nonce = generate_token() + + # Generate a nonce 32 characters long. When encoded with base64url later on, + # the nonce will be 43 characters when sent to the identity provider. + # + # While RFC7636 does not specify a minimum length for the `nonce` + # parameter, the TI-Messenger IDP_FD spec v1.7.3 does require it to be + # between 43 and 128 characters. This spec concerns using Matrix for + # communication in German healthcare. + # + # As increasing the length only strengthens security, we use this length + # to allow TI-Messenger deployments using Synapse to satisfy this + # external spec. + # + # See https://github.com/element-hq/synapse/pull/18109 for more context. + nonce = generate_token(length=32) code_verifier = "" if not client_redirect_url: From c53999dab871fa10d34487b0bf654eee6827a711 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:04:41 +0000 Subject: [PATCH 107/691] Bump types-bleach from 6.1.0.20240331 to 6.2.0.20241123 (#18082) Bumps [types-bleach](https://github.com/python/typeshed) from 6.1.0.20240331 to 6.2.0.20241123.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-bleach&package-manager=pip&previous-version=6.1.0.20240331&new-version=6.2.0.20241123)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9764904b81..36622aa13a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2706,13 +2706,13 @@ twisted = "*" [[package]] name = "types-bleach" -version = "6.1.0.20240331" +version = "6.2.0.20241123" description = "Typing stubs for bleach" optional = false python-versions = ">=3.8" files = [ - {file = "types-bleach-6.1.0.20240331.tar.gz", hash = "sha256:2ee858a84fb06fc2225ff56ba2f7f6c88b65638659efae0d7bfd6b24a1b5a524"}, - {file = "types_bleach-6.1.0.20240331-py3-none-any.whl", hash = "sha256:399bc59bfd20a36a56595f13f805e56c8a08e5a5c07903e5cf6fafb5a5107dd4"}, + {file = "types_bleach-6.2.0.20241123-py3-none-any.whl", hash = "sha256:c6e58b3646665ca7c6b29890375390f4569e84f0cf5c171e0fe1ddb71a7be86a"}, + {file = "types_bleach-6.2.0.20241123.tar.gz", hash = "sha256:dac5fe9015173514da3ac810c1a935619a3ccbcc5d66c4cbf4707eac00539057"}, ] [package.dependencies] From 579f4ac1cd6b0d15a12fabcca9d34d5ba3242ce2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:24:07 +0000 Subject: [PATCH 108/691] Bump serde_json from 1.0.135 to 1.0.137 (#18099) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.135 to 1.0.137.
Release notes

Sourced from serde_json's releases.

v1.0.137

  • Turn on "float_roundtrip" and "unbounded_depth" features for serde_json in play.rust-lang.org (#1231)

v1.0.136

  • Optimize serde_json::value::Serializer::serialize_map by using Map::with_capacity (#1230, thanks @​goffrie)
Commits
  • eb49e28 Release 1.0.137
  • 51c48ab Merge pull request #1231 from dtolnay/playground
  • 7d8f15b Enable "float_roundtrip" and "unbounded_depth" features in playground
  • a46f14c Release 1.0.136
  • eb9f3f6 Merge pull request #1230 from goffrie/patch-1
  • 513e5b2 Use Map::with_capacity in value::Serializer::serialize_map
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_json&package-manager=cargo&previous-version=1.0.135&new-version=1.0.137)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7312c0125..82ed544226 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,9 +449,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", From 8f27b3af0723feb201933266ad0e8688ce3cb37e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:28:00 +0000 Subject: [PATCH 109/691] Bump python-multipart from 0.0.18 to 0.0.20 (#18096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [python-multipart](https://github.com/Kludex/python-multipart) from 0.0.18 to 0.0.20.
Release notes

Sourced from python-multipart's releases.

Version 0.0.20

What's Changed

New Contributors

Full Changelog: https://github.com/Kludex/python-multipart/compare/0.0.19...0.0.20

Version 0.0.19

What's Changed

Full Changelog: https://github.com/Kludex/python-multipart/compare/0.0.18...0.0.19

Changelog

Sourced from python-multipart's changelog.

0.0.20 (2024-12-16)

  • Handle messages containing only end boundary #142.

0.0.19 (2024-11-30)

  • Don't warn when CRLF is found after last boundary on MultipartParser #193.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=python-multipart&package-manager=pip&previous-version=0.0.18&new-version=0.0.20)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 36622aa13a..c9c567d6e3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1960,13 +1960,13 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.18" +version = "0.0.20" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.8" files = [ - {file = "python_multipart-0.0.18-py3-none-any.whl", hash = "sha256:efe91480f485f6a361427a541db4796f9e1591afc0fb8e7a4ba06bfbc6708996"}, - {file = "python_multipart-0.0.18.tar.gz", hash = "sha256:7a68db60c8bfb82e460637fa4750727b45af1d5e2ed215593f917f64694d34fe"}, + {file = "python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104"}, + {file = "python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13"}, ] [[package]] From 628351b98de9e8f7340b139ef77cd530da286d2d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 28 Jan 2025 00:37:24 +0000 Subject: [PATCH 110/691] Never autojoin deactivated & suspended users. (#18073) This PR changes the logic so that deactivated users are always ignored. Suspended users were already effectively ignored as Synapse forbids a join while suspended. --------- Co-authored-by: Devon Hudson --- changelog.d/18073.bugfix | 1 + synapse/events/auto_accept_invites.py | 85 ++++++----- tests/events/test_auto_accept_invites.py | 181 ++++++++++++++++++++++- 3 files changed, 232 insertions(+), 35 deletions(-) create mode 100644 changelog.d/18073.bugfix diff --git a/changelog.d/18073.bugfix b/changelog.d/18073.bugfix new file mode 100644 index 0000000000..eeb56a7a61 --- /dev/null +++ b/changelog.d/18073.bugfix @@ -0,0 +1 @@ +Deactivated users will no longer automatically accept an invite when `auto_accept_invites` is enabled. \ No newline at end of file diff --git a/synapse/events/auto_accept_invites.py b/synapse/events/auto_accept_invites.py index d88ec51d9d..4295107c47 100644 --- a/synapse/events/auto_accept_invites.py +++ b/synapse/events/auto_accept_invites.py @@ -66,50 +66,67 @@ class InviteAutoAccepter: event: The incoming event. """ # Check if the event is an invite for a local user. - is_invite_for_local_user = ( - event.type == EventTypes.Member - and event.is_state() - and event.membership == Membership.INVITE - and self._api.is_mine(event.state_key) - ) + if ( + event.type != EventTypes.Member + or event.is_state() is False + or event.membership != Membership.INVITE + or self._api.is_mine(event.state_key) is False + ): + return # Only accept invites for direct messages if the configuration mandates it. is_direct_message = event.content.get("is_direct", False) - is_allowed_by_direct_message_rules = ( - not self._config.accept_invites_only_for_direct_messages - or is_direct_message is True - ) + if ( + self._config.accept_invites_only_for_direct_messages + and is_direct_message is False + ): + return # Only accept invites from remote users if the configuration mandates it. is_from_local_user = self._api.is_mine(event.sender) - is_allowed_by_local_user_rules = ( - not self._config.accept_invites_only_from_local_users - or is_from_local_user is True + if ( + self._config.accept_invites_only_from_local_users + and is_from_local_user is False + ): + return + + # Check the user is activated. + recipient = await self._api.get_userinfo_by_id(event.state_key) + + # Ignore if the user doesn't exist. + if recipient is None: + return + + # Never accept invites for deactivated users. + if recipient.is_deactivated: + return + + # Never accept invites for suspended users. + if recipient.suspended: + return + + # Never accept invites for locked users. + if recipient.locked: + return + + # Make the user join the room. We run this as a background process to circumvent a race condition + # that occurs when responding to invites over federation (see https://github.com/matrix-org/synapse-auto-accept-invite/issues/12) + run_as_background_process( + "retry_make_join", + self._retry_make_join, + event.state_key, + event.state_key, + event.room_id, + "join", + bg_start_span=False, ) - if ( - is_invite_for_local_user - and is_allowed_by_direct_message_rules - and is_allowed_by_local_user_rules - ): - # Make the user join the room. We run this as a background process to circumvent a race condition - # that occurs when responding to invites over federation (see https://github.com/matrix-org/synapse-auto-accept-invite/issues/12) - run_as_background_process( - "retry_make_join", - self._retry_make_join, - event.state_key, - event.state_key, - event.room_id, - "join", - bg_start_span=False, + if is_direct_message: + # Mark this room as a direct message! + await self._mark_room_as_direct_message( + event.state_key, event.sender, event.room_id ) - if is_direct_message: - # Mark this room as a direct message! - await self._mark_room_as_direct_message( - event.state_key, event.sender, event.room_id - ) - async def _mark_room_as_direct_message( self, user_id: str, dm_user_id: str, room_id: str ) -> None: diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 7fb4d4fa90..d4e87b1b7f 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -39,7 +39,7 @@ from synapse.module_api import ModuleApi from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer -from synapse.types import StreamToken, create_requester +from synapse.types import StreamToken, UserID, UserInfo, create_requester from synapse.util import Clock from tests.handlers.test_sync import generate_sync_config @@ -349,6 +349,169 @@ class AutoAcceptInvitesTestCase(FederatingHomeserverTestCase): join_updates, _ = sync_join(self, invited_user_id) self.assertEqual(len(join_updates), 0) + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_missing_user(self) -> None: + """Tests that receiving an invite for a missing user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass") + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = "@fake:" + self.hs.config.server.server_name + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, _ = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_deactivated_user(self) -> None: + """Tests that receiving an invite for a deactivated user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + "/_synapse/admin/v2/users/%s" % invited_user_id, + {"deactivated": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_suspended_user(self) -> None: + """Tests that receiving an invite for a suspended user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/suspend/{invited_user_id}", + {"suspend": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + + @override_config( + { + "auto_accept_invites": { + "enabled": True, + }, + } + ) + async def test_ignore_invite_for_locked_user(self) -> None: + """Tests that receiving an invite for a suspended user is ignored.""" + inviting_user_id = self.register_user("inviter", "pass", admin=True) + inviting_user_tok = self.login("inviter", "pass") + + # A local user who receives an invite + invited_user_id = self.register_user("invitee", "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + inviting_user_id, + tok=inviting_user_tok, + ) + + channel = self.make_request( + "PUT", + f"/_synapse/admin/v2/users/{invited_user_id}", + {"locked": True}, + access_token=inviting_user_tok, + ) + + assert channel.code == 200 + + self.helper.invite( + room_id, + inviting_user_id, + invited_user_id, + tok=inviting_user_tok, + ) + + join_updates, b = sync_join(self, inviting_user_id) + # Assert that the last event in the room was not a member event for the target user. + self.assertEqual( + join_updates[0].timeline.events[-1].content["membership"], "invite" + ) + _request_key = 0 @@ -647,6 +810,22 @@ def create_module( module_api.is_mine.side_effect = lambda a: a.split(":")[1] == "test" module_api.worker_name = worker_name module_api.sleep.return_value = make_multiple_awaitable(None) + module_api.get_userinfo_by_id.return_value = UserInfo( + user_id=UserID.from_string("@user:test"), + is_admin=False, + is_guest=False, + consent_server_notice_sent=None, + consent_ts=None, + consent_version=None, + appservice_id=None, + creation_ts=0, + user_type=None, + is_deactivated=False, + locked=False, + is_shadow_banned=False, + approved=True, + suspended=False, + ) if config_override is None: config_override = {} From 3d8535b1def2fde26a10bdf15d0b9dbd941a6005 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 28 Jan 2025 08:37:58 -0700 Subject: [PATCH 111/691] 1.123.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f3ad507de8..cc6426751d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.123.0 (2025-01-28) + +No significant changes since 1.123.0rc1. + + + + # Synapse 1.123.0rc1 (2025-01-21) ### Features diff --git a/debian/changelog b/debian/changelog index 2a18b776f6..a470dff676 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.123.0) stable; urgency=medium + + * New Synapse release 1.123.0. + + -- Synapse Packaging team Tue, 28 Jan 2025 08:37:34 -0700 + matrix-synapse-py3 (1.123.0~rc1) stable; urgency=medium * New Synapse release 1.123.0rc1. diff --git a/pyproject.toml b/pyproject.toml index e6899207da..1cd874716e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.123.0rc1" +version = "1.123.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From a0b70473fc08611aa7ecdc50ce4404e882d9e7f9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 Jan 2025 18:14:02 -0600 Subject: [PATCH 112/691] Raise an error if someone is using an incorrect suffix in a config duration string (#18112) Previously, a value like `5q` would be interpreted as 5 milliseconds. We should just raise an error instead of letting someone run with a misconfiguration. --- changelog.d/18112.bugfix | 1 + synapse/config/_base.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18112.bugfix diff --git a/changelog.d/18112.bugfix b/changelog.d/18112.bugfix new file mode 100644 index 0000000000..61c94280d8 --- /dev/null +++ b/changelog.d/18112.bugfix @@ -0,0 +1 @@ +Raise an error if someone is using an incorrect suffix in a config duration string. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index adce34c03a..912346a423 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -221,9 +221,13 @@ class Config: The number of milliseconds in the duration. Raises: - TypeError, if given something other than an integer or a string + TypeError: if given something other than an integer or a string, or the + duration is using an incorrect suffix. ValueError: if given a string not of the form described above. """ + # For integers, we prefer to use `type(value) is int` instead of + # `isinstance(value, int)` because we want to exclude subclasses of int, such as + # bool. if type(value) is int: # noqa: E721 return value elif isinstance(value, str): @@ -246,9 +250,20 @@ class Config: if suffix in sizes: value = value[:-1] size = sizes[suffix] + elif suffix.isdigit(): + # No suffix is treated as milliseconds. + value = value + size = 1 + else: + raise TypeError( + f"Bad duration suffix {value} (expected no suffix or one of these suffixes: {sizes.keys()})" + ) + return int(value) * size else: - raise TypeError(f"Bad duration {value!r}") + raise TypeError( + f"Bad duration type {value!r} (expected int or string duration)" + ) @staticmethod def abspath(file_path: str) -> str: From ac1bf682ff012ee8af5153318eec5d25ed786e90 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:48:12 +0000 Subject: [PATCH 113/691] Allow (un)block_room storage functions to be called on workers (#18119) This is so workers can call these functions. This was preventing the [Delete Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#version-2-new-version) from succeeding when `block: true` was specified. This was because we had `run_background_tasks_on` configured to run on a separate worker. As workers weren't able to call the `block_room` storage function before this PR, the (delete room) task failed when taken off the queue by the worker. --- changelog.d/18119.bugfix | 1 + synapse/storage/databases/main/room.py | 88 +++++++++++++------------- 2 files changed, 45 insertions(+), 44 deletions(-) create mode 100644 changelog.d/18119.bugfix diff --git a/changelog.d/18119.bugfix b/changelog.d/18119.bugfix new file mode 100644 index 0000000000..c8ac53f9d4 --- /dev/null +++ b/changelog.d/18119.bugfix @@ -0,0 +1 @@ +Fix a bug where the [Delete Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#version-2-new-version) would fail if the `block` parameter was set to `true` and a worker other than the main process was configured to handle background tasks. \ No newline at end of file diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 2522bebd72..d673adba16 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1181,6 +1181,50 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): return total_media_quarantined + async def block_room(self, room_id: str, user_id: str) -> None: + """Marks the room as blocked. + + Can be called multiple times (though we'll only track the last user to + block this room). + + Can be called on a room unknown to this homeserver. + + Args: + room_id: Room to block + user_id: Who blocked it + """ + await self.db_pool.simple_upsert( + table="blocked_rooms", + keyvalues={"room_id": room_id}, + values={}, + insertion_values={"user_id": user_id}, + desc="block_room", + ) + await self.db_pool.runInteraction( + "block_room_invalidation", + self._invalidate_cache_and_stream, + self.is_room_blocked, + (room_id,), + ) + + async def unblock_room(self, room_id: str) -> None: + """Remove the room from blocking list. + + Args: + room_id: Room to unblock + """ + await self.db_pool.simple_delete( + table="blocked_rooms", + keyvalues={"room_id": room_id}, + desc="unblock_room", + ) + await self.db_pool.runInteraction( + "block_room_invalidation", + self._invalidate_cache_and_stream, + self.is_room_blocked, + (room_id,), + ) + async def get_rooms_for_retention_period_in_range( self, min_ms: Optional[int], max_ms: Optional[int], include_null: bool = False ) -> Dict[str, RetentionPolicy]: @@ -2500,50 +2544,6 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): ) return next_id - async def block_room(self, room_id: str, user_id: str) -> None: - """Marks the room as blocked. - - Can be called multiple times (though we'll only track the last user to - block this room). - - Can be called on a room unknown to this homeserver. - - Args: - room_id: Room to block - user_id: Who blocked it - """ - await self.db_pool.simple_upsert( - table="blocked_rooms", - keyvalues={"room_id": room_id}, - values={}, - insertion_values={"user_id": user_id}, - desc="block_room", - ) - await self.db_pool.runInteraction( - "block_room_invalidation", - self._invalidate_cache_and_stream, - self.is_room_blocked, - (room_id,), - ) - - async def unblock_room(self, room_id: str) -> None: - """Remove the room from blocking list. - - Args: - room_id: Room to unblock - """ - await self.db_pool.simple_delete( - table="blocked_rooms", - keyvalues={"room_id": room_id}, - desc="unblock_room", - ) - await self.db_pool.runInteraction( - "block_room_invalidation", - self._invalidate_cache_and_stream, - self.is_room_blocked, - (room_id,), - ) - async def clear_partial_state_room(self, room_id: str) -> Optional[int]: """Clears the partial state flag for a room. From aa6e5c2ecbe58dec1c38c33949fc70c88f39c242 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Feb 2025 18:29:15 +0100 Subject: [PATCH 114/691] Add locking to more safely delete state groups: Part 1 (#18107) Currently we don't really have anything that stops us from deleting state groups when an in-flight event references it. This is a fairly rare race currently, but we want to be able to more aggressively delete state groups so it is important to address this to ensure that the database remains valid. This implements the locking, but doesn't actually use it. See the class docstring of the new data store for an explanation for how this works. --------- Co-authored-by: Devon Hudson --- changelog.d/18107.bugfix | 1 + synapse/handlers/federation_event.py | 18 +- synapse/state/__init__.py | 61 ++- synapse/storage/controllers/persist_events.py | 32 +- synapse/storage/databases/__init__.py | 10 +- synapse/storage/databases/state/deletion.py | 446 ++++++++++++++++++ synapse/storage/databases/state/store.py | 39 +- synapse/storage/schema/__init__.py | 5 +- .../delta/89/01_state_groups_deletion.sql | 39 ++ tests/handlers/test_federation_event.py | 9 +- tests/rest/client/test_rooms.py | 4 +- tests/storage/test_state_deletion.py | 411 ++++++++++++++++ tests/test_state.py | 18 +- 13 files changed, 1047 insertions(+), 46 deletions(-) create mode 100644 changelog.d/18107.bugfix create mode 100644 synapse/storage/databases/state/deletion.py create mode 100644 synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql create mode 100644 tests/storage/test_state_deletion.py diff --git a/changelog.d/18107.bugfix b/changelog.d/18107.bugfix new file mode 100644 index 0000000000..4d0c19fab9 --- /dev/null +++ b/changelog.d/18107.bugfix @@ -0,0 +1 @@ +Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 1b535ea2cb..1e738f484f 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -151,6 +151,8 @@ class FederationEventHandler: def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self._store = hs.get_datastores().main + self._state_store = hs.get_datastores().state + self._state_deletion_store = hs.get_datastores().state_deletion self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state @@ -580,7 +582,9 @@ class FederationEventHandler: room_version.identifier, state_maps_to_resolve, event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: @@ -1179,7 +1183,9 @@ class FederationEventHandler: room_version, state_maps, event_map={event_id: event}, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) except Exception as e: @@ -1874,7 +1880,9 @@ class FederationEventHandler: room_version, [local_state_id_map, claimed_auth_events_id_map], event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: @@ -2014,7 +2022,9 @@ class FederationEventHandler: room_version, state_sets, event_map=None, - state_res_store=StateResolutionStore(self._store), + state_res_store=StateResolutionStore( + self._store, self._state_deletion_store + ), ) ) else: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 72b291889b..5b746f2037 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -59,11 +59,13 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.metrics import Measure, measure_func +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer from synapse.storage.controllers import StateStorageController from synapse.storage.databases.main import DataStore + from synapse.storage.databases.state.deletion import StateDeletionDataStore logger = logging.getLogger(__name__) metrics_logger = logging.getLogger("synapse.state.metrics") @@ -194,6 +196,8 @@ class StateHandler: self._storage_controllers = hs.get_storage_controllers() self._events_shard_config = hs.config.worker.events_shard_config self._instance_name = hs.get_instance_name() + self._state_store = hs.get_datastores().state + self._state_deletion_store = hs.get_datastores().state_deletion self._update_current_state_client = ( ReplicationUpdateCurrentStateRestServlet.make_client(hs) @@ -475,7 +479,10 @@ class StateHandler: @trace @measure_func() async def resolve_state_groups_for_events( - self, room_id: str, event_ids: StrCollection, await_full_state: bool = True + self, + room_id: str, + event_ids: StrCollection, + await_full_state: bool = True, ) -> _StateCacheEntry: """Given a list of event_ids this method fetches the state at each event, resolves conflicts between them and returns them. @@ -511,6 +518,17 @@ class StateHandler: ) = await self._state_storage_controller.get_state_group_delta( state_group_id ) + + if prev_group: + # Ensure that we still have the prev group, and ensure we don't + # delete it while we're persisting the event. + missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( + {prev_group} + ) + if missing_state_group: + prev_group = None + delta_ids = None + return _StateCacheEntry( state=None, state_group=state_group_id, @@ -531,7 +549,9 @@ class StateHandler: room_version, state_to_resolve, None, - state_res_store=StateResolutionStore(self.store), + state_res_store=StateResolutionStore( + self.store, self._state_deletion_store + ), ) return result @@ -663,7 +683,25 @@ class StateResolutionHandler: async with self.resolve_linearizer.queue(group_names): cache = self._state_cache.get(group_names, None) if cache: - return cache + # Check that the returned cache entry doesn't point to deleted + # state groups. + state_groups_to_check = set() + if cache.state_group is not None: + state_groups_to_check.add(cache.state_group) + + if cache.prev_group is not None: + state_groups_to_check.add(cache.prev_group) + + missing_state_groups = await state_res_store.state_deletion_store.check_state_groups_and_bump_deletion( + state_groups_to_check + ) + + if not missing_state_groups: + return cache + else: + # There are missing state groups, so let's remove the stale + # entry and continue as if it was a cache miss. + self._state_cache.pop(group_names, None) logger.info( "Resolving state for %s with groups %s", @@ -671,6 +709,16 @@ class StateResolutionHandler: list(group_names), ) + # We double check that none of the state groups have been deleted. + # They shouldn't be as all these state groups should be referenced. + missing_state_groups = await state_res_store.state_deletion_store.check_state_groups_and_bump_deletion( + group_names + ) + if missing_state_groups: + raise Exception( + f"State groups have been deleted: {shortstr(missing_state_groups)}" + ) + state_groups_histogram.observe(len(state_groups_ids)) new_state = await self.resolve_events_with_store( @@ -884,7 +932,8 @@ class StateResolutionStore: in well defined way. """ - store: "DataStore" + main_store: "DataStore" + state_deletion_store: "StateDeletionDataStore" def get_events( self, event_ids: StrCollection, allow_rejected: bool = False @@ -899,7 +948,7 @@ class StateResolutionStore: An awaitable which resolves to a dict from event_id to event. """ - return self.store.get_events( + return self.main_store.get_events( event_ids, redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, @@ -920,4 +969,4 @@ class StateResolutionStore: An awaitable that resolves to a set of event IDs. """ - return self.store.get_auth_chain_difference(room_id, state_sets) + return self.main_store.get_auth_chain_difference(room_id, state_sets) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 879ee9039e..7963905479 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -332,6 +332,7 @@ class EventsPersistenceStorageController: # store for now. self.main_store = stores.main self.state_store = stores.state + self._state_deletion_store = stores.state_deletion assert stores.persist_events self.persist_events_store = stores.persist_events @@ -549,7 +550,9 @@ class EventsPersistenceStorageController: room_version, state_maps_by_state_group, event_map=None, - state_res_store=StateResolutionStore(self.main_store), + state_res_store=StateResolutionStore( + self.main_store, self._state_deletion_store + ), ) return await res.get_state(self._state_controller, StateFilter.all()) @@ -635,15 +638,20 @@ class EventsPersistenceStorageController: room_id, [e for e, _ in chunk] ) - await self.persist_events_store._persist_events_and_state_updates( - room_id, - chunk, - state_delta_for_room=state_delta_for_room, - new_forward_extremities=new_forward_extremities, - use_negative_stream_ordering=backfilled, - inhibit_local_membership_updates=backfilled, - new_event_links=new_event_links, - ) + # Stop the state groups from being deleted while we're persisting + # them. + async with self._state_deletion_store.persisting_state_group_references( + events_and_contexts + ): + await self.persist_events_store._persist_events_and_state_updates( + room_id, + chunk, + state_delta_for_room=state_delta_for_room, + new_forward_extremities=new_forward_extremities, + use_negative_stream_ordering=backfilled, + inhibit_local_membership_updates=backfilled, + new_event_links=new_event_links, + ) return replaced_events @@ -965,7 +973,9 @@ class EventsPersistenceStorageController: room_version, state_groups, events_map, - state_res_store=StateResolutionStore(self.main_store), + state_res_store=StateResolutionStore( + self.main_store, self._state_deletion_store + ), ) state_resolutions_during_persistence.inc() diff --git a/synapse/storage/databases/__init__.py b/synapse/storage/databases/__init__.py index dd9fc01fb0..81886ff765 100644 --- a/synapse/storage/databases/__init__.py +++ b/synapse/storage/databases/__init__.py @@ -26,6 +26,7 @@ from synapse.storage._base import SQLBaseStore from synapse.storage.database import DatabasePool, make_conn from synapse.storage.databases.main.events import PersistEventsStore from synapse.storage.databases.state import StateGroupDataStore +from synapse.storage.databases.state.deletion import StateDeletionDataStore from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database @@ -49,12 +50,14 @@ class Databases(Generic[DataStoreT]): main state persist_events + state_deletion """ databases: List[DatabasePool] main: "DataStore" # FIXME: https://github.com/matrix-org/synapse/issues/11165: actually an instance of `main_store_class` state: StateGroupDataStore persist_events: Optional[PersistEventsStore] + state_deletion: StateDeletionDataStore def __init__(self, main_store_class: Type[DataStoreT], hs: "HomeServer"): # Note we pass in the main store class here as workers use a different main @@ -63,6 +66,7 @@ class Databases(Generic[DataStoreT]): self.databases = [] main: Optional[DataStoreT] = None state: Optional[StateGroupDataStore] = None + state_deletion: Optional[StateDeletionDataStore] = None persist_events: Optional[PersistEventsStore] = None for database_config in hs.config.database.databases: @@ -114,7 +118,8 @@ class Databases(Generic[DataStoreT]): if state: raise Exception("'state' data store already configured") - state = StateGroupDataStore(database, db_conn, hs) + state_deletion = StateDeletionDataStore(database, db_conn, hs) + state = StateGroupDataStore(database, db_conn, hs, state_deletion) db_conn.commit() @@ -135,7 +140,7 @@ class Databases(Generic[DataStoreT]): if not main: raise Exception("No 'main' database configured") - if not state: + if not state or not state_deletion: raise Exception("No 'state' database configured") # We use local variables here to ensure that the databases do not have @@ -143,3 +148,4 @@ class Databases(Generic[DataStoreT]): self.main = main # type: ignore[assignment] self.state = state self.persist_events = persist_events + self.state_deletion = state_deletion diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py new file mode 100644 index 0000000000..07dbbc8e75 --- /dev/null +++ b/synapse/storage/databases/state/deletion.py @@ -0,0 +1,446 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import contextlib +from typing import ( + TYPE_CHECKING, + AbstractSet, + AsyncIterator, + Collection, + Mapping, + Set, + Tuple, +) + +from synapse.events import EventBase +from synapse.events.snapshot import EventContext +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, + make_in_list_sql_clause, +) +from synapse.storage.engines import PostgresEngine +from synapse.util.stringutils import shortstr + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class StateDeletionDataStore: + """Manages deletion of state groups in a safe manner. + + Deleting state groups is challenging as before we actually delete them we + need to ensure that there are no in-flight events that refer to the state + groups that we want to delete. + + To handle this, we take two approaches. First, before we persist any event + we ensure that the state group still exists and mark in the + `state_groups_persisting` table that the state group is about to be used. + (Note that we have to have the extra table here as state groups and events + can be in different databases, and thus we can't check for the existence of + state groups in the persist event transaction). Once the event has been + persisted, we can remove the row from `state_groups_persisting`. So long as + we check that table before deleting state groups, we can ensure that we + never persist events that reference deleted state groups, maintaining + database integrity. + + However, we want to avoid throwing exceptions so deep in the process of + persisting events. So instead of deleting state groups immediately, we mark + them as pending/proposed for deletion and wait for a certain amount of time + before performing the deletion. When we come to handle new events that + reference state groups, we check if they are pending deletion and bump the + time for when they'll be deleted (to give a chance for the event to be + persisted, or not). + + When deleting, we need to check that state groups remain unreferenced. There + is a race here where we a) fetch state groups that are ready for deletion, + b) check they're unreferenced, c) the state group becomes referenced but + then gets marked as pending deletion again, d) during the deletion + transaction we recheck `state_groups_pending_deletion` table again and see + that it exists and so continue with the deletion. To prevent this from + happening we add a `sequence_number` column to + `state_groups_pending_deletion`, and during deletion we ensure that for a + state group we're about to delete that the sequence number doesn't change + between steps (a) and (d). So long as we always bump the sequence number + whenever an event may become used the race can never happen. + """ + + # How long to wait before we delete state groups. This should be long enough + # for any in-flight events to be persisted. If events take longer to persist + # and any of the state groups they reference have been deleted, then the + # event will fail to persist (as well as any event in the same batch). + DELAY_BEFORE_DELETION_MS = 10 * 60 * 1000 + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + self._clock = hs.get_clock() + self.db_pool = database + self._instance_name = hs.get_instance_name() + + # TODO: Clear from `state_groups_persisting` any holdovers from previous + # running instance. + + async def check_state_groups_and_bump_deletion( + self, state_groups: AbstractSet[int] + ) -> Collection[int]: + """Checks to make sure that the state groups haven't been deleted, and + if they're pending deletion we delay it (allowing time for any event + that will use them to finish persisting). + + Returns: + The state groups that are missing, if any. + """ + + return await self.db_pool.runInteraction( + "check_state_groups_and_bump_deletion", + self._check_state_groups_and_bump_deletion_txn, + state_groups, + ) + + def _check_state_groups_and_bump_deletion_txn( + self, txn: LoggingTransaction, state_groups: AbstractSet[int] + ) -> Collection[int]: + existing_state_groups = self._get_existing_groups_with_lock(txn, state_groups) + + self._bump_deletion_txn(txn, existing_state_groups) + + missing_state_groups = state_groups - existing_state_groups + if missing_state_groups: + return missing_state_groups + + return () + + def _bump_deletion_txn( + self, txn: LoggingTransaction, state_groups: Collection[int] + ) -> None: + """Update any pending deletions of the state group that they may now be + referenced.""" + + if not state_groups: + return + + now = self._clock.time_msec() + if isinstance(self.db_pool.engine, PostgresEngine): + clause, args = make_in_list_sql_clause( + self.db_pool.engine, "state_group", state_groups + ) + sql = f""" + UPDATE state_groups_pending_deletion + SET sequence_number = DEFAULT, insertion_ts = ? + WHERE {clause} + """ + args.insert(0, now) + txn.execute(sql, args) + else: + rows = self.db_pool.simple_select_many_txn( + txn, + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + keyvalues={}, + retcols=("state_group",), + ) + if not rows: + return + + state_groups_to_update = [state_group for (state_group,) in rows] + + self.db_pool.simple_delete_many_txn( + txn, + table="state_groups_pending_deletion", + column="state_group", + values=state_groups_to_update, + keyvalues={}, + ) + self.db_pool.simple_insert_many_txn( + txn, + table="state_groups_pending_deletion", + keys=("state_group", "insertion_ts"), + values=[(state_group, now) for state_group in state_groups_to_update], + ) + + def _get_existing_groups_with_lock( + self, txn: LoggingTransaction, state_groups: Collection[int] + ) -> AbstractSet[int]: + """Return which of the given state groups are in the database, and locks + those rows with `KEY SHARE` to ensure they don't get concurrently + deleted.""" + clause, args = make_in_list_sql_clause(self.db_pool.engine, "id", state_groups) + + sql = f""" + SELECT id FROM state_groups + WHERE {clause} + """ + if isinstance(self.db_pool.engine, PostgresEngine): + # On postgres we add a row level lock to the rows to ensure that we + # conflict with any concurrent DELETEs. `FOR KEY SHARE` lock will + # not conflict with other read + sql += """ + FOR KEY SHARE + """ + + txn.execute(sql, args) + return {state_group for (state_group,) in txn} + + @contextlib.asynccontextmanager + async def persisting_state_group_references( + self, event_and_contexts: Collection[Tuple[EventBase, EventContext]] + ) -> AsyncIterator[None]: + """Wraps the persistence of the given events and contexts, ensuring that + any state groups referenced still exist and that they don't get deleted + during this.""" + + referenced_state_groups: Set[int] = set() + for event, ctx in event_and_contexts: + if ctx.rejected or event.internal_metadata.is_outlier(): + continue + + assert ctx.state_group is not None + + referenced_state_groups.add(ctx.state_group) + + if ctx.state_group_before_event: + referenced_state_groups.add(ctx.state_group_before_event) + + if not referenced_state_groups: + # We don't reference any state groups, so nothing to do + yield + return + + await self.db_pool.runInteraction( + "mark_state_groups_as_persisting", + self._mark_state_groups_as_persisting_txn, + referenced_state_groups, + ) + + error = True + try: + yield None + error = False + finally: + await self.db_pool.runInteraction( + "finish_persisting", + self._finish_persisting_txn, + referenced_state_groups, + error=error, + ) + + def _mark_state_groups_as_persisting_txn( + self, txn: LoggingTransaction, state_groups: Set[int] + ) -> None: + """Marks the given state groups as being persisted.""" + + existing_state_groups = self._get_existing_groups_with_lock(txn, state_groups) + missing_state_groups = state_groups - existing_state_groups + if missing_state_groups: + raise Exception( + f"state groups have been deleted: {shortstr(missing_state_groups)}" + ) + + self.db_pool.simple_insert_many_txn( + txn, + table="state_groups_persisting", + keys=("state_group", "instance_name"), + values=[(state_group, self._instance_name) for state_group in state_groups], + ) + + def _finish_persisting_txn( + self, txn: LoggingTransaction, state_groups: Collection[int], error: bool + ) -> None: + """Mark the state groups as having finished persistence. + + If `error` is true then we assume the state groups were not persisted, + and so we do not clear them from the pending deletion table. + """ + self.db_pool.simple_delete_many_txn( + txn, + table="state_groups_persisting", + column="state_group", + values=state_groups, + keyvalues={"instance_name": self._instance_name}, + ) + + if error: + # The state groups may or may not have been persisted, so we need to + # bump the deletion to ensure we recheck if they have become + # referenced. + self._bump_deletion_txn(txn, state_groups) + return + + self.db_pool.simple_delete_many_batch_txn( + txn, + table="state_groups_pending_deletion", + keys=("state_group",), + values=[(state_group,) for state_group in state_groups], + ) + + async def mark_state_groups_as_pending_deletion( + self, state_groups: Collection[int] + ) -> None: + """Mark the given state groups as pending deletion""" + + now = self._clock.time_msec() + + await self.db_pool.simple_upsert_many( + table="state_groups_pending_deletion", + key_names=("state_group",), + key_values=[(state_group,) for state_group in state_groups], + value_names=("insertion_ts",), + value_values=[(now,) for _ in state_groups], + desc="mark_state_groups_as_pending_deletion", + ) + + async def get_pending_deletions( + self, state_groups: Collection[int] + ) -> Mapping[int, int]: + """Get which state groups are pending deletion. + + Returns: + a mapping from state groups that are pending deletion to their + sequence number + """ + + rows = await self.db_pool.simple_select_many_batch( + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + retcols=("state_group", "sequence_number"), + keyvalues={}, + desc="get_pending_deletions", + ) + + return dict(rows) + + def get_state_groups_ready_for_potential_deletion_txn( + self, + txn: LoggingTransaction, + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> Collection[int]: + """Given a set of state groups, return which state groups can + potentially be deleted. + + The state groups must have been checked to see if they remain + unreferenced before calling this function. + + Note: This must be called within the same transaction that the state + groups are deleted. + + Args: + state_groups_to_sequence_numbers: The state groups, and the sequence + numbers from before the state groups were checked to see if they + were unreferenced. + + Returns: + The subset of state groups that can safely be deleted + + """ + + if not state_groups_to_sequence_numbers: + return state_groups_to_sequence_numbers + + if isinstance(self.db_pool.engine, PostgresEngine): + # On postgres we want to lock the rows FOR UPDATE as early as + # possible to help conflicts. + clause, args = make_in_list_sql_clause( + self.db_pool.engine, "id", state_groups_to_sequence_numbers + ) + sql = f""" + SELECT id FROM state_groups + WHERE {clause} + FOR UPDATE + """ + txn.execute(sql, args) + + # Check the deletion status in the DB of the given state groups + clause, args = make_in_list_sql_clause( + self.db_pool.engine, + column="state_group", + iterable=state_groups_to_sequence_numbers, + ) + + sql = f""" + SELECT state_group, insertion_ts, sequence_number FROM ( + SELECT state_group, insertion_ts, sequence_number FROM state_groups_pending_deletion + UNION + SELECT state_group, null, null FROM state_groups_persisting + ) AS s + WHERE {clause} + """ + + txn.execute(sql, args) + + # The above query will return potentially two rows per state group (one + # for each table), so we track which state groups have enough time + # elapsed and which are not ready to be persisted. + ready_to_be_deleted = set() + not_ready_to_be_deleted = set() + + now = self._clock.time_msec() + for state_group, insertion_ts, sequence_number in txn: + if insertion_ts is None: + # A null insertion_ts means that we are currently persisting + # events that reference the state group, so we don't delete + # them. + not_ready_to_be_deleted.add(state_group) + continue + + # We know this can't be None if insertion_ts is not None + assert sequence_number is not None + + # Check if the sequence number has changed, if it has then it + # indicates that the state group may have become referenced since we + # checked. + if state_groups_to_sequence_numbers[state_group] != sequence_number: + not_ready_to_be_deleted.add(state_group) + continue + + if now - insertion_ts < self.DELAY_BEFORE_DELETION_MS: + # Not enough time has elapsed to allow us to delete. + not_ready_to_be_deleted.add(state_group) + continue + + ready_to_be_deleted.add(state_group) + + can_be_deleted = ready_to_be_deleted - not_ready_to_be_deleted + if not_ready_to_be_deleted: + # If there are any state groups that aren't ready to be deleted, + # then we also need to remove any state groups that are referenced + # by them. + clause, args = make_in_list_sql_clause( + self.db_pool.engine, + column="state_group", + iterable=state_groups_to_sequence_numbers, + ) + sql = f""" + WITH RECURSIVE ancestors(state_group) AS ( + SELECT DISTINCT prev_state_group + FROM state_group_edges WHERE {clause} + UNION + SELECT prev_state_group + FROM state_group_edges + INNER JOIN ancestors USING (state_group) + ) + SELECT state_group FROM ancestors + """ + txn.execute(sql, args) + + can_be_deleted.difference_update(state_group for (state_group,) in txn) + + return can_be_deleted diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 9944f90015..7e986e0601 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -36,7 +36,10 @@ import attr from synapse.api.constants import EventTypes from synapse.events import EventBase -from synapse.events.snapshot import UnpersistedEventContext, UnpersistedEventContextBase +from synapse.events.snapshot import ( + UnpersistedEventContext, + UnpersistedEventContextBase, +) from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( @@ -55,6 +58,7 @@ from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer + from synapse.storage.databases.state.deletion import StateDeletionDataStore logger = logging.getLogger(__name__) @@ -83,8 +87,10 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): database: DatabasePool, db_conn: LoggingDatabaseConnection, hs: "HomeServer", + state_deletion_store: "StateDeletionDataStore", ): super().__init__(database, db_conn, hs) + self._state_deletion_store = state_deletion_store # Originally the state store used a single DictionaryCache to cache the # event IDs for the state types in a given state group to avoid hammering @@ -467,14 +473,15 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): Returns: A list of state groups """ - is_in_db = self.db_pool.simple_select_one_onecol_txn( - txn, - table="state_groups", - keyvalues={"id": prev_group}, - retcol="id", - allow_none=True, + + # We need to check that the prev group isn't about to be deleted + is_missing = ( + self._state_deletion_store._check_state_groups_and_bump_deletion_txn( + txn, + {prev_group}, + ) ) - if not is_in_db: + if is_missing: raise Exception( "Trying to persist state with unpersisted prev_group: %r" % (prev_group,) @@ -546,6 +553,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): for key, state_id in context.state_delta_due_to_event.items() ], ) + return events_and_context return await self.db_pool.runInteraction( @@ -601,14 +609,15 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): The state group if successfully created, or None if the state needs to be persisted as a full state. """ - is_in_db = self.db_pool.simple_select_one_onecol_txn( - txn, - table="state_groups", - keyvalues={"id": prev_group}, - retcol="id", - allow_none=True, + + # We need to check that the prev group isn't about to be deleted + is_missing = ( + self._state_deletion_store._check_state_groups_and_bump_deletion_txn( + txn, + {prev_group}, + ) ) - if not is_in_db: + if is_missing: raise Exception( "Trying to persist state with unpersisted prev_group: %r" % (prev_group,) diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 934e1ccced..49e648a92f 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 88 # remember to update the list below when updating +SCHEMA_VERSION = 89 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -155,6 +155,9 @@ Changes in SCHEMA_VERSION = 88 be posted in response to a resettable timeout or an on-demand action. - Add background update to fix data integrity issue in the `sliding_sync_membership_snapshots` -> `forgotten` column + +Changes in SCHEMA_VERSION = 89 + - Add `state_groups_pending_deletion` and `state_groups_persisting` tables. """ diff --git a/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql b/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql new file mode 100644 index 0000000000..d4cb27a3a2 --- /dev/null +++ b/synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql @@ -0,0 +1,39 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- See the `StateDeletionDataStore` for details of these tables. + +-- We add state groups to this table when we want to later delete them. The +-- `insertion_ts` column indicates when the state group was proposed for +-- deletion (rather than when it should be deleted). +CREATE TABLE IF NOT EXISTS state_groups_pending_deletion ( + sequence_number $%AUTO_INCREMENT_PRIMARY_KEY%$, + state_group BIGINT NOT NULL, + insertion_ts BIGINT NOT NULL +); + +CREATE UNIQUE INDEX state_groups_pending_deletion_state_group ON state_groups_pending_deletion(state_group); +CREATE INDEX state_groups_pending_deletion_insertion_ts ON state_groups_pending_deletion(insertion_ts); + + +-- Holds the state groups the worker is currently persisting. +-- +-- The `sequence_number` column of the `state_groups_pending_deletion` table +-- *must* be updated whenever a state group may have become referenced. +CREATE TABLE IF NOT EXISTS state_groups_persisting ( + state_group BIGINT NOT NULL, + instance_name TEXT NOT NULL, + PRIMARY KEY (state_group, instance_name) +); + +CREATE INDEX state_groups_persisting_instance_name ON state_groups_persisting(instance_name); diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 61b0efb87e..51eca56c3b 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -807,6 +807,7 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): OTHER_USER = f"@user:{self.OTHER_SERVER_NAME}" main_store = self.hs.get_datastores().main + state_deletion_store = self.hs.get_datastores().state_deletion # Create the room. kermit_user_id = self.register_user("kermit", "test") @@ -958,7 +959,9 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): bert_member_event.event_id: bert_member_event, rejected_kick_event.event_id: rejected_kick_event, }, - state_res_store=StateResolutionStore(main_store), + state_res_store=StateResolutionStore( + main_store, state_deletion_store + ), ) ), [bert_member_event.event_id, rejected_kick_event.event_id], @@ -1003,7 +1006,9 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): rejected_power_levels_event.event_id, ], event_map={}, - state_res_store=StateResolutionStore(main_store), + state_res_store=StateResolutionStore( + main_store, state_deletion_store + ), full_conflicted_set=set(), ) ), diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 833bd6fff8..3bb539bf87 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -742,7 +742,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(34, channel.resource_usage.db_txn_count) + self.assertEqual(36, channel.resource_usage.db_txn_count) def test_post_room_initial_state(self) -> None: # POST with initial_state config key, expect new room id @@ -755,7 +755,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(36, channel.resource_usage.db_txn_count) + self.assertEqual(38, channel.resource_usage.db_txn_count) def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id diff --git a/tests/storage/test_state_deletion.py b/tests/storage/test_state_deletion.py new file mode 100644 index 0000000000..19b290b554 --- /dev/null +++ b/tests/storage/test_state_deletion.py @@ -0,0 +1,411 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.util import Clock + +from tests.test_utils.event_injection import create_event +from tests.unittest import HomeserverTestCase + +logger = logging.getLogger(__name__) + + +class StateDeletionStoreTestCase(HomeserverTestCase): + """Tests for the StateDeletionStore.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.state_store = hs.get_datastores().state + self.state_deletion_store = hs.get_datastores().state_deletion + + self.user_id = self.register_user("test", "password") + tok = self.login("test", "password") + self.room_id = self.helper.create_room_as(self.user_id, tok=tok) + + def check_if_can_be_deleted(self, state_group: int) -> bool: + """Check if the state group is pending deletion.""" + + state_group_to_sequence_number = self.get_success( + self.state_deletion_store.get_pending_deletions([state_group]) + ) + + can_be_deleted = self.get_success( + self.state_deletion_store.db_pool.runInteraction( + "test_existing_pending_deletion_is_cleared", + self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn, + state_group_to_sequence_number, + ) + ) + + return state_group in can_be_deleted + + def test_no_deletion(self) -> None: + """Test that calling persisting_state_group_references is fine if + nothing is pending deletion""" + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + sender=self.user_id, + ) + ) + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + + self.get_success(ctx_mgr.__aenter__()) + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + def test_no_deletion_error(self) -> None: + """Test that calling persisting_state_group_references is fine if + nothing is pending deletion, but an error occurs.""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + sender=self.user_id, + ) + ) + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + + self.get_success(ctx_mgr.__aenter__()) + self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None)) + + def test_existing_pending_deletion_is_cleared(self) -> None: + """Test that the pending deletion flag gets cleared when the state group + gets persisted.""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + # Mark a state group that we're referencing as pending deletion. + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + + self.get_success(ctx_mgr.__aenter__()) + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # The pending deletion flag should be cleared + pending_deletion = self.get_success( + self.state_deletion_store.db_pool.simple_select_one_onecol( + table="state_groups_pending_deletion", + keyvalues={"state_group": context.state_group}, + retcol="1", + allow_none=True, + desc="test_existing_pending_deletion_is_cleared", + ) + ) + self.assertIsNone(pending_deletion) + + def test_pending_deletion_is_cleared_during_persist(self) -> None: + """Test that the pending deletion flag is cleared when a state group + gets marked for deletion during persistence""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + self.get_success(ctx_mgr.__aenter__()) + + # Mark the state group that we're referencing as pending deletion, + # *after* we have started persisting. + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # The pending deletion flag should be cleared + pending_deletion = self.get_success( + self.state_deletion_store.db_pool.simple_select_one_onecol( + table="state_groups_pending_deletion", + keyvalues={"state_group": context.state_group}, + retcol="1", + allow_none=True, + desc="test_existing_pending_deletion_is_cleared", + ) + ) + self.assertIsNone(pending_deletion) + + def test_deletion_check(self) -> None: + """Test that the `get_state_groups_that_can_be_purged_txn` check is + correct during different points of the lifecycle of persisting an + event.""" + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + # We shouldn't be able to delete the state group as not enough time as passed + can_be_deleted = self.check_if_can_be_deleted(context.state_group) + self.assertFalse(can_be_deleted) + + # After enough time we can delete the state group + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + can_be_deleted = self.check_if_can_be_deleted(context.state_group) + self.assertTrue(can_be_deleted) + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + self.get_success(ctx_mgr.__aenter__()) + + # But once we start persisting we can't delete the state group + can_be_deleted = self.check_if_can_be_deleted(context.state_group) + self.assertFalse(can_be_deleted) + + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # The pending deletion flag should remain cleared after persistence has + # finished. + can_be_deleted = self.check_if_can_be_deleted(context.state_group) + self.assertFalse(can_be_deleted) + + def test_deletion_error_during_persistence(self) -> None: + """Test that state groups remain marked as pending deletion if persisting + the event fails.""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + # Mark a state group that we're referencing as pending deletion. + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + + self.get_success(ctx_mgr.__aenter__()) + self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None)) + + # We should be able to delete the state group after a certain amount of + # time + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + can_be_deleted = self.check_if_can_be_deleted(context.state_group) + self.assertTrue(can_be_deleted) + + def test_race_between_check_and_insert(self) -> None: + """Check that we correctly handle the race where we go to delete a + state group, check that it is unreferenced, and then it becomes + referenced just before we delete it.""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + # Mark a state group that we're referencing as pending deletion. + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + # Advance time enough so we can delete the state group + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # Check that we'd be able to delete this state group. + state_group_to_sequence_number = self.get_success( + self.state_deletion_store.get_pending_deletions([context.state_group]) + ) + + can_be_deleted = self.get_success( + self.state_deletion_store.db_pool.runInteraction( + "test_existing_pending_deletion_is_cleared", + self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn, + state_group_to_sequence_number, + ) + ) + self.assertCountEqual(can_be_deleted, [context.state_group]) + + # ... in the real world we'd check that the state group isn't referenced here ... + + # Now we persist the event to reference the state group, *after* we + # check that the state group wasn't referenced + ctx_mgr = self.state_deletion_store.persisting_state_group_references( + [(event, context)] + ) + + self.get_success(ctx_mgr.__aenter__()) + self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None)) + + # We simulate a pause (required to hit the race) + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We should no longer be able to delete the state group, without having + # to recheck if its referenced. + can_be_deleted = self.get_success( + self.state_deletion_store.db_pool.runInteraction( + "test_existing_pending_deletion_is_cleared", + self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn, + state_group_to_sequence_number, + ) + ) + self.assertCountEqual(can_be_deleted, []) + + def test_remove_ancestors_from_can_delete(self) -> None: + """Test that if a state group is not ready to be deleted, we also don't + delete anything that is refernced by it""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + # Create a new state group that refernces the one from the event + new_state_group = self.get_success( + self.state_store.store_state_group( + event.event_id, + event.room_id, + prev_group=context.state_group, + delta_ids={}, + current_state_ids=None, + ) + ) + + # Mark them both as pending deletion + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group, new_state_group] + ) + ) + + # Advance time enough so we can delete the state group so they're both + # ready for deletion. + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We can now delete both state groups + self.assertTrue(self.check_if_can_be_deleted(context.state_group)) + self.assertTrue(self.check_if_can_be_deleted(new_state_group)) + + # Use the new_state_group to bump its deletion time + self.get_success( + self.state_store.store_state_group( + event.event_id, + event.room_id, + prev_group=new_state_group, + delta_ids={}, + current_state_ids=None, + ) + ) + + # We should now not be able to delete either of the state groups. + state_group_to_sequence_number = self.get_success( + self.state_deletion_store.get_pending_deletions( + [context.state_group, new_state_group] + ) + ) + + # We shouldn't be able to delete the state group as not enough time has passed + can_be_deleted = self.get_success( + self.state_deletion_store.db_pool.runInteraction( + "test_existing_pending_deletion_is_cleared", + self.state_deletion_store.get_state_groups_ready_for_potential_deletion_txn, + state_group_to_sequence_number, + ) + ) + self.assertCountEqual(can_be_deleted, []) diff --git a/tests/test_state.py b/tests/test_state.py index 311a590693..dce56fe78a 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -31,7 +31,7 @@ from typing import ( Tuple, cast, ) -from unittest.mock import Mock +from unittest.mock import AsyncMock, Mock from twisted.internet import defer @@ -221,7 +221,16 @@ class Graph: class StateTestCase(unittest.TestCase): def setUp(self) -> None: self.dummy_store = _DummyStore() - storage_controllers = Mock(main=self.dummy_store, state=self.dummy_store) + + # Add a dummy epoch store that always retruns that we have all the + # necessary state groups. + dummy_deletion_store = AsyncMock() + dummy_deletion_store.check_state_groups_and_bump_deletion.return_value = [] + + storage_controllers = Mock( + main=self.dummy_store, + state=self.dummy_store, + ) hs = Mock( spec_set=[ "config", @@ -241,7 +250,10 @@ class StateTestCase(unittest.TestCase): ) clock = cast(Clock, MockClock()) hs.config = default_config("tesths", True) - hs.get_datastores.return_value = Mock(main=self.dummy_store) + hs.get_datastores.return_value = Mock( + main=self.dummy_store, + state_deletion=dummy_deletion_store, + ) hs.get_state_handler.return_value = None hs.get_clock.return_value = clock hs.get_macaroon_generator.return_value = MacaroonGenerator( From 27dbb1b4290b9de64e24a11f892777378810b595 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Feb 2025 18:58:55 +0100 Subject: [PATCH 115/691] Add locking to more safely delete state groups: Part 2 (#18130) This actually makes it so that deleting state groups goes via the new mechanism. c.f. #18107 --- changelog.d/18130.bugfix | 1 + synapse/storage/controllers/purge_events.py | 80 +++++++++++++++++++-- synapse/storage/databases/state/deletion.py | 65 +++++++++++++++++ synapse/storage/databases/state/store.py | 28 ++++++-- tests/rest/client/utils.py | 2 +- tests/storage/test_purge.py | 67 +++++++++++++++++ tests/storage/test_state_deletion.py | 68 +++++++++++++++++- 7 files changed, 297 insertions(+), 14 deletions(-) create mode 100644 changelog.d/18130.bugfix diff --git a/changelog.d/18130.bugfix b/changelog.d/18130.bugfix new file mode 100644 index 0000000000..4d0c19fab9 --- /dev/null +++ b/changelog.d/18130.bugfix @@ -0,0 +1 @@ +Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 15c04ffef8..2d6f80f770 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -21,9 +21,10 @@ import itertools import logging -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING, Collection, Mapping, Set from synapse.logging.context import nested_logging_context +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases import Databases if TYPE_CHECKING: @@ -38,6 +39,11 @@ class PurgeEventsStorageController: def __init__(self, hs: "HomeServer", stores: Databases): self.stores = stores + if hs.config.worker.run_background_tasks: + self._delete_state_loop_call = hs.get_clock().looping_call( + self._delete_state_groups_loop, 60 * 1000 + ) + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room""" @@ -68,11 +74,15 @@ class PurgeEventsStorageController: logger.info("[purge] finding state groups that can be deleted") sg_to_delete = await self._find_unreferenced_groups(state_groups) - await self.stores.state.purge_unreferenced_state_groups( - room_id, sg_to_delete + # Mark these state groups as pending deletion, they will actually + # get deleted automatically later. + await self.stores.state_deletion.mark_state_groups_as_pending_deletion( + sg_to_delete ) - async def _find_unreferenced_groups(self, state_groups: Set[int]) -> Set[int]: + async def _find_unreferenced_groups( + self, state_groups: Collection[int] + ) -> Set[int]: """Used when purging history to figure out which state groups can be deleted. @@ -121,3 +131,65 @@ class PurgeEventsStorageController: to_delete = state_groups_seen - referenced_groups return to_delete + + @wrap_as_background_process("_delete_state_groups_loop") + async def _delete_state_groups_loop(self) -> None: + """Background task that deletes any state groups that may be pending + deletion.""" + + while True: + next_to_delete = await self.stores.state_deletion.get_next_state_group_collection_to_delete() + if next_to_delete is None: + break + + (room_id, groups_to_sequences) = next_to_delete + made_progress = await self._delete_state_groups( + room_id, groups_to_sequences + ) + + # If no progress was made in deleting the state groups, then we + # break to allow a pause before trying again next time we get + # called. + if not made_progress: + break + + async def _delete_state_groups( + self, room_id: str, groups_to_sequences: Mapping[int, int] + ) -> bool: + """Tries to delete the given state groups. + + Returns: + Whether we made progress in deleting the state groups (or marking + them as referenced). + """ + + # We double check if any of the state groups have become referenced. + # This shouldn't happen, as any usages should cause the state group to + # be removed as pending deletion. + referenced_state_groups = await self.stores.main.get_referenced_state_groups( + groups_to_sequences + ) + + if referenced_state_groups: + # We mark any state groups that have become referenced as being + # used. + await self.stores.state_deletion.mark_state_groups_as_used( + referenced_state_groups + ) + + # Update list of state groups to remove referenced ones + groups_to_sequences = { + state_group: sequence_number + for state_group, sequence_number in groups_to_sequences.items() + if state_group not in referenced_state_groups + } + + if not groups_to_sequences: + # We made progress here as long as we marked some state groups as + # now referenced. + return len(referenced_state_groups) > 0 + + return await self.stores.state.purge_unreferenced_state_groups( + room_id, + groups_to_sequences, + ) diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index 07dbbc8e75..4853e5aa2f 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -20,6 +20,7 @@ from typing import ( AsyncIterator, Collection, Mapping, + Optional, Set, Tuple, ) @@ -307,6 +308,17 @@ class StateDeletionDataStore: desc="mark_state_groups_as_pending_deletion", ) + async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None: + """Mark the given state groups as now being referenced""" + + await self.db_pool.simple_delete_many( + table="state_groups_pending_deletion", + column="state_group", + iterable=state_groups, + keyvalues={}, + desc="mark_state_groups_as_used", + ) + async def get_pending_deletions( self, state_groups: Collection[int] ) -> Mapping[int, int]: @@ -444,3 +456,56 @@ class StateDeletionDataStore: can_be_deleted.difference_update(state_group for (state_group,) in txn) return can_be_deleted + + async def get_next_state_group_collection_to_delete( + self, + ) -> Optional[Tuple[str, Mapping[int, int]]]: + """Get the next set of state groups to try and delete + + Returns: + 2-tuple of room_id and mapping of state groups to sequence number. + """ + return await self.db_pool.runInteraction( + "get_next_state_group_collection_to_delete", + self._get_next_state_group_collection_to_delete_txn, + ) + + def _get_next_state_group_collection_to_delete_txn( + self, + txn: LoggingTransaction, + ) -> Optional[Tuple[str, Mapping[int, int]]]: + """Implementation of `get_next_state_group_collection_to_delete`""" + + # We want to return chunks of state groups that were marked for deletion + # at the same time (this isn't necessary, just more efficient). We do + # this by looking for the oldest insertion_ts, and then pulling out all + # rows that have the same insertion_ts (and room ID). + now = self._clock.time_msec() + + sql = """ + SELECT room_id, insertion_ts + FROM state_groups_pending_deletion AS sd + INNER JOIN state_groups AS sg ON (id = sd.state_group) + LEFT JOIN state_groups_persisting AS sp USING (state_group) + WHERE insertion_ts < ? AND sp.state_group IS NULL + ORDER BY insertion_ts + LIMIT 1 + """ + txn.execute(sql, (now - self.DELAY_BEFORE_DELETION_MS,)) + row = txn.fetchone() + if not row: + return None + + (room_id, insertion_ts) = row + + sql = """ + SELECT state_group, sequence_number + FROM state_groups_pending_deletion AS sd + INNER JOIN state_groups AS sg ON (id = sd.state_group) + LEFT JOIN state_groups_persisting AS sp USING (state_group) + WHERE room_id = ? AND insertion_ts = ? AND sp.state_group IS NULL + ORDER BY insertion_ts + """ + txn.execute(sql, (room_id, insertion_ts)) + + return room_id, dict(txn) diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 7e986e0601..0f47642ae5 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -22,10 +22,10 @@ import logging from typing import ( TYPE_CHECKING, - Collection, Dict, Iterable, List, + Mapping, Optional, Set, Tuple, @@ -735,8 +735,10 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): ) async def purge_unreferenced_state_groups( - self, room_id: str, state_groups_to_delete: Collection[int] - ) -> None: + self, + room_id: str, + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> bool: """Deletes no longer referenced state groups and de-deltas any state groups that reference them. @@ -744,21 +746,31 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): room_id: The room the state groups belong to (must all be in the same room). state_groups_to_delete: Set of all state groups to delete. + + Returns: + Whether any state groups were actually deleted. """ - await self.db_pool.runInteraction( + return await self.db_pool.runInteraction( "purge_unreferenced_state_groups", self._purge_unreferenced_state_groups, room_id, - state_groups_to_delete, + state_groups_to_sequence_numbers, ) def _purge_unreferenced_state_groups( self, txn: LoggingTransaction, room_id: str, - state_groups_to_delete: Collection[int], - ) -> None: + state_groups_to_sequence_numbers: Mapping[int, int], + ) -> bool: + state_groups_to_delete = self._state_deletion_store.get_state_groups_ready_for_potential_deletion_txn( + txn, state_groups_to_sequence_numbers + ) + + if not state_groups_to_delete: + return False + logger.info( "[purge] found %i state groups to delete", len(state_groups_to_delete) ) @@ -821,6 +833,8 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): [(sg,) for sg in state_groups_to_delete], ) + return True + @trace @tag_args async def get_previous_state_groups( diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index dbd6049f9f..e766630afb 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -548,7 +548,7 @@ class RestHelper: room_id: str, event_type: str, body: Dict[str, Any], - tok: Optional[str], + tok: Optional[str] = None, expect_code: int = HTTPStatus.OK, state_key: str = "", ) -> JsonDict: diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 080d5640a5..efd8d25bd1 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -23,6 +23,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import NotFoundError, SynapseError from synapse.rest.client import room from synapse.server import HomeServer +from synapse.types.state import StateFilter from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -40,6 +41,8 @@ class PurgeTests(HomeserverTestCase): self.room_id = self.helper.create_room_as(self.user_id) self.store = hs.get_datastores().main + self.state_store = hs.get_datastores().state + self.state_deletion_store = hs.get_datastores().state_deletion self._storage_controllers = self.hs.get_storage_controllers() def test_purge_history(self) -> None: @@ -128,3 +131,67 @@ class PurgeTests(HomeserverTestCase): self.store._invalidate_local_get_event_cache(create_event.event_id) self.get_failure(self.store.get_event(create_event.event_id), NotFoundError) self.get_failure(self.store.get_event(first["event_id"]), NotFoundError) + + def test_purge_history_deletes_state_groups(self) -> None: + """Test that unreferenced state groups get cleaned up after purge""" + + # Send four state changes to the room. + first = self.helper.send_state( + self.room_id, event_type="m.foo", body={"test": 1} + ) + second = self.helper.send_state( + self.room_id, event_type="m.foo", body={"test": 2} + ) + third = self.helper.send_state( + self.room_id, event_type="m.foo", body={"test": 3} + ) + last = self.helper.send_state( + self.room_id, event_type="m.foo", body={"test": 4} + ) + + # Get references to the state groups + event_to_groups = self.get_success( + self.store._get_state_group_for_events( + [ + first["event_id"], + second["event_id"], + third["event_id"], + last["event_id"], + ] + ) + ) + + # Get the topological token + token = self.get_success( + self.store.get_topological_token_for_event(last["event_id"]) + ) + token_str = self.get_success(token.to_string(self.hs.get_datastores().main)) + + # Purge everything before this topological token + self.get_success( + self._storage_controllers.purge_events.purge_history( + self.room_id, token_str, True + ) + ) + + # Advance so that the background jobs to delete the state groups runs + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We expect all the state groups associated with events above, except + # the last one, should return no state. + state_groups = self.get_success( + self.state_store._get_state_groups_from_groups( + list(event_to_groups.values()), StateFilter.all() + ) + ) + first_state = state_groups[event_to_groups[first["event_id"]]] + second_state = state_groups[event_to_groups[second["event_id"]]] + third_state = state_groups[event_to_groups[third["event_id"]]] + last_state = state_groups[event_to_groups[last["event_id"]]] + + self.assertEqual(first_state, {}) + self.assertEqual(second_state, {}) + self.assertEqual(third_state, {}) + self.assertNotEqual(last_state, {}) diff --git a/tests/storage/test_state_deletion.py b/tests/storage/test_state_deletion.py index 19b290b554..a4d318ae20 100644 --- a/tests/storage/test_state_deletion.py +++ b/tests/storage/test_state_deletion.py @@ -41,6 +41,11 @@ class StateDeletionStoreTestCase(HomeserverTestCase): self.store = hs.get_datastores().main self.state_store = hs.get_datastores().state self.state_deletion_store = hs.get_datastores().state_deletion + self.purge_events = hs.get_storage_controllers().purge_events + + # We want to disable the automatic deletion of state groups in the + # background, so we can do controlled tests. + self.purge_events._delete_state_loop_call.stop() self.user_id = self.register_user("test", "password") tok = self.login("test", "password") @@ -341,7 +346,7 @@ class StateDeletionStoreTestCase(HomeserverTestCase): def test_remove_ancestors_from_can_delete(self) -> None: """Test that if a state group is not ready to be deleted, we also don't - delete anything that is refernced by it""" + delete anything that is referenced by it""" event, context = self.get_success( create_event( @@ -354,7 +359,7 @@ class StateDeletionStoreTestCase(HomeserverTestCase): ) assert context.state_group is not None - # Create a new state group that refernces the one from the event + # Create a new state group that references the one from the event new_state_group = self.get_success( self.state_store.store_state_group( event.event_id, @@ -409,3 +414,62 @@ class StateDeletionStoreTestCase(HomeserverTestCase): ) ) self.assertCountEqual(can_be_deleted, []) + + def test_newly_referenced_state_group_gets_removed_from_pending(self) -> None: + """Check that if a state group marked for deletion becomes referenced + (without being removed from pending deletion table), it gets removed + from pending deletion table.""" + + event, context = self.get_success( + create_event( + self.hs, + room_id=self.room_id, + type="m.test", + state_key="", + sender=self.user_id, + ) + ) + assert context.state_group is not None + + # Mark a state group that we're referencing as pending deletion. + self.get_success( + self.state_deletion_store.mark_state_groups_as_pending_deletion( + [context.state_group] + ) + ) + + # Advance time enough so we can delete the state group so they're both + # ready for deletion. + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # Manually insert into the table to mimic the state group getting used. + self.get_success( + self.store.db_pool.simple_insert( + table="event_to_state_groups", + values={"state_group": context.state_group, "event_id": event.event_id}, + desc="test_newly_referenced_state_group_gets_removed_from_pending", + ) + ) + + # Manually run the background task to delete pending state groups. + self.get_success(self.purge_events._delete_state_groups_loop()) + + # The pending deletion flag should be cleared... + pending_deletion = self.get_success( + self.state_deletion_store.db_pool.simple_select_one_onecol( + table="state_groups_pending_deletion", + keyvalues={"state_group": context.state_group}, + retcol="1", + allow_none=True, + desc="test_newly_referenced_state_group_gets_removed_from_pending", + ) + ) + self.assertIsNone(pending_deletion) + + # .. but the state should not have been deleted. + state = self.get_success( + self.state_store._get_state_for_groups([context.state_group]) + ) + self.assertGreater(len(state[context.state_group]), 0) From c46d452c7cdf7289ae7e8677b44a88c747761dce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Feb 2025 20:04:19 +0100 Subject: [PATCH 116/691] Fix bug where purging history could lead to increase in disk space usage (#18131) When purging history, we try and delete any state groups that become unreferenced (i.e. there are no longer any events that directly reference them). When we delete a state group that is referenced by another state group, we "de-delta" that state group so that it no longer refers to the state group that is deleted. There are two bugs with this approach that we fix here: 1. There is a common pattern where we end up storing two state groups when persisting a state event: the state before and after the new state event, where the latter is stored as a delta to the former. When deleting state groups we only deleted the "new" state and left (and potentially de-deltaed) the old state. This was due to a bug/typo when trying to find referenced state groups. 2. There are times where we store unreferenced state groups in the DB, during the purging of history these would not get rechecked and instead always de-deltaed. Instead, we should check for this case and delete any unreferenced state groups rather than de-deltaing them. The effect of the above bugs is that when purging history we'd end up with lots of unreferenced state groups that had been de-deltaed (i.e. stored as the full state). This can lead to dramatic increases in storage space used. --- changelog.d/18131.bugfix | 1 + synapse/storage/controllers/purge_events.py | 10 +++ synapse/storage/databases/state/store.py | 31 ++++++++- tests/storage/test_purge.py | 75 +++++++++++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18131.bugfix diff --git a/changelog.d/18131.bugfix b/changelog.d/18131.bugfix new file mode 100644 index 0000000000..4d0c19fab9 --- /dev/null +++ b/changelog.d/18131.bugfix @@ -0,0 +1 @@ +Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 2d6f80f770..47cec8c469 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -128,6 +128,16 @@ class PurgeEventsStorageController: next_to_search |= prevs state_groups_seen |= prevs + # We also check to see if anything referencing the state groups are + # also unreferenced. This helps ensure that we delete unreferenced + # state groups, if we don't then we will de-delta them when we + # delete the other state groups leading to increased DB usage. + next_edges = await self.stores.state.get_next_state_groups(current_search) + nexts = set(next_edges.keys()) + nexts -= state_groups_seen + next_to_search |= nexts + state_groups_seen |= nexts + to_delete = state_groups_seen - referenced_groups return to_delete diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 0f47642ae5..8c7980e719 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -853,7 +853,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): List[Tuple[int, int]], await self.db_pool.simple_select_many_batch( table="state_group_edges", - column="prev_state_group", + column="state_group", iterable=state_groups, keyvalues={}, retcols=("state_group", "prev_state_group"), @@ -863,6 +863,35 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): return dict(rows) + @trace + @tag_args + async def get_next_state_groups( + self, state_groups: Iterable[int] + ) -> Dict[int, int]: + """Fetch the groups that have the given state groups as their previous + state groups. + + Args: + state_groups + + Returns: + A mapping from state group to previous state group. + """ + + rows = cast( + List[Tuple[int, int]], + await self.db_pool.simple_select_many_batch( + table="state_group_edges", + column="prev_state_group", + iterable=state_groups, + keyvalues={}, + retcols=("state_group", "prev_state_group"), + desc="get_next_state_groups", + ), + ) + + return dict(rows) + async def purge_room_state(self, room_id: str) -> None: return await self.db_pool.runInteraction( "purge_room_state", diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index efd8d25bd1..5d6a8518c0 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -195,3 +195,78 @@ class PurgeTests(HomeserverTestCase): self.assertEqual(second_state, {}) self.assertEqual(third_state, {}) self.assertNotEqual(last_state, {}) + + def test_purge_unreferenced_state_group(self) -> None: + """Test that purging a room also gets rid of unreferenced state groups + it encounters during the purge. + + This is important, as otherwise these unreferenced state groups get + "de-deltaed" during the purge process, consuming lots of disk space. + """ + + self.helper.send(self.room_id, body="test1") + state1 = self.helper.send_state( + self.room_id, "org.matrix.test", body={"number": 2} + ) + state2 = self.helper.send_state( + self.room_id, "org.matrix.test", body={"number": 3} + ) + self.helper.send(self.room_id, body="test4") + last = self.helper.send(self.room_id, body="test5") + + # Create an unreferenced state group that has a prev group of one of the + # to-be-purged events. + prev_group = self.get_success( + self.store._get_state_group_for_event(state1["event_id"]) + ) + unreferenced_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=prev_group, + delta_ids={("org.matrix.test", ""): state2["event_id"]}, + current_state_ids=None, + ) + ) + + # Get the topological token + token = self.get_success( + self.store.get_topological_token_for_event(last["event_id"]) + ) + token_str = self.get_success(token.to_string(self.hs.get_datastores().main)) + + # Purge everything before this topological token + self.get_success( + self._storage_controllers.purge_events.purge_history( + self.room_id, token_str, True + ) + ) + + # Advance so that the background jobs to delete the state groups runs + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We expect that the unreferenced state group has been deleted. + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": unreferenced_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + # We expect there to now only be one state group for the room, which is + # the state group of the last event (as the only outlier). + state_groups = self.get_success( + self.state_store.db_pool.simple_select_onecol( + table="state_groups", + keyvalues={"room_id": self.room_id}, + retcol="id", + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertEqual(len(state_groups), 1) From 37e893499f54872903c365aacc86a9d3eb6b7619 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Feb 2025 11:53:27 +0000 Subject: [PATCH 117/691] 1.124.0rc1 --- CHANGES.md | 27 +++++++++++++++++++++++++++ changelog.d/18000.bugfix | 1 - changelog.d/18073.bugfix | 1 - changelog.d/18075.bugfix | 1 - changelog.d/18089.bugfix | 2 -- changelog.d/18107.bugfix | 1 - changelog.d/18109.misc | 1 - changelog.d/18112.bugfix | 1 - changelog.d/18119.bugfix | 1 - changelog.d/18130.bugfix | 1 - changelog.d/18131.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 13 files changed, 34 insertions(+), 12 deletions(-) delete mode 100644 changelog.d/18000.bugfix delete mode 100644 changelog.d/18073.bugfix delete mode 100644 changelog.d/18075.bugfix delete mode 100644 changelog.d/18089.bugfix delete mode 100644 changelog.d/18107.bugfix delete mode 100644 changelog.d/18109.misc delete mode 100644 changelog.d/18112.bugfix delete mode 100644 changelog.d/18119.bugfix delete mode 100644 changelog.d/18130.bugfix delete mode 100644 changelog.d/18131.bugfix diff --git a/CHANGES.md b/CHANGES.md index cc6426751d..051ab130b2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,30 @@ +# Synapse 1.124.0rc1 (2025-02-04) + +### Bugfixes + +- Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. ([\#18000](https://github.com/element-hq/synapse/issues/18000)) +- Deactivated users will no longer automatically accept an invite when `auto_accept_invites` is enabled. ([\#18073](https://github.com/element-hq/synapse/issues/18073)) +- Fix join being denied after being invited over federation. Also fixes other out-of-band membership transitions. ([\#18075](https://github.com/element-hq/synapse/issues/18075)) +- Updates contributed `docker-compose.yml` file to PostgreSQL v15, as v12 is no longer supported by Synapse. + Contributed by @maxkratz. ([\#18089](https://github.com/element-hq/synapse/issues/18089)) +- Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. ([\#18107](https://github.com/element-hq/synapse/issues/18107), [\#18130](https://github.com/element-hq/synapse/issues/18130), [\#18131](https://github.com/element-hq/synapse/issues/18131)) +- Raise an error if someone is using an incorrect suffix in a config duration string. ([\#18112](https://github.com/element-hq/synapse/issues/18112)) +- Fix a bug where the [Delete Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#version-2-new-version) would fail if the `block` parameter was set to `true` and a worker other than the main process was configured to handle background tasks. ([\#18119](https://github.com/element-hq/synapse/issues/18119)) + +### Internal Changes + +- Increase the length of the generated `nonce` parameter when perfoming OIDC logins to comply with the TI-Messenger spec. ([\#18109](https://github.com/element-hq/synapse/issues/18109)) + + + +### Updates to locked dependencies + +* Bump dawidd6/action-download-artifact from 7 to 8. ([\#18108](https://github.com/element-hq/synapse/issues/18108)) +* Bump log from 0.4.22 to 0.4.25. ([\#18098](https://github.com/element-hq/synapse/issues/18098)) +* Bump python-multipart from 0.0.18 to 0.0.20. ([\#18096](https://github.com/element-hq/synapse/issues/18096)) +* Bump serde_json from 1.0.135 to 1.0.137. ([\#18099](https://github.com/element-hq/synapse/issues/18099)) +* Bump types-bleach from 6.1.0.20240331 to 6.2.0.20241123. ([\#18082](https://github.com/element-hq/synapse/issues/18082)) + # Synapse 1.123.0 (2025-01-28) No significant changes since 1.123.0rc1. diff --git a/changelog.d/18000.bugfix b/changelog.d/18000.bugfix deleted file mode 100644 index a8f1545bf5..0000000000 --- a/changelog.d/18000.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. diff --git a/changelog.d/18073.bugfix b/changelog.d/18073.bugfix deleted file mode 100644 index eeb56a7a61..0000000000 --- a/changelog.d/18073.bugfix +++ /dev/null @@ -1 +0,0 @@ -Deactivated users will no longer automatically accept an invite when `auto_accept_invites` is enabled. \ No newline at end of file diff --git a/changelog.d/18075.bugfix b/changelog.d/18075.bugfix deleted file mode 100644 index 95b486bed1..0000000000 --- a/changelog.d/18075.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix join being denied after being invited over federation. Also fixes other out-of-band membership transitions. diff --git a/changelog.d/18089.bugfix b/changelog.d/18089.bugfix deleted file mode 100644 index 607fab7112..0000000000 --- a/changelog.d/18089.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Updates contributed `docker-compose.yml` file to PostgreSQL v15, as v12 is no longer supported by Synapse. -Contributed by @maxkratz. \ No newline at end of file diff --git a/changelog.d/18107.bugfix b/changelog.d/18107.bugfix deleted file mode 100644 index 4d0c19fab9..0000000000 --- a/changelog.d/18107.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/changelog.d/18109.misc b/changelog.d/18109.misc deleted file mode 100644 index c310e76f78..0000000000 --- a/changelog.d/18109.misc +++ /dev/null @@ -1 +0,0 @@ -Increase the length of the generated `nonce` parameter when perfoming OIDC logins to comply with the TI-Messenger spec. \ No newline at end of file diff --git a/changelog.d/18112.bugfix b/changelog.d/18112.bugfix deleted file mode 100644 index 61c94280d8..0000000000 --- a/changelog.d/18112.bugfix +++ /dev/null @@ -1 +0,0 @@ -Raise an error if someone is using an incorrect suffix in a config duration string. diff --git a/changelog.d/18119.bugfix b/changelog.d/18119.bugfix deleted file mode 100644 index c8ac53f9d4..0000000000 --- a/changelog.d/18119.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where the [Delete Room Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#version-2-new-version) would fail if the `block` parameter was set to `true` and a worker other than the main process was configured to handle background tasks. \ No newline at end of file diff --git a/changelog.d/18130.bugfix b/changelog.d/18130.bugfix deleted file mode 100644 index 4d0c19fab9..0000000000 --- a/changelog.d/18130.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/changelog.d/18131.bugfix b/changelog.d/18131.bugfix deleted file mode 100644 index 4d0c19fab9..0000000000 --- a/changelog.d/18131.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rare edge case where state groups could be deleted while we are persisting new events that reference them. diff --git a/debian/changelog b/debian/changelog index a470dff676..0b6d50daaa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.124.0~rc1) stable; urgency=medium + + * New Synapse release 1.124.0rc1. + + -- Synapse Packaging team Tue, 04 Feb 2025 11:53:05 +0000 + matrix-synapse-py3 (1.123.0) stable; urgency=medium * New Synapse release 1.123.0. diff --git a/pyproject.toml b/pyproject.toml index 1cd874716e..8febdc4c23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.123.0" +version = "1.124.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From e41174cae3a74a2e6f9cb1386eb6e0b5510d6fce Mon Sep 17 00:00:00 2001 From: V02460 Date: Tue, 4 Feb 2025 19:45:33 +0100 Subject: [PATCH 118/691] Add MSC3861 config options admin_token_path and client_secret_path (#18004) Another PR on my quest to a `*_path` variant for every secret. Adds two config options `admin_token_path` and `client_secret_path` to the experimental config under `experimental_features.msc3861`. Also includes tests. I tried to be a good citizen here by following `attrs` conventions and not rewriting the corresponding non-path variants in the class, but instead adding methods to retrieve the value. Reading secrets from files has the security advantage of separating the secrets from the config. It also simplifies secrets management in Kubernetes. Also useful to NixOS users. --- changelog.d/18004.feature | 1 + synapse/api/auth/msc3861_delegated.py | 14 ++-- synapse/config/experimental.py | 103 ++++++++++++++++++++++-- tests/config/test_load.py | 10 +++ tests/handlers/test_oauth_delegation.py | 2 +- 5 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 changelog.d/18004.feature diff --git a/changelog.d/18004.feature b/changelog.d/18004.feature new file mode 100644 index 0000000000..8cacd1a0ef --- /dev/null +++ b/changelog.d/18004.feature @@ -0,0 +1 @@ +Add experimental config options `admin_token_path` and `client_secret_path` for MSC 3861. \ No newline at end of file diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 802ea51d18..f825b5c95e 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -19,7 +19,7 @@ # # import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from urllib.parse import urlencode from authlib.oauth2 import ClientAuth @@ -119,7 +119,7 @@ class MSC3861DelegatedAuth(BaseAuth): self._clock = hs.get_clock() self._http_client = hs.get_proxied_http_client() self._hostname = hs.hostname - self._admin_token = self._config.admin_token + self._admin_token: Callable[[], Optional[str]] = self._config.admin_token self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( self._load_metadata @@ -133,9 +133,10 @@ class MSC3861DelegatedAuth(BaseAuth): ) else: # Else use the client secret - assert self._config.client_secret, "No client_secret provided" + client_secret = self._config.client_secret() + assert client_secret, "No client_secret provided" self._client_auth = ClientAuth( - self._config.client_id, self._config.client_secret, auth_method + self._config.client_id, client_secret, auth_method ) async def _load_metadata(self) -> OpenIDProviderMetadata: @@ -283,7 +284,7 @@ class MSC3861DelegatedAuth(BaseAuth): requester = await self.get_user_by_access_token(access_token, allow_expired) # Do not record requests from MAS using the virtual `__oidc_admin` user. - if access_token != self._admin_token: + if access_token != self._admin_token(): await self._record_request(request, requester) if not allow_guest and requester.is_guest: @@ -324,7 +325,8 @@ class MSC3861DelegatedAuth(BaseAuth): token: str, allow_expired: bool = False, ) -> Requester: - if self._admin_token is not None and token == self._admin_token: + admin_token = self._admin_token() + if admin_token is not None and token == admin_token: # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 94a25c7ee8..3beaeb8869 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -20,14 +20,15 @@ # import enum -from typing import TYPE_CHECKING, Any, Optional +from functools import cache +from typing import TYPE_CHECKING, Any, Iterable, Optional import attr import attr.validators from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config import ConfigError -from synapse.config._base import Config, RootConfig +from synapse.config._base import Config, RootConfig, read_file from synapse.types import JsonDict # Determine whether authlib is installed. @@ -43,6 +44,12 @@ if TYPE_CHECKING: from authlib.jose.rfc7517 import JsonWebKey +@cache +def read_secret_from_file_once(file_path: Any, config_path: Iterable[str]) -> str: + """Returns the memoized secret read from file.""" + return read_file(file_path, config_path).strip() + + class ClientAuthMethod(enum.Enum): """List of supported client auth methods.""" @@ -63,6 +70,40 @@ def _parse_jwks(jwks: Optional[JsonDict]) -> Optional["JsonWebKey"]: return JsonWebKey.import_key(jwks) +def _check_client_secret( + instance: "MSC3861", _attribute: attr.Attribute, _value: Optional[str] +) -> None: + if instance._client_secret and instance._client_secret_path: + raise ConfigError( + ( + "You have configured both " + "`experimental_features.msc3861.client_secret` and " + "`experimental_features.msc3861.client_secret_path`. " + "These are mutually incompatible." + ), + ("experimental", "msc3861", "client_secret"), + ) + # Check client secret can be retrieved + instance.client_secret() + + +def _check_admin_token( + instance: "MSC3861", _attribute: attr.Attribute, _value: Optional[str] +) -> None: + if instance._admin_token and instance._admin_token_path: + raise ConfigError( + ( + "You have configured both " + "`experimental_features.msc3861.admin_token` and " + "`experimental_features.msc3861.admin_token_path`. " + "These are mutually incompatible." + ), + ("experimental", "msc3861", "admin_token"), + ) + # Check client secret can be retrieved + instance.admin_token() + + @attr.s(slots=True, frozen=True) class MSC3861: """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC""" @@ -97,15 +138,30 @@ class MSC3861: ) """The auth method used when calling the introspection endpoint.""" - client_secret: Optional[str] = attr.ib( + _client_secret: Optional[str] = attr.ib( default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_client_secret, + ], ) """ The client secret to use when calling the introspection endpoint, when using any of the client_secret_* client auth methods. """ + _client_secret_path: Optional[str] = attr.ib( + default=None, + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_client_secret, + ], + ) + """ + Alternative to `client_secret`: allows the secret to be specified in an + external file. + """ + jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks) """ The JWKS to use when calling the introspection endpoint, @@ -133,7 +189,7 @@ class MSC3861: ClientAuthMethod.CLIENT_SECRET_BASIC, ClientAuthMethod.CLIENT_SECRET_JWT, ) - and self.client_secret is None + and self.client_secret() is None ): raise ConfigError( f"A client secret must be provided when using the {value} client auth method", @@ -152,15 +208,48 @@ class MSC3861: ) """The URL of the My Account page on the OIDC Provider as per MSC2965.""" - admin_token: Optional[str] = attr.ib( + _admin_token: Optional[str] = attr.ib( default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_admin_token, + ], ) """ A token that should be considered as an admin token. This is used by the OIDC provider, to make admin calls to Synapse. """ + _admin_token_path: Optional[str] = attr.ib( + default=None, + validator=[ + attr.validators.optional(attr.validators.instance_of(str)), + _check_admin_token, + ], + ) + """ + Alternative to `admin_token`: allows the secret to be specified in an + external file. + """ + + def client_secret(self) -> Optional[str]: + """Returns the secret given via `client_secret` or `client_secret_path`.""" + if self._client_secret_path: + return read_secret_from_file_once( + self._client_secret_path, + ("experimental_features", "msc3861", "client_secret_path"), + ) + return self._client_secret + + def admin_token(self) -> Optional[str]: + """Returns the admin token given via `admin_token` or `admin_token_path`.""" + if self._admin_token_path: + return read_secret_from_file_once( + self._admin_token_path, + ("experimental_features", "msc3861", "admin_token_path"), + ) + return self._admin_token + def check_config_conflicts(self, root: RootConfig) -> None: """Checks for any configuration conflicts with other parts of Synapse. diff --git a/tests/config/test_load.py b/tests/config/test_load.py index f8f7b72e40..220ca23aa7 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -132,6 +132,8 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "turn_shared_secret_path: /does/not/exist", "registration_shared_secret_path: /does/not/exist", "macaroon_secret_key_path: /does/not/exist", + "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", + "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] * (hiredis is not None), ] @@ -157,6 +159,14 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "macaroon_secret_key_path: {}", lambda c: c.key.macaroon_secret_key, ), + ( + "experimental_features:\n msc3861:\n client_secret_path: {}", + lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), + ), + ( + "experimental_features:\n msc3861:\n admin_token_path: {}", + lambda c: c.experimental.msc3861.admin_token().encode("utf-8"), + ), *[ ( "redis:\n enabled: true\n password_path: {}", diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 5f73469daa..ba2f8ff510 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -795,7 +795,7 @@ class MSC3861OAuthDelegation(HomeserverTestCase): req = SynapseRequest(channel, self.site) # type: ignore[arg-type] req.client.host = MAS_IPV4_ADDR req.requestHeaders.addRawHeader( - "Authorization", f"Bearer {self.auth._admin_token}" + "Authorization", f"Bearer {self.auth._admin_token()}" ) req.requestHeaders.addRawHeader("User-Agent", MAS_USER_AGENT) req.content = BytesIO(b"") From b8a333004a9d7591ecf2c477c50e06f122f40565 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Feb 2025 13:07:49 +0100 Subject: [PATCH 119/691] Fix legacy modules `check_username_for_spam` (#18135) Broke in #17916, as the signature inspection incorrectly looks at the wrapper function. We fix this by setting the signature on the wrapper function to that of the wrapped function via `@functools.wraps`. --- changelog.d/18135.bugfix | 1 + synapse/module_api/callbacks/spamchecker_callbacks.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/18135.bugfix diff --git a/changelog.d/18135.bugfix b/changelog.d/18135.bugfix new file mode 100644 index 0000000000..998914a811 --- /dev/null +++ b/changelog.d/18135.bugfix @@ -0,0 +1 @@ +Fix user directory search when using a legacy module with a `check_username_for_spam` callback. Broke in v1.122.0. diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index a2f328cafe..9bc572422b 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -19,6 +19,7 @@ # # +import functools import inspect import logging from typing import ( @@ -297,6 +298,7 @@ def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None: "Bad signature for callback check_registration_for_spam", ) + @functools.wraps(wrapped_func) def run(*args: Any, **kwargs: Any) -> Awaitable: # Assertion required because mypy can't prove we won't change `f` # back to `None`. See From 5b03265cfb30219eb09cba42125683381ea6a49d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Feb 2025 13:30:13 +0100 Subject: [PATCH 120/691] Fix 'Fix lint' GHA (#18136) c.f. #18121 --------- Co-authored-by: Quentin Gliech --- .github/workflows/fix_lint.yaml | 2 +- changelog.d/18136.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18136.misc diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 909b0a847f..d6aed83774 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -21,7 +21,7 @@ jobs: # We use nightly so that `fmt` correctly groups together imports, and # clippy correctly fixes up the benchmarks. toolchain: nightly-2022-12-01 - components: rustfmt + components: clippy, rustfmt - uses: Swatinem/rust-cache@v2 - name: Setup Poetry diff --git a/changelog.d/18136.misc b/changelog.d/18136.misc new file mode 100644 index 0000000000..ed88c38acf --- /dev/null +++ b/changelog.d/18136.misc @@ -0,0 +1 @@ +Fix the 'Fix linting' GitHub Actions workflow. From 6fe41d2b479c25cd8cf34c12a07235a6098a11e4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 5 Feb 2025 13:40:10 +0000 Subject: [PATCH 121/691] make dual licensing explicit (#18134) Update readme & LICENSE files to make it explicit that you can buy a commercial license as an AGPL alternative from Element. --- LICENSE => LICENSE-AGPL-3.0 | 0 LICENSE-COMMERCIAL | 6 ++++++ README.rst | 27 +++++++++++++++++++++------ changelog.d/18134.misc | 1 + 4 files changed, 28 insertions(+), 6 deletions(-) rename LICENSE => LICENSE-AGPL-3.0 (100%) create mode 100644 LICENSE-COMMERCIAL create mode 100644 changelog.d/18134.misc diff --git a/LICENSE b/LICENSE-AGPL-3.0 similarity index 100% rename from LICENSE rename to LICENSE-AGPL-3.0 diff --git a/LICENSE-COMMERCIAL b/LICENSE-COMMERCIAL new file mode 100644 index 0000000000..173e03e0c0 --- /dev/null +++ b/LICENSE-COMMERCIAL @@ -0,0 +1,6 @@ +Licensees holding a valid commercial license with Element may use this +software in accordance with the terms contained in a written agreement +between you and Element. + +To purchase a commercial license please contact our sales team at +licensing@element.io diff --git a/README.rst b/README.rst index 2fe4a7e43f..77f861e788 100644 --- a/README.rst +++ b/README.rst @@ -10,14 +10,15 @@ implementation, written and maintained by `Element `_. `Matrix `__ is the open standard for secure and interoperable real time communications. You can directly run and manage the source code in this repository, available under an AGPL -license. There is no support provided from Element unless you have a -subscription. +license (or alternatively under a commercial license from Element). +There is no support provided by Element unless you have a +subscription from Element. -Subscription alternative -======================== +Subscription +============ -Alternatively, for those that need an enterprise-ready solution, Element -Server Suite (ESS) is `available as a subscription `_. +For those that need an enterprise-ready solution, Element +Server Suite (ESS) is `available via subscription `_. ESS builds on Synapse to offer a complete Matrix-based backend including the full `Admin Console product `_, giving admins the power to easily manage an organization-wide @@ -249,6 +250,20 @@ Developers might be particularly interested in: Alongside all that, join our developer community on Matrix: `#synapse-dev:matrix.org `_, featuring real humans! +Copyright and Licensing +======================= + +Copyright 2014-2017 OpenMarket Ltd +Copyright 2017 Vector Creations Ltd +Copyright 2017-2025 New Vector Ltd + +This software is dual-licensed by New Vector Ltd (Element). It can be used either: + +(1) for free under the terms of the GNU Affero General Public License (as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version); OR + +(2) under the terms of a paid-for Element Commercial License agreement between you and Element (the terms of which may vary depending on what you and Element have agreed to). +Unless required by applicable law or agreed to in writing, software distributed under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the specific language governing permissions and limitations under the Licenses. + .. |support| image:: https://img.shields.io/badge/matrix-community%20support-success :alt: (get community support in #synapse:matrix.org) diff --git a/changelog.d/18134.misc b/changelog.d/18134.misc new file mode 100644 index 0000000000..be90691516 --- /dev/null +++ b/changelog.d/18134.misc @@ -0,0 +1 @@ +Make it explicit that you can buy an AGPL-alternative commercial license from Element. From 3391da348fc759786952637abd6a58360b081bb6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Feb 2025 17:26:07 +0100 Subject: [PATCH 122/691] Fix bug where persisting some events fails after unclean shutdown. (#18137) Introduced in #18107 `UniqueViolation: duplicate key value violates unique constraint "state_groups_persisting_pkey"` --- changelog.d/18137.bugfix | 1 + synapse/storage/databases/state/deletion.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18137.bugfix diff --git a/changelog.d/18137.bugfix b/changelog.d/18137.bugfix new file mode 100644 index 0000000000..d7b9d0426f --- /dev/null +++ b/changelog.d/18137.bugfix @@ -0,0 +1 @@ +Fix regression where persisting events in some rooms could fail after a previous unclean shutdown. Introduced in v1.124.0rc1. diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index 4853e5aa2f..d0949261f2 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -95,8 +95,18 @@ class StateDeletionDataStore: self.db_pool = database self._instance_name = hs.get_instance_name() - # TODO: Clear from `state_groups_persisting` any holdovers from previous - # running instance. + with db_conn.cursor(txn_name="_clear_existing_persising") as txn: + self._clear_existing_persising(txn) + + def _clear_existing_persising(self, txn: LoggingTransaction) -> None: + """On startup we clear any entries in `state_groups_persisting` that + match our instance name, in case of a previous unclean shutdown""" + + self.db_pool.simple_delete_txn( + txn, + table="state_groups_persisting", + keyvalues={"instance_name": self._instance_name}, + ) async def check_state_groups_and_bump_deletion( self, state_groups: AbstractSet[int] From 553e9882bf8e00a59ce7cb505c7eca730f769305 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Feb 2025 16:35:55 +0000 Subject: [PATCH 123/691] 1.124.0rc2 --- CHANGES.md | 9 +++++++++ changelog.d/18137.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18137.bugfix diff --git a/CHANGES.md b/CHANGES.md index 051ab130b2..3c92bedda3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.124.0rc2 (2025-02-05) + +### Bugfixes + +- Fix regression where persisting events in some rooms could fail after a previous unclean shutdown. Introduced in v1.124.0rc1. ([\#18137](https://github.com/element-hq/synapse/issues/18137)) + + + + # Synapse 1.124.0rc1 (2025-02-04) ### Bugfixes diff --git a/changelog.d/18137.bugfix b/changelog.d/18137.bugfix deleted file mode 100644 index d7b9d0426f..0000000000 --- a/changelog.d/18137.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression where persisting events in some rooms could fail after a previous unclean shutdown. Introduced in v1.124.0rc1. diff --git a/debian/changelog b/debian/changelog index 0b6d50daaa..8c84eecb20 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.124.0~rc2) stable; urgency=medium + + * New Synapse release 1.124.0rc2. + + -- Synapse Packaging team Wed, 05 Feb 2025 16:35:53 +0000 + matrix-synapse-py3 (1.124.0~rc1) stable; urgency=medium * New Synapse release 1.124.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 8febdc4c23..3f577a1973 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.124.0rc1" +version = "1.124.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From dcf7b392762067e5ee264df629b2ea65b3473609 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Feb 2025 11:18:32 +0100 Subject: [PATCH 124/691] Fix performance of `check_state_groups_and_bump_deletion` (#18141) Regressed as part of https://github.com/element-hq/synapse/pull/18107 This does two things: 1. Only check if the state groups have been deleted when calculating the event context (as that's when we will insert them). This avoids lots of checks for read operations. 2. Don't lock the `state_groups` rows when doing the check. This adds overhead, and it doesn't prevent any races. --- changelog.d/18141.bugfix | 1 + synapse/state/__init__.py | 32 ++++++++++++++------- synapse/storage/databases/state/deletion.py | 26 +++++++++++++---- tests/rest/client/test_rooms.py | 4 +-- 4 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 changelog.d/18141.bugfix diff --git a/changelog.d/18141.bugfix b/changelog.d/18141.bugfix new file mode 100644 index 0000000000..7826a519fa --- /dev/null +++ b/changelog.d/18141.bugfix @@ -0,0 +1 @@ +Fix regression in performance of sending events due to superfluous reads and locks. Introduced in v1.124.0rc1. diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 5b746f2037..9e48e09270 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -359,6 +359,28 @@ class StateHandler: await_full_state=False, ) + # Ensure we still have the state groups we're relying on, and bump + # their usage time to avoid them being deleted from under us. + if entry.state_group: + missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( + {entry.state_group} + ) + if missing_state_group: + raise Exception(f"Missing state group: {entry.state_group}") + elif entry.prev_group: + # We only rely on the prev group when persisting the event if we + # don't have an `entry.state_group`. + missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( + {entry.prev_group} + ) + + if missing_state_group: + # If we're missing the prev group then we can just clear the + # entries, and rely on `entry._state` (which must exist if + # `entry.state_group` is None) + entry.prev_group = None + entry.delta_ids = None + state_group_before_event_prev_group = entry.prev_group deltas_to_state_group_before_event = entry.delta_ids state_ids_before_event = None @@ -519,16 +541,6 @@ class StateHandler: state_group_id ) - if prev_group: - # Ensure that we still have the prev group, and ensure we don't - # delete it while we're persisting the event. - missing_state_group = await self._state_deletion_store.check_state_groups_and_bump_deletion( - {prev_group} - ) - if missing_state_group: - prev_group = None - delta_ids = None - return _StateCacheEntry( state=None, state_group=state_group_id, diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index d0949261f2..d4b1c20a45 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -123,12 +123,28 @@ class StateDeletionDataStore: "check_state_groups_and_bump_deletion", self._check_state_groups_and_bump_deletion_txn, state_groups, + # We don't need to lock if we're just doing a quick check, as the + # lock doesn't prevent any races here. + lock=False, ) def _check_state_groups_and_bump_deletion_txn( - self, txn: LoggingTransaction, state_groups: AbstractSet[int] + self, txn: LoggingTransaction, state_groups: AbstractSet[int], lock: bool = True ) -> Collection[int]: - existing_state_groups = self._get_existing_groups_with_lock(txn, state_groups) + """Checks to make sure that the state groups haven't been deleted, and + if they're pending deletion we delay it (allowing time for any event + that will use them to finish persisting). + + The `lock` flag sets if we should lock the `state_group` rows we're + checking, which we should do when storing new groups. + + Returns: + The state groups that are missing, if any. + """ + + existing_state_groups = self._get_existing_groups_with_lock( + txn, state_groups, lock=lock + ) self._bump_deletion_txn(txn, existing_state_groups) @@ -188,18 +204,18 @@ class StateDeletionDataStore: ) def _get_existing_groups_with_lock( - self, txn: LoggingTransaction, state_groups: Collection[int] + self, txn: LoggingTransaction, state_groups: Collection[int], lock: bool = True ) -> AbstractSet[int]: """Return which of the given state groups are in the database, and locks those rows with `KEY SHARE` to ensure they don't get concurrently - deleted.""" + deleted (if `lock` is true).""" clause, args = make_in_list_sql_clause(self.db_pool.engine, "id", state_groups) sql = f""" SELECT id FROM state_groups WHERE {clause} """ - if isinstance(self.db_pool.engine, PostgresEngine): + if lock and isinstance(self.db_pool.engine, PostgresEngine): # On postgres we add a row level lock to the rows to ensure that we # conflict with any concurrent DELETEs. `FOR KEY SHARE` lock will # not conflict with other read diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 3bb539bf87..5473a8e769 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -742,7 +742,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(36, channel.resource_usage.db_txn_count) + self.assertEqual(35, channel.resource_usage.db_txn_count) def test_post_room_initial_state(self) -> None: # POST with initial_state config key, expect new room id @@ -755,7 +755,7 @@ class RoomsCreateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) assert channel.resource_usage is not None - self.assertEqual(38, channel.resource_usage.db_txn_count) + self.assertEqual(37, channel.resource_usage.db_txn_count) def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id From 0076197c97114b38906a972dcf1eb291fbd16f0f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Feb 2025 13:42:57 +0000 Subject: [PATCH 125/691] 1.124.0rc3 --- CHANGES.md | 9 +++++++++ changelog.d/18141.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18141.bugfix diff --git a/CHANGES.md b/CHANGES.md index 3c92bedda3..d310241766 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.124.0rc3 (2025-02-07) + +### Bugfixes + +- Fix regression in performance of sending events due to superfluous reads and locks. Introduced in v1.124.0rc1. ([\#18141](https://github.com/element-hq/synapse/issues/18141)) + + + + # Synapse 1.124.0rc2 (2025-02-05) ### Bugfixes diff --git a/changelog.d/18141.bugfix b/changelog.d/18141.bugfix deleted file mode 100644 index 7826a519fa..0000000000 --- a/changelog.d/18141.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression in performance of sending events due to superfluous reads and locks. Introduced in v1.124.0rc1. diff --git a/debian/changelog b/debian/changelog index 8c84eecb20..886b871b72 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.124.0~rc3) stable; urgency=medium + + * New Synapse release 1.124.0rc3. + + -- Synapse Packaging team Fri, 07 Feb 2025 13:42:55 +0000 + matrix-synapse-py3 (1.124.0~rc2) stable; urgency=medium * New Synapse release 1.124.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 3f577a1973..6b9f4e0dda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.124.0rc2" +version = "1.124.0rc3" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From deb09b38362be0c567da166bd697ef8abff422b2 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:17:26 +0200 Subject: [PATCH 126/691] Add Oracle Linux installation instructions (#17436) ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) I forgot the guide applies to Oracle Linux as well. In fact, I ran a small homeserver on OEL a few months back. I did minimal installations on Rocky and OEL on VirtualBox and noticed Codeready/Powertools repos aren't required, so I removed those commands from the guide. I switched `RHEL`-references to `EL`. #17423 was merged before I remembered about OEL but a new PR shouldn't hurt :) --------- Co-authored-by: Quentin Gliech --- changelog.d/17436.doc | 1 + docs/setup/installation.md | 17 +++-------------- 2 files changed, 4 insertions(+), 14 deletions(-) create mode 100644 changelog.d/17436.doc diff --git a/changelog.d/17436.doc b/changelog.d/17436.doc new file mode 100644 index 0000000000..7846d3a8bf --- /dev/null +++ b/changelog.d/17436.doc @@ -0,0 +1 @@ +Add Oracle Linux 8 and 9 installation instructions. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index d0081a88d9..0853496ab7 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -310,29 +310,18 @@ sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ sudo dnf group install "Development Tools" ``` -##### Red Hat Enterprise Linux / Rocky Linux +##### Red Hat Enterprise Linux / Rocky Linux / Oracle Linux -*Note: The term "RHEL" below refers to both Red Hat Enterprise Linux and Rocky Linux. The distributions are 1:1 binary compatible.* +*Note: The term "RHEL" below refers to Red Hat Enterprise Linux, Oracle Linux and Rocky Linux. The distributions are 1:1 binary compatible.* It's recommended to use the latest Python versions. -RHEL 8 in particular ships with Python 3.6 by default which is EOL and therefore no longer supported by Synapse. RHEL 9 ship with Python 3.9 which is still supported by the Python core team as of this writing. However, newer Python versions provide significant performance improvements and they're available in official distributions' repositories. Therefore it's recommended to use them. +RHEL 8 in particular ships with Python 3.6 by default which is EOL and therefore no longer supported by Synapse. RHEL 9 ships with Python 3.9 which is still supported by the Python core team as of this writing. However, newer Python versions provide significant performance improvements and they're available in official distributions' repositories. Therefore it's recommended to use them. Python 3.11 and 3.12 are available for both RHEL 8 and 9. These commands should be run as root user. -RHEL 8 -```bash -# Enable PowerTools repository -dnf config-manager --set-enabled powertools -``` -RHEL 9 -```bash -# Enable CodeReady Linux Builder repository -crb enable -``` - Install new version of Python. You only need one of these: ```bash # Python 3.11 From 4c84c9c4ad0d3a1e30b5384604e49f8569c0e015 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Feb 2025 16:27:46 +0100 Subject: [PATCH 127/691] Don't log exceptions for obviously incorrect stream tokens (#18139) We log incorrect ones as we want to catch bugs where Synapse returns bad tokens. However, sometimes clients just send tokens that are e.g. empty. --------- Co-authored-by: Eric Eastwood --- changelog.d/18139.misc | 1 + synapse/types/__init__.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/18139.misc diff --git a/changelog.d/18139.misc b/changelog.d/18139.misc new file mode 100644 index 0000000000..f753033b8e --- /dev/null +++ b/changelog.d/18139.misc @@ -0,0 +1 @@ +Do not log at the exception-level when clients provide empty `since` token to `/sync` API. diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 26783c5622..3d15c04faa 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -664,6 +664,11 @@ class RoomStreamToken(AbstractMultiWriterStreamToken): @classmethod async def parse(cls, store: "PurgeEventsStore", string: str) -> "RoomStreamToken": + # Check that it looks like a Synapse token first. We do this so that + # we don't log at the exception-level for obviously incorrect tokens. + if not string or string[0] not in ("s", "t", "m"): + raise SynapseError(400, f"Invalid room stream token {string:!r}") + try: if string[0] == "s": return cls(topological=None, stream=int(string[1:])) From 8f07ef5c93baed5b1259ed9ce3ed1087b7a2d168 Mon Sep 17 00:00:00 2001 From: meise Date: Mon, 10 Feb 2025 16:36:21 +0100 Subject: [PATCH 128/691] feat: Allow multiple values for SSO attribute_requirements via comma separation (#17949) In the current `attribute_requirements` implementation it is only possible to allow exact matching attribute values. Multiple allowed values for one attribute are not possible as described in #13238. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Sebastian Neuser Co-authored-by: Quentin Gliech --- changelog.d/17949.feature | 1 + .../configuration/config_documentation.md | 9 ++-- synapse/config/sso.py | 20 ++++++-- synapse/handlers/sso.py | 6 ++- tests/handlers/test_oidc.py | 32 +++++++++++++ tests/handlers/test_saml.py | 46 +++++++++++++++++++ 6 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 changelog.d/17949.feature diff --git a/changelog.d/17949.feature b/changelog.d/17949.feature new file mode 100644 index 0000000000..ee9bb51e03 --- /dev/null +++ b/changelog.d/17949.feature @@ -0,0 +1 @@ +Add functionality to be able to use multiple values in SSO feature `attribute_requirements`. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index a1e671ab8e..a5f23149ec 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3337,8 +3337,9 @@ This setting has the following sub-options: The default is 'uid'. * `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes match particular values. The requirements can be listed under - `attribute_requirements` as shown in the example. All of the listed attributes must - match for the login to be permitted. + `attribute_requirements` as shown in the example. All of the listed attributes must + match for the login to be permitted. Values can be specified in a `one_of` list to allow + multiple values for an attribute. * `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` option must be set to the entity to redirect users to. Most deployments only have a single IdP entity and so should omit this option. @@ -3419,7 +3420,9 @@ saml2_config: - attribute: userGroup value: "staff" - attribute: department - value: "sales" + one_of: + - "sales" + - "admins" idp_entityid: 'https://our_idp/entityid' ``` diff --git a/synapse/config/sso.py b/synapse/config/sso.py index d7a2187e7d..97b85e47ea 100644 --- a/synapse/config/sso.py +++ b/synapse/config/sso.py @@ -19,7 +19,7 @@ # # import logging -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import attr @@ -43,13 +43,23 @@ class SsoAttributeRequirement: """Object describing a single requirement for SSO attributes.""" attribute: str - # If a value is not given, than the attribute must simply exist. - value: Optional[str] + # If neither value nor one_of is given, the attribute must simply exist. This is + # only true for CAS configs which use a different JSON schema than the one below. + value: Optional[str] = None + one_of: Optional[List[str]] = None JSON_SCHEMA = { "type": "object", - "properties": {"attribute": {"type": "string"}, "value": {"type": "string"}}, - "required": ["attribute", "value"], + "properties": { + "attribute": {"type": "string"}, + "value": {"type": "string"}, + "one_of": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["attribute"], + "oneOf": [ + {"required": ["value"]}, + {"required": ["one_of"]}, + ], } diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index cee2eefbb3..531ed57110 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -1277,12 +1277,16 @@ def _check_attribute_requirement( return False # If the requirement is None, the attribute existing is enough. - if req.value is None: + if req.value is None and req.one_of is None: return True values = attributes[req.attribute] if req.value in values: return True + if req.one_of: + for value in req.one_of: + if value in values: + return True logger.info( "SSO attribute %s did not match required value '%s' (was '%s')", diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index a81501979d..1b43ee43c6 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -1267,6 +1267,38 @@ class OidcHandlerTestCase(HomeserverTestCase): auth_provider_session_id=None, ) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [ + {"attribute": "test", "one_of": ["foo", "bar"]} + ], + } + } + ) + def test_attribute_requirements_one_of(self) -> None: + """Test that auth succeeds if userinfo attribute has multiple values and CONTAINS required value""" + # userinfo with "test": ["bar"] attribute should succeed. + userinfo = { + "sub": "tester", + "username": "tester", + "test": ["bar"], + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # check that the auth handler got called as expected + self.complete_sso_login.assert_called_once_with( + "@tester:test", + self.provider.idp_id, + request, + ANY, + None, + new_user=True, + auth_provider_session_id=None, + ) + @override_config( { "oidc_config": { diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py index 6ab8fda6e7..1aca354826 100644 --- a/tests/handlers/test_saml.py +++ b/tests/handlers/test_saml.py @@ -363,6 +363,52 @@ class SamlHandlerTestCase(HomeserverTestCase): auth_provider_session_id=None, ) + @override_config( + { + "saml2_config": { + "attribute_requirements": [ + {"attribute": "userGroup", "one_of": ["staff", "admin"]}, + ], + }, + } + ) + def test_attribute_requirements_one_of(self) -> None: + """The required attributes can be comma-separated.""" + + # stub out the auth handler + auth_handler = self.hs.get_auth_handler() + auth_handler.complete_sso_login = AsyncMock() # type: ignore[method-assign] + + # The response doesn't have the proper department. + saml_response = FakeAuthnResponse( + {"uid": "test_user", "username": "test_user", "userGroup": ["nogroup"]} + ) + request = _mock_request() + self.get_success( + self.handler._handle_authn_response(request, saml_response, "redirect_uri") + ) + auth_handler.complete_sso_login.assert_not_called() + + # Add the proper attributes and it should succeed. + saml_response = FakeAuthnResponse( + {"uid": "test_user", "username": "test_user", "userGroup": ["admin"]} + ) + request.reset_mock() + self.get_success( + self.handler._handle_authn_response(request, saml_response, "redirect_uri") + ) + + # check that the auth handler got called as expected + auth_handler.complete_sso_login.assert_called_once_with( + "@test_user:test", + "saml", + request, + "redirect_uri", + None, + new_user=True, + auth_provider_session_id=None, + ) + def _mock_request() -> Mock: """Returns a mock which will stand in as a SynapseRequest""" From e4074749d296ca703c857dedf00c6d0543a0679b Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Mon, 10 Feb 2025 10:37:05 -0500 Subject: [PATCH 129/691] Overload "allow_none" on DB pool static method (#17616) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/17616.misc | 1 + synapse/storage/database.py | 24 +++++++++++++++---- .../storage/databases/main/e2e_room_keys.py | 23 ++++++++---------- .../storage/databases/main/registration.py | 15 ++++++------ synapse/storage/databases/main/stream.py | 15 ++++++------ 5 files changed, 45 insertions(+), 33 deletions(-) create mode 100644 changelog.d/17616.misc diff --git a/changelog.d/17616.misc b/changelog.d/17616.misc new file mode 100644 index 0000000000..8250832dcd --- /dev/null +++ b/changelog.d/17616.misc @@ -0,0 +1 @@ +Overload DatabasePool.simple_select_one_txn to return non-None when the allow_none parameter is False. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index cb4a5857be..8272e39340 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -2159,10 +2159,26 @@ class DatabasePool: if rowcount > 1: raise StoreError(500, "More than one row matched (%s)" % (table,)) - # Ideally we could use the overload decorator here to specify that the - # return type is only optional if allow_none is True, but this does not work - # when you call a static method from an instance. - # See https://github.com/python/mypy/issues/7781 + @overload + @staticmethod + def simple_select_one_txn( + txn: LoggingTransaction, + table: str, + keyvalues: Dict[str, Any], + retcols: Collection[str], + allow_none: Literal[False] = False, + ) -> Tuple[Any, ...]: ... + + @overload + @staticmethod + def simple_select_one_txn( + txn: LoggingTransaction, + table: str, + keyvalues: Dict[str, Any], + retcols: Collection[str], + allow_none: Literal[True] = True, + ) -> Optional[Tuple[Any, ...]]: ... + @staticmethod def simple_select_one_txn( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index c2c93e12d9..a618a2de69 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -510,19 +510,16 @@ class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore): # it isn't there. raise StoreError(404, "No backup with that version exists") - row = cast( - Tuple[int, str, str, Optional[int]], - self.db_pool.simple_select_one_txn( - txn, - table="e2e_room_keys_versions", - keyvalues={ - "user_id": user_id, - "version": this_version, - "deleted": 0, - }, - retcols=("version", "algorithm", "auth_data", "etag"), - allow_none=False, - ), + row = self.db_pool.simple_select_one_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": this_version, + "deleted": 0, + }, + retcols=("version", "algorithm", "auth_data", "etag"), + allow_none=False, ) return { "auth_data": db_to_json(row[2]), diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index d7cbe33411..8380930c70 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -1510,15 +1510,14 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): # Override type because the return type is only optional if # allow_none is True, and we don't want mypy throwing errors # about None not being indexable. - pending, completed = cast( - Tuple[int, int], - self.db_pool.simple_select_one_txn( - txn, - "registration_tokens", - keyvalues={"token": token}, - retcols=["pending", "completed"], - ), + row = self.db_pool.simple_select_one_txn( + txn, + "registration_tokens", + keyvalues={"token": token}, + retcols=("pending", "completed"), ) + pending = int(row[0]) + completed = int(row[1]) # Decrement pending and increment completed self.db_pool.simple_update_one_txn( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index b4258a4436..40b0bff164 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -1837,15 +1837,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): dict """ - stream_ordering, topological_ordering = cast( - Tuple[int, int], - self.db_pool.simple_select_one_txn( - txn, - "events", - keyvalues={"event_id": event_id, "room_id": room_id}, - retcols=["stream_ordering", "topological_ordering"], - ), + row = self.db_pool.simple_select_one_txn( + txn, + "events", + keyvalues={"event_id": event_id, "room_id": room_id}, + retcols=("stream_ordering", "topological_ordering"), ) + stream_ordering = int(row[0]) + topological_ordering = int(row[1]) # Paginating backwards includes the event at the token, but paginating # forward doesn't. From 068e22b4b7c5d38064cf04353de36fbc1ead6ba9 Mon Sep 17 00:00:00 2001 From: V02460 Date: Mon, 10 Feb 2025 17:53:24 +0100 Subject: [PATCH 130/691] Cleanup Python 3.8 leftovers (#17967) Some small cleanups after Python3.8 became EOL. - Move some type imports from `typing_extensions` to `typing` - Remove the `abi3-py38` feature from pyo3 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/17967.misc | 1 + rust/Cargo.toml | 2 +- synapse/_scripts/synapse_port_db.py | 2 +- synapse/api/auth/__init__.py | 4 +--- synapse/event_auth.py | 2 +- synapse/events/__init__.py | 2 +- synapse/federation/sender/__init__.py | 2 +- synapse/federation/transport/server/__init__.py | 4 +--- synapse/federation/transport/server/federation.py | 3 +-- synapse/handlers/directory.py | 4 +--- synapse/handlers/e2e_room_keys.py | 4 +--- synapse/handlers/oidc.py | 2 +- synapse/handlers/register.py | 3 +-- synapse/handlers/sso.py | 2 +- synapse/http/client.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/http/server.py | 2 +- synapse/http/servlet.py | 3 +-- synapse/logging/context.py | 3 ++- synapse/logging/filter.py | 3 +-- synapse/metrics/jemalloc.py | 3 +-- .../module_api/callbacks/spamchecker_callbacks.py | 4 +--- synapse/push/push_types.py | 4 +--- synapse/rest/client/account.py | 3 +-- synapse/rest/client/directory.py | 4 +--- synapse/rest/client/login.py | 3 +-- synapse/state/v2.py | 4 ++-- synapse/storage/database.py | 3 ++- synapse/storage/databases/main/client_ips.py | 13 +++++++++++-- synapse/storage/databases/main/devices.py | 2 +- synapse/storage/databases/main/e2e_room_keys.py | 15 ++++++++++++--- synapse/storage/databases/main/end_to_end_keys.py | 2 +- synapse/storage/databases/main/events.py | 2 +- synapse/storage/databases/main/events_worker.py | 2 +- synapse/storage/databases/main/stream.py | 3 ++- synapse/storage/databases/main/user_directory.py | 3 +-- synapse/storage/types.py | 3 +-- synapse/types/__init__.py | 3 ++- synapse/util/async_helpers.py | 3 ++- synapse/util/caches/dictionary_cache.py | 13 +++++++++++-- synapse/util/caches/expiringcache.py | 3 +-- synapse/util/caches/lrucache.py | 3 +-- synapse/util/iterutils.py | 3 +-- synapse/util/macaroons.py | 3 +-- synapse/util/metrics.py | 13 +++++++++++-- tests/media/test_media_storage.py | 3 +-- tests/metrics/test_metrics.py | 3 +-- tests/rest/client/test_login.py | 2 +- tests/rest/client/test_media.py | 3 +-- tests/rest/client/test_models.py | 3 +-- tests/rest/client/test_rooms.py | 3 +-- tests/rest/client/utils.py | 2 +- tests/unittest.py | 3 ++- tests/util/test_linearizer.py | 4 +--- tests/utils.py | 3 ++- 55 files changed, 104 insertions(+), 94 deletions(-) create mode 100644 changelog.d/17967.misc diff --git a/changelog.d/17967.misc b/changelog.d/17967.misc new file mode 100644 index 0000000000..60e61a6f7d --- /dev/null +++ b/changelog.d/17967.misc @@ -0,0 +1 @@ +Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library. \ No newline at end of file diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7eebeb3b55..a8a24ee1c7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -34,7 +34,7 @@ pyo3 = { version = "0.23.2", features = [ "macros", "anyhow", "abi3", - "abi3-py38", + "abi3-py39", ] } pyo3-log = "0.12.0" pythonize = "0.23.0" diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index d8f6f8ebdc..3f67a739a0 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -42,12 +42,12 @@ from typing import ( Set, Tuple, Type, + TypedDict, TypeVar, cast, ) import yaml -from typing_extensions import TypedDict from twisted.internet import defer, reactor as reactor_ diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index d5241afe73..1b801d3ad3 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -18,9 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Optional, Tuple - -from typing_extensions import Protocol +from typing import TYPE_CHECKING, Optional, Protocol, Tuple from twisted.web.server import Request diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 3fe344ac93..5ecf493f98 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -32,6 +32,7 @@ from typing import ( Mapping, MutableMapping, Optional, + Protocol, Set, Tuple, Union, @@ -41,7 +42,6 @@ from typing import ( from canonicaljson import encode_canonical_json from signedjson.key import decode_verify_key_bytes from signedjson.sign import SignatureVerifyException, verify_signed_json -from typing_extensions import Protocol from unpaddedbase64 import decode_base64 from synapse.api.constants import ( diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 8e9d27138c..13d41592b3 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -30,6 +30,7 @@ from typing import ( Generic, Iterable, List, + Literal, Optional, Tuple, Type, @@ -39,7 +40,6 @@ from typing import ( ) import attr -from typing_extensions import Literal from unpaddedbase64 import encode_base64 from synapse.api.constants import EventTypes, RelationTypes diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 17cddf18a3..b95b3c629d 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -139,13 +139,13 @@ from typing import ( Hashable, Iterable, List, + Literal, Optional, Tuple, ) import attr from prometheus_client import Counter -from typing_extensions import Literal from twisted.internet import defer diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 43102567db..174d02ab6b 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -20,9 +20,7 @@ # # import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Tuple, Type from synapse.api.errors import FederationDeniedError, SynapseError from synapse.federation.transport.server._base import ( diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index 093ba30d31..eb96ff27f9 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -24,6 +24,7 @@ from typing import ( TYPE_CHECKING, Dict, List, + Literal, Mapping, Optional, Sequence, @@ -32,8 +33,6 @@ from typing import ( Union, ) -from typing_extensions import Literal - from synapse.api.constants import Direction, EduTypes from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersions diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 62ce16794f..48c7d411d5 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -21,9 +21,7 @@ import logging import string -from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Iterable, List, Literal, Optional, Sequence from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes from synapse.api.errors import ( diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index f397911f28..623fd33f13 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -20,9 +20,7 @@ # import logging -from typing import TYPE_CHECKING, Dict, Optional, cast - -from typing_extensions import Literal +from typing import TYPE_CHECKING, Dict, Literal, Optional, cast from synapse.api.errors import ( Codes, diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 701e828081..c9109c9e79 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -31,6 +31,7 @@ from typing import ( List, Optional, Type, + TypedDict, TypeVar, Union, ) @@ -52,7 +53,6 @@ from pymacaroons.exceptions import ( MacaroonInitException, MacaroonInvalidSignatureException, ) -from typing_extensions import TypedDict from twisted.web.client import readBody from twisted.web.http_headers import Headers diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c49db83ce7..ecfea175c7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,10 +23,9 @@ """Contains functions for registering clients.""" import logging -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict from prometheus_client import Counter -from typing_extensions import TypedDict from synapse import types from synapse.api.constants import ( diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 531ed57110..9c0d665461 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -33,12 +33,12 @@ from typing import ( Mapping, NoReturn, Optional, + Protocol, Set, ) from urllib.parse import urlencode import attr -from typing_extensions import Protocol from twisted.web.iweb import IRequest from twisted.web.server import Request diff --git a/synapse/http/client.py b/synapse/http/client.py index 559b1febf0..84a510fb42 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -31,6 +31,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Tuple, Union, ) @@ -40,7 +41,6 @@ import treq from canonicaljson import encode_canonical_json from netaddr import AddrFormatError, IPAddress, IPSet from prometheus_client import Counter -from typing_extensions import Protocol from zope.interface import implementer from OpenSSL import SSL diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e658c68e23..f6d2536957 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -34,6 +34,7 @@ from typing import ( Dict, Generic, List, + Literal, Optional, TextIO, Tuple, @@ -48,7 +49,6 @@ import treq from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json -from typing_extensions import Literal from twisted.internet import defer from twisted.internet.error import DNSLookupError diff --git a/synapse/http/server.py b/synapse/http/server.py index 792961a147..8bebafc18e 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -39,6 +39,7 @@ from typing import ( List, Optional, Pattern, + Protocol, Tuple, Union, ) @@ -46,7 +47,6 @@ from typing import ( import attr import jinja2 from canonicaljson import encode_canonical_json -from typing_extensions import Protocol from zope.interface import implementer from twisted.internet import defer, interfaces diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 0330f1c878..ed6ab08336 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -28,6 +28,7 @@ from http import HTTPStatus from typing import ( TYPE_CHECKING, List, + Literal, Mapping, Optional, Sequence, @@ -37,8 +38,6 @@ from typing import ( overload, ) -from typing_extensions import Literal - from twisted.web.server import Request from synapse._pydantic_compat import ( diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 8a2dfeba13..3ef97f23c9 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -40,6 +40,7 @@ from typing import ( Any, Awaitable, Callable, + Literal, Optional, Tuple, Type, @@ -49,7 +50,7 @@ from typing import ( ) import attr -from typing_extensions import Literal, ParamSpec +from typing_extensions import ParamSpec from twisted.internet import defer, threads from twisted.python.threadpool import ThreadPool diff --git a/synapse/logging/filter.py b/synapse/logging/filter.py index 11c27c63f2..16de488dbc 100644 --- a/synapse/logging/filter.py +++ b/synapse/logging/filter.py @@ -19,8 +19,7 @@ # # import logging - -from typing_extensions import Literal +from typing import Literal class MetadataFilter(logging.Filter): diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py index bd25985686..321ff58083 100644 --- a/synapse/metrics/jemalloc.py +++ b/synapse/metrics/jemalloc.py @@ -23,11 +23,10 @@ import ctypes import logging import os import re -from typing import Iterable, Optional, overload +from typing import Iterable, Literal, Optional, overload import attr from prometheus_client import REGISTRY, Metric -from typing_extensions import Literal from synapse.metrics import GaugeMetricFamily from synapse.metrics._types import Collector diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index 9bc572422b..a86b46ba54 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -29,15 +29,13 @@ from typing import ( Callable, Collection, List, + Literal, Optional, Tuple, Union, cast, ) -# `Literal` appears with Python 3.8. -from typing_extensions import Literal - import synapse from synapse.api.errors import Codes from synapse.logging.opentracing import trace diff --git a/synapse/push/push_types.py b/synapse/push/push_types.py index 201ec97219..57fa926a46 100644 --- a/synapse/push/push_types.py +++ b/synapse/push/push_types.py @@ -18,9 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import List, Optional - -from typing_extensions import TypedDict +from typing import List, Optional, TypedDict class EmailReason(TypedDict, total=False): diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 32fa7b4ec4..59dbad3582 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -21,11 +21,10 @@ # import logging import random -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from urllib.parse import urlparse import attr -from typing_extensions import Literal from twisted.web.server import Request diff --git a/synapse/rest/client/directory.py b/synapse/rest/client/directory.py index 98ba5c4c2a..479f489623 100644 --- a/synapse/rest/client/directory.py +++ b/synapse/rest/client/directory.py @@ -20,9 +20,7 @@ # import logging -from typing import TYPE_CHECKING, List, Optional, Tuple - -from typing_extensions import Literal +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from twisted.web.server import Request diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 3271b02d40..72b219447b 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -30,11 +30,10 @@ from typing import ( List, Optional, Tuple, + TypedDict, Union, ) -from typing_extensions import TypedDict - from synapse.api.constants import ApprovalNoticeMedium from synapse.api.errors import ( Codes, diff --git a/synapse/state/v2.py b/synapse/state/v2.py index da926ad146..d0c0a9fc96 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -29,15 +29,15 @@ from typing import ( Generator, Iterable, List, + Literal, Optional, + Protocol, Sequence, Set, Tuple, overload, ) -from typing_extensions import Literal, Protocol - from synapse import event_auth from synapse.api.constants import EventTypes from synapse.api.errors import AuthError diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 8272e39340..a4941e58f6 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -35,6 +35,7 @@ from typing import ( Iterable, Iterator, List, + Literal, Mapping, Optional, Sequence, @@ -47,7 +48,7 @@ from typing import ( import attr from prometheus_client import Counter, Histogram -from typing_extensions import Concatenate, Literal, ParamSpec +from typing_extensions import Concatenate, ParamSpec from twisted.enterprise import adbapi from twisted.internet.interfaces import IReactorCore diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index bf6cfcbfd9..e8c322ab5c 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -20,10 +20,19 @@ # import logging -from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Dict, + List, + Mapping, + Optional, + Tuple, + TypedDict, + Union, + cast, +) import attr -from typing_extensions import TypedDict from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 8088943253..0b6d1f2b05 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -27,6 +27,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, Optional, Set, @@ -35,7 +36,6 @@ from typing import ( ) from canonicaljson import encode_canonical_json -from typing_extensions import Literal from synapse.api.constants import EduTypes from synapse.api.errors import Codes, StoreError diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index a618a2de69..904ae5cb58 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -19,9 +19,18 @@ # # -from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Tuple, cast - -from typing_extensions import Literal, TypedDict +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + TypedDict, + cast, +) from synapse.api.errors import StoreError from synapse.logging.opentracing import log_kv, trace diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 3bb8fccb5e..b4c7069958 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -27,6 +27,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, Optional, Sequence, @@ -39,7 +40,6 @@ from typing import ( import attr from canonicaljson import encode_canonical_json -from typing_extensions import Literal from synapse.api.constants import DeviceKeyAlgorithms from synapse.appservice import ( diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index a23aaf5096..26fbc1a483 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -35,12 +35,12 @@ from typing import ( Sequence, Set, Tuple, + TypedDict, cast, ) import attr from prometheus_client import Counter -from typing_extensions import TypedDict import synapse.metrics from synapse.api.constants import ( diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 222df8757a..82b2ad4408 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -30,6 +30,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, MutableMapping, Optional, @@ -41,7 +42,6 @@ from typing import ( import attr from prometheus_client import Gauge -from typing_extensions import Literal from twisted.internet import defer diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 40b0bff164..00e5208674 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -50,6 +50,7 @@ from typing import ( Dict, Iterable, List, + Literal, Mapping, Optional, Protocol, @@ -61,7 +62,7 @@ from typing import ( import attr from immutabledict import immutabledict -from typing_extensions import Literal, assert_never +from typing_extensions import assert_never from twisted.internet import defer diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 51cffb0986..a51182de55 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -31,6 +31,7 @@ from typing import ( Sequence, Set, Tuple, + TypedDict, cast, ) @@ -44,8 +45,6 @@ try: except ModuleNotFoundError: USE_ICU = False -from typing_extensions import TypedDict - from synapse.api.errors import StoreError from synapse.util.stringutils import non_null_str_or_none diff --git a/synapse/storage/types.py b/synapse/storage/types.py index 74f60cc590..4329d88c9a 100644 --- a/synapse/storage/types.py +++ b/synapse/storage/types.py @@ -26,14 +26,13 @@ from typing import ( List, Mapping, Optional, + Protocol, Sequence, Tuple, Type, Union, ) -from typing_extensions import Protocol - """ Some very basic protocol definitions for the DB-API2 classes specified in PEP-249 """ diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 3d15c04faa..e9cdd19868 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -40,6 +40,7 @@ from typing import ( Set, Tuple, Type, + TypedDict, TypeVar, Union, overload, @@ -49,7 +50,7 @@ import attr from immutabledict import immutabledict from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey -from typing_extensions import Self, TypedDict +from typing_extensions import Self from unpaddedbase64 import decode_base64 from zope.interface import Interface diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index e1eb8a4863..e596e1ed20 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -41,6 +41,7 @@ from typing import ( Hashable, Iterable, List, + Literal, Optional, Set, Tuple, @@ -51,7 +52,7 @@ from typing import ( ) import attr -from typing_extensions import Concatenate, Literal, ParamSpec, Unpack +from typing_extensions import Concatenate, ParamSpec, Unpack from twisted.internet import defer from twisted.internet.defer import CancelledError diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 1e6696332f..14bd3ba3b0 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -21,10 +21,19 @@ import enum import logging import threading -from typing import Dict, Generic, Iterable, Optional, Set, Tuple, TypeVar, Union +from typing import ( + Dict, + Generic, + Iterable, + Literal, + Optional, + Set, + Tuple, + TypeVar, + Union, +) import attr -from typing_extensions import Literal from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 8017c031ee..3198fdd2ed 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -21,10 +21,9 @@ import logging from collections import OrderedDict -from typing import Any, Generic, Iterable, Optional, TypeVar, Union, overload +from typing import Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload import attr -from typing_extensions import Literal from twisted.internet import defer diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 481a1a621e..2e5efa3a52 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -34,6 +34,7 @@ from typing import ( Generic, Iterable, List, + Literal, Optional, Set, Tuple, @@ -44,8 +45,6 @@ from typing import ( overload, ) -from typing_extensions import Literal - from twisted.internet import reactor from twisted.internet.interfaces import IReactorTime diff --git a/synapse/util/iterutils.py b/synapse/util/iterutils.py index b73f690b88..ff6adeb716 100644 --- a/synapse/util/iterutils.py +++ b/synapse/util/iterutils.py @@ -30,14 +30,13 @@ from typing import ( Iterator, List, Mapping, + Protocol, Set, Sized, Tuple, TypeVar, ) -from typing_extensions import Protocol - T = TypeVar("T") S = TypeVar("S", bound="_SelfSlice") diff --git a/synapse/util/macaroons.py b/synapse/util/macaroons.py index 84ae226207..6fa15543ec 100644 --- a/synapse/util/macaroons.py +++ b/synapse/util/macaroons.py @@ -22,12 +22,11 @@ """Utilities for manipulating macaroons""" -from typing import Callable, Optional +from typing import Callable, Literal, Optional import attr import pymacaroons from pymacaroons.exceptions import MacaroonVerificationFailedException -from typing_extensions import Literal from synapse.util import Clock, stringutils diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 020618598c..6a389f7a7e 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -22,10 +22,19 @@ import logging from functools import wraps from types import TracebackType -from typing import Awaitable, Callable, Dict, Generator, Optional, Type, TypeVar +from typing import ( + Awaitable, + Callable, + Dict, + Generator, + Optional, + Protocol, + Type, + TypeVar, +) from prometheus_client import CollectorRegistry, Counter, Metric -from typing_extensions import Concatenate, ParamSpec, Protocol +from typing_extensions import Concatenate, ParamSpec from synapse.logging.context import ( ContextResourceUsage, diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f4fbc0544a..c2e0e592d7 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -23,14 +23,13 @@ import shutil import tempfile from binascii import unhexlify from io import BytesIO -from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Tuple, Union +from typing import Any, BinaryIO, ClassVar, Dict, List, Literal, Optional, Tuple, Union from unittest.mock import MagicMock, Mock, patch from urllib import parse import attr from parameterized import parameterized, parameterized_class from PIL import Image as Image -from typing_extensions import Literal from twisted.internet import defer from twisted.internet.defer import Deferred diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 80f24814e8..2e7004df3a 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -19,12 +19,11 @@ # # from importlib import metadata -from typing import Dict, Tuple +from typing import Dict, Protocol, Tuple from unittest.mock import patch from pkg_resources import parse_version from prometheus_client.core import Sample -from typing_extensions import Protocol from synapse.app._base import _set_prometheus_client_use_created_metrics from synapse.metrics import REGISTRY, InFlightGauge, generate_latest diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index 1451fd7c29..d7148917d0 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -27,6 +27,7 @@ from typing import ( Collection, Dict, List, + Literal, Optional, Tuple, Union, @@ -35,7 +36,6 @@ from unittest.mock import Mock from urllib.parse import urlencode import pymacaroons -from typing_extensions import Literal from twisted.test.proto_helpers import MemoryReactor from twisted.web.resource import Resource diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 4060525efe..7d7dbd342b 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -24,14 +24,13 @@ import json import os import re import shutil -from typing import Any, BinaryIO, Dict, List, Optional, Sequence, Tuple, Type +from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Sequence, Tuple, Type from unittest.mock import MagicMock, Mock, patch from urllib import parse from urllib.parse import quote, urlencode from parameterized import parameterized, parameterized_class from PIL import Image as Image -from typing_extensions import ClassVar from twisted.internet import defer from twisted.internet._resolver import HostResolution diff --git a/tests/rest/client/test_models.py b/tests/rest/client/test_models.py index f14585ccac..75479e6235 100644 --- a/tests/rest/client/test_models.py +++ b/tests/rest/client/test_models.py @@ -19,8 +19,7 @@ # # import unittest as stdlib_unittest - -from typing_extensions import Literal +from typing import Literal from synapse._pydantic_compat import BaseModel, ValidationError from synapse.types.rest.client import EmailRequestTokenBody diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 5473a8e769..604b585150 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -25,12 +25,11 @@ import json from http import HTTPStatus -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union from unittest.mock import AsyncMock, Mock, call, patch from urllib import parse as urlparse from parameterized import param, parameterized -from typing_extensions import Literal from twisted.test.proto_helpers import MemoryReactor diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index e766630afb..53f1782d59 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -31,6 +31,7 @@ from typing import ( AnyStr, Dict, Iterable, + Literal, Mapping, MutableMapping, Optional, @@ -40,7 +41,6 @@ from typing import ( from urllib.parse import urlencode import attr -from typing_extensions import Literal from twisted.test.proto_helpers import MemoryReactorClock from twisted.web.server import Site diff --git a/tests/unittest.py b/tests/unittest.py index 6a32861a3e..24077d79d6 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -40,6 +40,7 @@ from typing import ( Mapping, NoReturn, Optional, + Protocol, Tuple, Type, TypeVar, @@ -50,7 +51,7 @@ from unittest.mock import Mock, patch import canonicaljson import signedjson.key import unpaddedbase64 -from typing_extensions import Concatenate, ParamSpec, Protocol +from typing_extensions import Concatenate, ParamSpec from twisted.internet.defer import Deferred, ensureDeferred from twisted.python.failure import Failure diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py index 7cbb1007da..7510657b85 100644 --- a/tests/util/test_linearizer.py +++ b/tests/util/test_linearizer.py @@ -19,9 +19,7 @@ # # -from typing import Hashable, Tuple - -from typing_extensions import Protocol +from typing import Hashable, Protocol, Tuple from twisted.internet import defer, reactor from twisted.internet.base import ReactorBase diff --git a/tests/utils.py b/tests/utils.py index d4aebc3069..0006bd7a8d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -28,6 +28,7 @@ from typing import ( Callable, Dict, List, + Literal, Optional, Tuple, Type, @@ -37,7 +38,7 @@ from typing import ( ) import attr -from typing_extensions import Literal, ParamSpec +from typing_extensions import ParamSpec from synapse.api.constants import EventTypes from synapse.api.room_versions import RoomVersions From 703f2e8c43883516e618013c528cd92b42a722ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 00:07:43 +0700 Subject: [PATCH 131/691] Bump types-pyyaml from 6.0.12.20240917 to 6.0.12.20241230 (#18097) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index c9c567d6e3..40f1be243b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2829,13 +2829,13 @@ types-cffi = "*" [[package]] name = "types-pyyaml" -version = "6.0.12.20240917" +version = "6.0.12.20241230" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" files = [ - {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, - {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, + {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"}, + {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"}, ] [[package]] From c1815bf5a10629efc0f7f2b1c073e14d5b0e8f57 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:39:12 +0700 Subject: [PATCH 132/691] Add `rc_presence` ratelimiting config to demo/start.sh (#18145) Missed in https://github.com/element-hq/synapse/pull/18000 --- changelog.d/18145.bugfix | 1 + demo/start.sh | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/18145.bugfix diff --git a/changelog.d/18145.bugfix b/changelog.d/18145.bugfix new file mode 100644 index 0000000000..ad9d353834 --- /dev/null +++ b/changelog.d/18145.bugfix @@ -0,0 +1 @@ +Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. \ No newline at end of file diff --git a/demo/start.sh b/demo/start.sh index 06ec6f985f..7636c41f1f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -138,6 +138,10 @@ for port in 8080 8081 8082; do per_user: per_second: 1000 burst_count: 1000 + rc_presence: + per_user: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" From c1b7c6b12e7b9b2cf586a3210145f70a5e30ed67 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 11 Feb 2025 11:56:50 +0100 Subject: [PATCH 133/691] 1.124.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d310241766..58b410decb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.124.0 (2025-02-11) + +No significant changes since 1.124.0rc3. + + + + # Synapse 1.124.0rc3 (2025-02-07) ### Bugfixes diff --git a/debian/changelog b/debian/changelog index 886b871b72..3e20885c86 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.124.0) stable; urgency=medium + + * New Synapse release 1.124.0. + + -- Synapse Packaging team Tue, 11 Feb 2025 11:55:22 +0100 + matrix-synapse-py3 (1.124.0~rc3) stable; urgency=medium * New Synapse release 1.124.0rc3. diff --git a/pyproject.toml b/pyproject.toml index 6b9f4e0dda..2d982eae70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.124.0rc3" +version = "1.124.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 26331cbbd5ec23ad37783a462e218ce289e56fc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:24:38 +0000 Subject: [PATCH 134/691] Bump serde_json from 1.0.137 to 1.0.138 (#18129) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82ed544226..195b34920f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,9 +449,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", From 30418653fd5fed04e32f169701de075028a59529 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:24:47 +0000 Subject: [PATCH 135/691] Bump gitpython from 3.1.43 to 3.1.44 (#18128) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 40f1be243b..8cdbe0f8bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -472,20 +472,20 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.43" +version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, + {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, + {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] [[package]] From 5a833ebbc883bed18e66b7fcb46138c95f38e16e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:25:21 +0000 Subject: [PATCH 136/691] Bump bcrypt from 4.2.0 to 4.2.1 (#18127) --- poetry.lock | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8cdbe0f8bf..7dee7d098f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -64,38 +64,36 @@ visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] [[package]] name = "bcrypt" -version = "4.2.0" +version = "4.2.1" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.7" files = [ - {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, - {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, - {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, - {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, - {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, - {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, - {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, - {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, - {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, - {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, - {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"}, - {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"}, - {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"}, - {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"}, - {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, + {file = "bcrypt-4.2.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:1340411a0894b7d3ef562fb233e4b6ed58add185228650942bdc885362f32c17"}, + {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ee315739bc8387aa36ff127afc99120ee452924e0df517a8f3e4c0187a0f5f"}, + {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dbd0747208912b1e4ce730c6725cb56c07ac734b3629b60d4398f082ea718ad"}, + {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:aaa2e285be097050dba798d537b6efd9b698aa88eef52ec98d23dcd6d7cf6fea"}, + {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:76d3e352b32f4eeb34703370e370997065d28a561e4a18afe4fef07249cb4396"}, + {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b7703ede632dc945ed1172d6f24e9f30f27b1b1a067f32f68bf169c5f08d0425"}, + {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:89df2aea2c43be1e1fa066df5f86c8ce822ab70a30e4c210968669565c0f4685"}, + {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:04e56e3fe8308a88b77e0afd20bec516f74aecf391cdd6e374f15cbed32783d6"}, + {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cfdf3d7530c790432046c40cda41dfee8c83e29482e6a604f8930b9930e94139"}, + {file = "bcrypt-4.2.1-cp37-abi3-win32.whl", hash = "sha256:adadd36274510a01f33e6dc08f5824b97c9580583bd4487c564fc4617b328005"}, + {file = "bcrypt-4.2.1-cp37-abi3-win_amd64.whl", hash = "sha256:8c458cd103e6c5d1d85cf600e546a639f234964d0228909d8f8dbeebff82d526"}, + {file = "bcrypt-4.2.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8ad2f4528cbf0febe80e5a3a57d7a74e6635e41af1ea5675282a33d769fba413"}, + {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:909faa1027900f2252a9ca5dfebd25fc0ef1417943824783d1c8418dd7d6df4a"}, + {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cde78d385d5e93ece5479a0a87f73cd6fa26b171c786a884f955e165032b262c"}, + {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:533e7f3bcf2f07caee7ad98124fab7499cb3333ba2274f7a36cf1daee7409d99"}, + {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:687cf30e6681eeda39548a93ce9bfbb300e48b4d445a43db4298d2474d2a1e54"}, + {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:041fa0155c9004eb98a232d54da05c0b41d4b8e66b6fc3cb71b4b3f6144ba837"}, + {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f85b1ffa09240c89aa2e1ae9f3b1c687104f7b2b9d2098da4e923f1b7082d331"}, + {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c6f5fa3775966cca251848d4d5393ab016b3afed251163c1436fefdec3b02c84"}, + {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:807261df60a8b1ccd13e6599c779014a362ae4e795f5c59747f60208daddd96d"}, + {file = "bcrypt-4.2.1-cp39-abi3-win32.whl", hash = "sha256:b588af02b89d9fad33e5f98f7838bf590d6d692df7153647724a7f20c186f6bf"}, + {file = "bcrypt-4.2.1-cp39-abi3-win_amd64.whl", hash = "sha256:e84e0e6f8e40a242b11bce56c313edc2be121cec3e0ec2d76fce01f6af33c07c"}, + {file = "bcrypt-4.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76132c176a6d9953cdc83c296aeaed65e1a708485fd55abf163e0d9f8f16ce0e"}, + {file = "bcrypt-4.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e158009a54c4c8bc91d5e0da80920d048f918c61a581f0a63e4e93bb556d362f"}, + {file = "bcrypt-4.2.1.tar.gz", hash = "sha256:6765386e3ab87f569b276988742039baab087b2cdb01e809d74e74503c2faafe"}, ] [package.extras] From c2748392340b25deee42d640e1c399c7170f6664 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:29:23 +0000 Subject: [PATCH 137/691] Bump bytes from 1.9.0 to 1.10.0 (#18149) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 195b34920f..c1055f5674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,9 +61,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "cfg-if" From fe3f462b791dc29671f8b154d9c8aaf41d724bb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:30:36 +0000 Subject: [PATCH 138/691] Bump sigstore/cosign-installer from 3.7.0 to 3.8.0 (#18147) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 82cacdfeb3..b439cb0915 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 + uses: sigstore/cosign-installer@v3.8.0 - name: Checkout repository uses: actions/checkout@v4 From aaffc3566e3303f7b43be262532a09f187c11346 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:33:14 +0000 Subject: [PATCH 139/691] Bump ulid from 1.1.4 to 1.2.0 (#18148) --- Cargo.lock | 130 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1055f5674..8831f7e6fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + [[package]] name = "blake2" version = "0.10.6" @@ -119,13 +125,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", "wasi", + "windows-targets", ] [[package]] @@ -364,20 +371,20 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ - "libc", "rand_chacha", "rand_core", + "zerocopy", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", @@ -385,11 +392,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom", + "zerocopy", ] [[package]] @@ -536,9 +544,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ulid" -version = "1.1.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f294bff79170ed1c5633812aff1e565c35d993a36e757f9bc0accf5eec4e6045" +checksum = "ab82fc73182c29b02e2926a6df32f2241dbadb5cfc111fd595515b3598f46bb3" dependencies = [ "rand", "web-time", @@ -564,9 +572,12 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] [[package]] name = "wasm-bindgen" @@ -631,3 +642,96 @@ dependencies = [ "js-sys", "wasm-bindgen", ] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] From 816054b0122fa88cde71ca03aee7f97f382af89b Mon Sep 17 00:00:00 2001 From: qashlan Date: Wed, 12 Feb 2025 16:06:21 +0200 Subject: [PATCH 140/691] Fix internal server error when updating 3pid address with invalid email (#18125) When updating 3pid for a user email from admin api and sending invalid email the server throws 500 internal server error. changed to 400 Bad request and returned the error message Signed-off-by: qashlan Signed-off-by: Ahmed Qashlan --- changelog.d/18125.bugfix | 1 + synapse/handlers/auth.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18125.bugfix diff --git a/changelog.d/18125.bugfix b/changelog.d/18125.bugfix new file mode 100644 index 0000000000..0e6aa704ff --- /dev/null +++ b/changelog.d/18125.bugfix @@ -0,0 +1 @@ +Fix a bug when updating a user 3pid with invalid returns 500 server error change to 400 with a message. \ No newline at end of file diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1f4264ad7e..e96922c08d 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1579,7 +1579,10 @@ class AuthHandler: # for the presence of an email address during password reset was # case sensitive). if medium == "email": - address = canonicalise_email(address) + try: + address = canonicalise_email(address) + except ValueError as e: + raise SynapseError(400, str(e)) await self.store.user_add_threepid( user_id, medium, address, validated_at, self.hs.get_clock().time_msec() @@ -1610,7 +1613,10 @@ class AuthHandler: """ # 'Canonicalise' email addresses as per above if medium == "email": - address = canonicalise_email(address) + try: + address = canonicalise_email(address) + except ValueError as e: + raise SynapseError(400, str(e)) await self.store.user_delete_threepid(user_id, medium, address) From 74aa47828d1013e7c13bee0ec9fd5b1901f20f76 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 12 Feb 2025 14:37:56 +0000 Subject: [PATCH 141/691] Add log message when worker lock timeouts get large (#18124) This is to help track down a possible, but very rare, worker deadlock that was seen on matrix.org. In theory, you could work back from an instance of these new logs to the approximate time when the lock was obtained and focus the diagnostic efforts there. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18124.misc | 1 + synapse/handlers/worker_lock.py | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 changelog.d/18124.misc diff --git a/changelog.d/18124.misc b/changelog.d/18124.misc new file mode 100644 index 0000000000..8ac6a73a20 --- /dev/null +++ b/changelog.d/18124.misc @@ -0,0 +1 @@ +Add log message when worker lock timeouts get large. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index db998f6701..e58a416026 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -19,6 +19,7 @@ # # +import logging import random from types import TracebackType from typing import ( @@ -269,6 +270,10 @@ class WaitingLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + logging.warning( + f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + ) return next * random.uniform(0.9, 1.1) @@ -344,4 +349,8 @@ class WaitingMultiLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + logging.warning( + f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + ) return next * random.uniform(0.9, 1.1) From 0b1830b121510c04adf856f252bcc2839c35d738 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Feb 2025 11:16:06 -0600 Subject: [PATCH 142/691] Document missing server config options (#18122) I was looking into the `USE_FROZEN_DICTS` option during the review of https://github.com/element-hq/synapse/pull/18103#discussion_r1935876168 and noticed that there are several other server config options that aren't in the docs. --- changelog.d/18122.doc | 1 + .../configuration/config_documentation.md | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 changelog.d/18122.doc diff --git a/changelog.d/18122.doc b/changelog.d/18122.doc new file mode 100644 index 0000000000..b7bb7107e2 --- /dev/null +++ b/changelog.d/18122.doc @@ -0,0 +1 @@ +Document missing server config options (`daemonize`, `print_pidfile`, `user_agent_suffix`, `use_frozen_dicts`, `manhole`). diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index a5f23149ec..fdca48caf0 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -162,6 +162,53 @@ Example configuration: pid_file: DATADIR/homeserver.pid ``` --- +### `daemonize` + +Specifies whether Synapse should be started as a daemon process. If Synapse is being +managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to +`false`. + +This can also be set by the `--daemonize` (`-D`) argument when starting Synapse. + +See `worker_daemonize` for more information on daemonizing workers. + +Example configuration: +```yaml +daemonize: true +``` +--- +### `print_pidfile` + +Print the path to the pidfile just before daemonizing. Defaults to false. + +This can also be set by the `--print-pidfile` argument when starting Synapse. + +Example configuration: +```yaml +print_pidfile: true +``` +--- +### `user_agent_suffix` + +A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). Defaults +to None + +Example configuration: +```yaml +user_agent_suffix: " (I'm a teapot; Linux x86_64)" +``` +--- +### `use_frozen_dicts` + +Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing +prevents bugs where we accidentally share e.g. signature dicts. However, freezing a +dict is expensive. Defaults to false. + +Example configuration: +```yaml +use_frozen_dicts: true +``` +--- ### `web_client_location` The absolute URL to the web client which `/` will redirect to. Defaults to none. @@ -595,6 +642,17 @@ listeners: - names: [client, federation] ``` +--- +### `manhole` + +Turn on the Twisted telnet manhole service on the given port. Defaults to none. + +This can also be set by the `--manhole` argument when starting Synapse. + +Example configuration: +```yaml +manhole: 1234 +``` --- ### `manhole_settings` From 74a70190ab3bc17aa7f31d00daa98444f7577b61 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 14 Feb 2025 17:23:40 +0700 Subject: [PATCH 143/691] Update `rc_presence` config docs with int burst_count (#18159) --- changelog.d/18159.doc | 1 + docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18159.doc diff --git a/changelog.d/18159.doc b/changelog.d/18159.doc new file mode 100644 index 0000000000..031f8a149f --- /dev/null +++ b/changelog.d/18159.doc @@ -0,0 +1 @@ +Make `burst_count` field an integer in `rc_presence` config documentation example. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index fdca48caf0..92ab6438c1 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1944,7 +1944,7 @@ Example configuration: rc_presence: per_user: per_second: 0.05 - burst_count: 0.5 + burst_count: 1 ``` --- ### `federation_rr_transactions_per_room_per_second` From 22bb3c50d1c10e6687ea4346fb848b7c160441d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:08:11 +0000 Subject: [PATCH 144/691] Bump twine from 6.0.1 to 6.1.0 (#18170) --- poetry.lock | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7dee7d098f..3f9f940fd1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,6 +603,25 @@ files = [ [package.dependencies] idna = ">=2.5" +[[package]] +name = "id" +version = "1.5.0" +description = "A tool for generating OIDC identities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "id-1.5.0-py3-none-any.whl", hash = "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658"}, + {file = "id-1.5.0.tar.gz", hash = "sha256:292cb8a49eacbbdbce97244f47a97b4c62540169c976552e497fd57df0734c1d"}, +] + +[package.dependencies] +requests = "*" + +[package.extras] +dev = ["build", "bump (>=1.3.2)", "id[lint,test]"] +lint = ["bandit", "interrogate", "mypy", "ruff (<0.8.2)", "types-requests"] +test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.10" @@ -1544,20 +1563,6 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole typing = ["typing-extensions"] xmp = ["defusedxml"] -[[package]] -name = "pkginfo" -version = "1.9.6" -description = "Query metadata from sdists / bdists / installed packages." -optional = false -python-versions = ">=3.6" -files = [ - {file = "pkginfo-1.9.6-py3-none-any.whl", hash = "sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546"}, - {file = "pkginfo-1.9.6.tar.gz", hash = "sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046"}, -] - -[package.extras] -testing = ["pytest", "pytest-cov"] - [[package]] name = "prometheus-client" version = "0.21.0" @@ -2625,20 +2630,20 @@ docs = ["sphinx (<7.0.0)"] [[package]] name = "twine" -version = "6.0.1" +version = "6.1.0" description = "Collection of utilities for publishing packages on PyPI" optional = false python-versions = ">=3.8" files = [ - {file = "twine-6.0.1-py3-none-any.whl", hash = "sha256:9c6025b203b51521d53e200f4a08b116dee7500a38591668c6a6033117bdc218"}, - {file = "twine-6.0.1.tar.gz", hash = "sha256:36158b09df5406e1c9c1fb8edb24fc2be387709443e7376689b938531582ee27"}, + {file = "twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384"}, + {file = "twine-6.1.0.tar.gz", hash = "sha256:be324f6272eff91d07ee93f251edf232fc647935dd585ac003539b42404a8dbd"}, ] [package.dependencies] +id = "*" importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} keyring = {version = ">=15.1", markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} -packaging = "*" -pkginfo = ">=1.8.1" +packaging = ">=24.0" readme-renderer = ">=35.0" requests = ">=2.20" requests-toolbelt = ">=0.8.0,<0.9.0 || >0.9.0" From 8ae9d9e8c547293889b9198b62f7c5bf87693f0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:08:21 +0000 Subject: [PATCH 145/691] Bump service-identity from 24.1.0 to 24.2.0 (#18171) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3f9f940fd1..dfd9a838ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2376,13 +2376,13 @@ tornado = ["tornado (>=6)"] [[package]] name = "service-identity" -version = "24.1.0" +version = "24.2.0" description = "Service identity verification for pyOpenSSL & cryptography." optional = false python-versions = ">=3.8" files = [ - {file = "service_identity-24.1.0-py3-none-any.whl", hash = "sha256:a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a"}, - {file = "service_identity-24.1.0.tar.gz", hash = "sha256:6829c9d62fb832c2e1c435629b0a8c476e1929881f28bee4d20bc24161009221"}, + {file = "service_identity-24.2.0-py3-none-any.whl", hash = "sha256:6b047fbd8a84fd0bb0d55ebce4031e400562b9196e1e0d3e0fe2b8a59f6d4a85"}, + {file = "service_identity-24.2.0.tar.gz", hash = "sha256:b8683ba13f0d39c6cd5d625d2c5f65421d6d707b013b375c355751557cbe8e09"}, ] [package.dependencies] @@ -2392,7 +2392,7 @@ pyasn1 = "*" pyasn1-modules = "*" [package.extras] -dev = ["pyopenssl", "service-identity[idna,mypy,tests]"] +dev = ["coverage[toml] (>=5.0.2)", "idna", "mypy", "pyopenssl", "pytest", "types-pyopenssl"] docs = ["furo", "myst-parser", "pyopenssl", "sphinx", "sphinx-notfound-page"] idna = ["idna"] mypy = ["idna", "mypy", "types-pyopenssl"] From 3e34f5ccc706e443d70791e46748520f77dc65a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:08:51 +0000 Subject: [PATCH 146/691] Bump hiredis from 3.0.0 to 3.1.0 (#18169) --- poetry.lock | 205 ++++++++++++++++++++++++++++------------------------ 1 file changed, 110 insertions(+), 95 deletions(-) diff --git a/poetry.lock b/poetry.lock index dfd9a838ce..0d388eff6a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -488,105 +488,120 @@ test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", [[package]] name = "hiredis" -version = "3.0.0" +version = "3.1.0" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" files = [ - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:4b182791c41c5eb1d9ed736f0ff81694b06937ca14b0d4dadde5dadba7ff6dae"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:13c275b483a052dd645eb2cb60d6380f1f5215e4c22d6207e17b86be6dd87ffa"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1018cc7f12824506f165027eabb302735b49e63af73eb4d5450c66c88f47026"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83a29cc7b21b746cb6a480189e49f49b2072812c445e66a9e38d2004d496b81c"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e241fab6332e8fb5f14af00a4a9c6aefa22f19a336c069b7ddbf28ef8341e8d6"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1fb8de899f0145d6c4d5d4bd0ee88a78eb980a7ffabd51e9889251b8f58f1785"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b23291951959141173eec10f8573538e9349fa27f47a0c34323d1970bf891ee5"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e421ac9e4b5efc11705a0d5149e641d4defdc07077f748667f359e60dc904420"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:77c8006c12154c37691b24ff293c077300c22944018c3ff70094a33e10c1d795"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:41afc0d3c18b59eb50970479a9c0e5544fb4b95e3a79cf2fbaece6ddefb926fe"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:04ccae6dcd9647eae6025425ab64edb4d79fde8b9e6e115ebfabc6830170e3b2"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fe91d62b0594db5ea7d23fc2192182b1a7b6973f628a9b8b2e0a42a2be721ac6"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:99516d99316062824a24d145d694f5b0d030c80da693ea6f8c4ecf71a251d8bb"}, - {file = "hiredis-3.0.0-cp310-cp310-win32.whl", hash = "sha256:562eaf820de045eb487afaa37e6293fe7eceb5b25e158b5a1974b7e40bf04543"}, - {file = "hiredis-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a1c81c89ed765198da27412aa21478f30d54ef69bf5e4480089d9c3f77b8f882"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:4664dedcd5933364756d7251a7ea86d60246ccf73a2e00912872dacbfcef8978"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:47de0bbccf4c8a9f99d82d225f7672b9dd690d8fd872007b933ef51a302c9fa6"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e43679eca508ba8240d016d8cca9d27342d70184773c15bea78a23c87a1922f1"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13c345e7278c210317e77e1934b27b61394fee0dec2e8bd47e71570900f75823"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00018f22f38530768b73ea86c11f47e8d4df65facd4e562bd78773bd1baef35e"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ea3a86405baa8eb0d3639ced6926ad03e07113de54cb00fd7510cb0db76a89d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c073848d2b1d5561f3903879ccf4e1a70c9b1e7566c7bdcc98d082fa3e7f0a1d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a8dffb5f5b3415a4669d25de48b617fd9d44b0bccfc4c2ab24b06406ecc9ecb"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:22c17c96143c2a62dfd61b13803bc5de2ac526b8768d2141c018b965d0333b66"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3ece960008dab66c6b8bb3a1350764677ee7c74ccd6270aaf1b1caf9ccebb46"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f75999ae00a920f7dce6ecae76fa5e8674a3110e5a75f12c7a2c75ae1af53396"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e069967cbd5e1900aafc4b5943888f6d34937fc59bf8918a1a546cb729b4b1e4"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0aacc0a78e1d94d843a6d191f224a35893e6bdfeb77a4a89264155015c65f126"}, - {file = "hiredis-3.0.0-cp311-cp311-win32.whl", hash = "sha256:719c32147ba29528cb451f037bf837dcdda4ff3ddb6cdb12c4216b0973174718"}, - {file = "hiredis-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:bdc144d56333c52c853c31b4e2e52cfbdb22d3da4374c00f5f3d67c42158970f"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:484025d2eb8f6348f7876fc5a2ee742f568915039fcb31b478fd5c242bb0fe3a"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fcdb552ffd97151dab8e7bc3ab556dfa1512556b48a367db94b5c20253a35ee1"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bb6f9fd92f147ba11d338ef5c68af4fd2908739c09e51f186e1d90958c68cc1"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa86bf9a0ed339ec9e8a9a9d0ae4dccd8671625c83f9f9f2640729b15e07fbfd"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e194a0d5df9456995d8f510eab9f529213e7326af6b94770abf8f8b7952ddcaa"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a1df39d74ec507d79c7a82c8063eee60bf80537cdeee652f576059b9cdd15c"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f91456507427ba36fd81b2ca11053a8e112c775325acc74e993201ea912d63e9"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9862db92ef67a8a02e0d5370f07d380e14577ecb281b79720e0d7a89aedb9ee5"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d10fcd9e0eeab835f492832b2a6edb5940e2f1230155f33006a8dfd3bd2c94e4"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:48727d7d405d03977d01885f317328dc21d639096308de126c2c4e9950cbd3c9"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e0bb6102ebe2efecf8a3292c6660a0e6fac98176af6de67f020bea1c2343717"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:df274e3abb4df40f4c7274dd3e587dfbb25691826c948bc98d5fead019dfb001"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:034925b5fb514f7b11aac38cd55b3fd7e9d3af23bd6497f3f20aa5b8ba58e232"}, - {file = "hiredis-3.0.0-cp312-cp312-win32.whl", hash = "sha256:120f2dda469b28d12ccff7c2230225162e174657b49cf4cd119db525414ae281"}, - {file = "hiredis-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:e584fe5f4e6681d8762982be055f1534e0170f6308a7a90f58d737bab12ff6a8"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:122171ff47d96ed8dd4bba6c0e41d8afaba3e8194949f7720431a62aa29d8895"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:ba9fc605ac558f0de67463fb588722878641e6fa1dabcda979e8e69ff581d0bd"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a631e2990b8be23178f655cae8ac6c7422af478c420dd54e25f2e26c29e766f1"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63482db3fadebadc1d01ad33afa6045ebe2ea528eb77ccaabd33ee7d9c2bad48"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f669212c390eebfbe03c4e20181f5970b82c5d0a0ad1df1785f7ffbe7d61150"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a49ef161739f8018c69b371528bdb47d7342edfdee9ddc75a4d8caddf45a6e"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98a152052b8878e5e43a2e3a14075218adafc759547c98668a21e9485882696c"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50a196af0ce657fcde9bf8a0bbe1032e22c64d8fcec2bc926a35e7ff68b3a166"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f2f312eef8aafc2255e3585dcf94d5da116c43ef837db91db9ecdc1bc930072d"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:6ca41fa40fa019cde42c21add74aadd775e71458051a15a352eabeb12eb4d084"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:6eecb343c70629f5af55a8b3e53264e44fa04e155ef7989de13668a0cb102a90"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:c3fdad75e7837a475900a1d3a5cc09aa024293c3b0605155da2d42f41bc0e482"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8854969e7480e8d61ed7549eb232d95082a743e94138d98d7222ba4e9f7ecacd"}, - {file = "hiredis-3.0.0-cp38-cp38-win32.whl", hash = "sha256:f114a6c86edbf17554672b050cce72abf489fe58d583c7921904d5f1c9691605"}, - {file = "hiredis-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7d99b91e42217d7b4b63354b15b41ce960e27d216783e04c4a350224d55842a4"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:4c6efcbb5687cf8d2aedcc2c3ed4ac6feae90b8547427d417111194873b66b06"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5b5cff42a522a0d81c2ae7eae5e56d0ee7365e0c4ad50c4de467d8957aff4414"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:82f794d564f4bc76b80c50b03267fe5d6589e93f08e66b7a2f674faa2fa76ebc"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a4c1791d7aa7e192f60fe028ae409f18ccdd540f8b1e6aeb0df7816c77e4a4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2537b2cd98192323fce4244c8edbf11f3cac548a9d633dbbb12b48702f379f4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fed69bbaa307040c62195a269f82fc3edf46b510a17abb6b30a15d7dab548df"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869f6d5537d243080f44253491bb30aa1ec3c21754003b3bddeadedeb65842b0"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d435ae89073d7cd51e6b6bf78369c412216261c9c01662e7008ff00978153729"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:204b79b30a0e6be0dc2301a4d385bb61472809f09c49f400497f1cdd5a165c66"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3ea635101b739c12effd189cc19b2671c268abb03013fd1f6321ca29df3ca625"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f359175197fd833c8dd7a8c288f1516be45415bb5c939862ab60c2918e1e1943"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac6d929cb33dd12ad3424b75725975f0a54b5b12dbff95f2a2d660c510aa106d"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:100431e04d25a522ef2c3b94f294c4219c4de3bfc7d557b6253296145a144c11"}, - {file = "hiredis-3.0.0-cp39-cp39-win32.whl", hash = "sha256:e1a9c14ae9573d172dc050a6f63a644457df5d01ec4d35a6a0f097f812930f83"}, - {file = "hiredis-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:54a6dd7b478e6eb01ce15b3bb5bf771e108c6c148315bf194eb2ab776a3cac4d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:50da7a9edf371441dfcc56288d790985ee9840d982750580710a9789b8f4a290"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9b285ef6bf1581310b0d5e8f6ce64f790a1c40e89c660e1320b35f7515433672"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcfa684966f25b335072115de2f920228a3c2caf79d4bfa2b30f6e4f674a948"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a41be8af1fd78ca97bc948d789a09b730d1e7587d07ca53af05758f31f4b985d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:038756db735e417ab36ee6fd7725ce412385ed2bd0767e8179a4755ea11b804f"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fcecbd39bd42cef905c0b51c9689c39d0cc8b88b1671e7f40d4fb213423aef3a"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a131377493a59fb0f5eaeb2afd49c6540cafcfba5b0b3752bed707be9e7c4eaf"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d22c53f0ec5c18ecb3d92aa9420563b1c5d657d53f01356114978107b00b860"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a91e9520fbc65a799943e5c970ffbcd67905744d8becf2e75f9f0a5e8414f0"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dc8043959b50141df58ab4f398e8ae84c6f9e673a2c9407be65fc789138f4a6"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b99cfac514173d7b8abdfe10338193e8a0eccdfe1870b646009d2fb7cbe4b5"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:fa1fcad89d8a41d8dc10b1e54951ec1e161deabd84ed5a2c95c3c7213bdb3514"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:898636a06d9bf575d2c594129085ad6b713414038276a4bfc5db7646b8a5be78"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:466f836dbcf86de3f9692097a7a01533dc9926986022c6617dc364a402b265c5"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23142a8af92a13fc1e3f2ca1d940df3dcf2af1d176be41fe8d89e30a837a0b60"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793c80a3d6b0b0e8196a2d5de37a08330125668c8012922685e17aa9108c33ac"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:467d28112c7faa29b7db743f40803d927c8591e9da02b6ce3d5fadc170a542a2"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dc384874a719c767b50a30750f937af18842ee5e288afba95a5a3ed703b1515a"}, - {file = "hiredis-3.0.0.tar.gz", hash = "sha256:fed8581ae26345dea1f1e0d1a96e05041a727a45e7d8d459164583e23c6ac441"}, + {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, + {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, + {file = "hiredis-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2af62070aa9433802cae7be7364d5e82f76462c6a2ae34e53008b637aaa9a156"}, + {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:072c162260ebb1d892683107da22d0d5da7a1414739eae4e185cac22fe89627f"}, + {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6b232c43e89755ba332c2745ddab059c0bc1a0f01448a3a14d506f8448b1ce6"}, + {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb5316c9a65c4dde80796aa245b76011bab64eb84461a77b0a61c1bf2970bcc9"}, + {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e812a4e656bbd1c1c15c844b28259c49e26bb384837e44e8d2aa55412c91d2f7"}, + {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93a6c9230e5a5565847130c0e1005c8d3aa5ca681feb0ed542c4651323d32feb"}, + {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a5f65e89ce50a94d9490d5442a649c6116f53f216c8c14eb37cf9637956482b2"}, + {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9b2d6e33601c67c074c367fdccdd6033e642284e7a56adc130f18f724c378ca8"}, + {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bad3b1e0c83849910f28c95953417106f539277035a4b515d1425f93947bc28f"}, + {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9646de31f5994e6218311dcf216e971703dbf804c510fd3f84ddb9813c495824"}, + {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:59a9230f3aa38a33d09d8171400de202f575d7a38869e5ce2947829bca6fe359"}, + {file = "hiredis-3.1.0-cp310-cp310-win32.whl", hash = "sha256:0322d70f3328b97da14b6e98b18f0090a12ed8a8bf7ae20932e2eb9d1bb0aa2c"}, + {file = "hiredis-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:802474c18e878b3f9905e160a8b7df87d57885758083eda76c5978265acb41aa"}, + {file = "hiredis-3.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:c339ff4b4739b2a40da463763dd566129762f72926bca611ad9a457a9fe64abd"}, + {file = "hiredis-3.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:0ffa2552f704a45954627697a378fc2f559004e53055b82f00daf30bd4305330"}, + {file = "hiredis-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9acf7f0e7106f631cd618eb60ec9bbd6e43045addd5310f66ba1177209567e59"}, + {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea4f5ecf9dbea93c827486f59c606684c3496ea71c7ba9a8131932780696e61a"}, + {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39efab176fca3d5111075f6ba56cd864f18db46d858289d39360c5672e0e5c3e"}, + {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1110eae007f30e70a058d743e369c24430327cd01fd97d99519d6794a58dd587"}, + {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b390f63191bcccbb6044d4c118acdf4fa55f38e5658ac4cfd5a33a6f0c07659"}, + {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72a98ccc7b8ec9ce0100ecf59f45f05d2023606e8e3676b07a316d1c1c364072"}, + {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7c76e751fd1e2f221dec09cdc24040ee486886e943d5d7ffc256e8cf15c75e51"}, + {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7d3880f213b6f14e9c69ce52beffd1748eecc8669698c4782761887273b6e1bd"}, + {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:87c2b3fe7e7c96eba376506a76e11514e07e848f737b254e0973e4b5c3a491e9"}, + {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d3cfb4089e96f8f8ee9554da93148a9261aa6612ad2cc202c1a494c7b712e31f"}, + {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f12018e5c5f866a1c3f7017cb2d88e5c6f9440df2281e48865a2b6c40f247f4"}, + {file = "hiredis-3.1.0-cp311-cp311-win32.whl", hash = "sha256:107b66ce977bb2dff8f2239e68344360a75d05fed3d9fa0570ac4d3020ce2396"}, + {file = "hiredis-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8f1240bde53d3d1676f0aba61b3661560dc9a681cae24d9de33e650864029aa4"}, + {file = "hiredis-3.1.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:f7c7f89e0bc4246115754e2eda078a111282f6d6ecc6fb458557b724fe6f2aac"}, + {file = "hiredis-3.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:3dbf9163296fa45fbddcfc4c5900f10e9ddadda37117dbfb641e327e536b53e0"}, + {file = "hiredis-3.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af46a4be0e82df470f68f35316fa16cd1e134d1c5092fc1082e1aad64cce716d"}, + {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc63d698c43aea500a84d8b083f830c03808b6cf3933ae4d35a27f0a3d881652"}, + {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:676b3d88674134bfaaf70dac181d1790b0f33b3187bfb9da9221e17e0e624f83"}, + {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aed10d9df1e2fb0011db2713ac64497462e9c2c0208b648c97569da772b959ca"}, + {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b5bd8adfe8742e331a94cccd782bffea251fa70d9a709e71f4510f50794d700"}, + {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fc4e35b4afb0af6da55495dd0742ad32ab88150428a6ecdbb3085cbd60714e8"}, + {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:89b83e76eb00ab0464e7b0752a3ffcb02626e742e9509bc141424a9c3202e8dc"}, + {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:98ebf08c907836b70a8f40e030df8ab6f174dc7f6fa765251d813e89f14069d8"}, + {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6c840b9cec086328f2ee2cfee0038b5d6bbb514bac7b5e579da6e346eaac056c"}, + {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:c5c44e9fa6f4462d0330cb5f5d46fa652512fc86b41d4d1974d0356f263e9105"}, + {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e665b14ab50aa175cfa306fcb00fffd4e3ff02ceb36ca6a4df00b1246d6a73c4"}, + {file = "hiredis-3.1.0-cp312-cp312-win32.whl", hash = "sha256:bd33db977ac7af97e8d035ffadb163b00546be22e5f1297b2123f5f9bf0f8a21"}, + {file = "hiredis-3.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:37aed4aa9348600145e2d019c7be27855e503ecc4906c6976ff2f3b52e3d5d97"}, + {file = "hiredis-3.1.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:b87cddd8107487863fed6994de51e5594a0be267b0b19e213694e99cdd614623"}, + {file = "hiredis-3.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:d302deff8cb63a7feffc1844e4dafc8076e566bbf10c5aaaf0f4fe791b8a6bd0"}, + {file = "hiredis-3.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a018340c073cf88cb635b2bedff96619df2f666018c655e7911f46fa2c1c178"}, + {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e8ba6414ac1ae536129e18c069f3eb497df5a74e136e3566471620a4fa5f95"}, + {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a86b9fef256c2beb162244791fdc025aa55f936d6358e86e2020e512fe2e4972"}, + {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7acdc68e29a446ad17aadaff19c981a36b3bd8c894c3520412c8a7ab1c3e0de7"}, + {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7e06baea05de57e1e7548064f505a6964e992674fe61b8f274afe2ac93b6371"}, + {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b5fc061c8a0dbfdb440053280504d6aaa8d9726bd4d1d0e1cfcbbdf0d60b73"}, + {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c89d2dcb271d24c44f02264233b75d5db8c58831190fa92456a90b87fa17b748"}, + {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:aa36688c10a08f626fddcf68c2b1b91b0e90b070c26e550a4151a877f5c2d431"}, + {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f3982a9c16c1c4bc05a00b65d01ffb8d80ea1a7b6b533be2f1a769d3e989d2c0"}, + {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d1a6f889514ee2452300c9a06862fceedef22a2891f1c421a27b1ba52ef130b2"}, + {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8a45ff7915392a55d9386bb235ea1d1eb9960615f301979f02143fc20036b699"}, + {file = "hiredis-3.1.0-cp313-cp313-win32.whl", hash = "sha256:539e5bb725b62b76a5319a4e68fc7085f01349abc2316ef3df608ea0883c51d2"}, + {file = "hiredis-3.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9020fd7e58f489fda6a928c31355add0e665fd6b87b21954e675cf9943eafa32"}, + {file = "hiredis-3.1.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:b621a89fc29b3f4b01be6640ec81a6a94b5382bc78fecb876408d57a071e45aa"}, + {file = "hiredis-3.1.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:363e21fba55e1a26349dc9ca7da6b14332123879b6359bcee4a9acecb40ca33b"}, + {file = "hiredis-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c156156798729eadc9ab76ffee96c88b93cc1c3b493f4dd0a4341f53939194ee"}, + {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e38d8a325f9a6afac1b1c72d996d1add9e1b99696ce9410538ba5e9aa8fdba02"}, + {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3004ef7436feb7bfa61c0b36d422b8fb8c29aaa1a514c9405f0fdee5e9694dd3"}, + {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13f5b16f97d0bbd1c04ce367c49097d1214d60e11f9fee7ef2a9b54e0a6645c8"}, + {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:230dd0e77cb0f525f58a1306a7b4aaf078037fc5229110922332ca46f90821bb"}, + {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d968116caddd19d63120d1298e62b1bbc694db3360ed0d5df8c3a97edbc12552"}, + {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:511e36a6fa41d3efab3cd5cd70ac388ed825993b9e66fa3b0e47cf27a2f5ffee"}, + {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c5cd20804e3cb0d31e7d899d8dd091f569c33fe40d4bade670a067ab7d31c2ac"}, + {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:09e89e7d34cfe5ca8f7a869fca827d1af0afe8aaddb26b38c01058730edb79ad"}, + {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:570cbf31413c77fe5e7c157f2943ca4400493ddd9cf2184731cfcafc753becd7"}, + {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:b9b4da8162cf289781732d6a5ba01d820c42c05943fcdb7de307d03639961db3"}, + {file = "hiredis-3.1.0-cp38-cp38-win32.whl", hash = "sha256:bc117a04bcb461d3bb1b2c5b417aee3442e1e8aa33ebc800481431f4c09fe0c5"}, + {file = "hiredis-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:34f3f5f0354db2d6797a6fb08d2c036a50af62a1d919d122c1c784304ef49347"}, + {file = "hiredis-3.1.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:a26fa888025badb5563f283cc19594c215a413e905729e59a5f7cf3f46d66c32"}, + {file = "hiredis-3.1.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:f50763cd819d4a52a47b5966d4bb47dee34b637c5fa6402509800eee6ecb61e6"}, + {file = "hiredis-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b6d1c9e1fce5e0a94072667ae2bf0142b89ebbb1917d3531184e060a43f3ee11"}, + {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e38d7a56b1a79ed0bbb9e6fe376d82e3f4dcc646ae47472f2c858e19a597c112"}, + {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ef5ad8b91530e4d10a68562b0a380ea22705a60e88cecee086d7c63a38564ce"}, + {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf3d2299b054e57a9f97ca08704c2843e44f29b57dc69b76a2592ecd212efe1a"}, + {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93811d60b0f73d0f049c86f4373a3833b4a38fce374ab151074d929553eb4304"}, + {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e703ff860c1d83abbcf57012b309ead02b56b60e85150c6c3bfb37cbb16ebf"}, + {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f9ea0678806c53d96758e74c6a898f9d506a2e3367a344757f768bef9e069366"}, + {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cf6844035abf47d52a1c3f4257255af3bf3b0f14d559b08eaa45885418c6c55d"}, + {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7acf35cfa7ec9e1e7559c04e7095628f7d06049b5f24dcb58c1a55ef6dc689f8"}, + {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b885695dce7a39b1fd9a609ed9c4cf312e53df2ec028d5a78af7a891b5fbea4d"}, + {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1c22fa74ddd063396b19fe8445a1ae8b4190eff755d5750dda48e860a45b2ee7"}, + {file = "hiredis-3.1.0-cp39-cp39-win32.whl", hash = "sha256:0614e16339f1784df3bbd2800322e20b4127d3f3a3509f00a5562efddb2521aa"}, + {file = "hiredis-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:c2bc713ee73ab9de4a0d68b0ab0f29612342b63173714742437b977584adb2d8"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:07ab990d0835f36bf358dbb84db4541ac0a8f533128ec09af8f80a576eef2e88"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c54a88eb9d8ebc4e5eefaadbe2102a4f7499f9e413654172f40aefd25350959"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8095ef159896e5999a795b0f80e4d64281301a109e442a8d29cd750ca6bd8303"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f8ca13e2476ffd6d5be4763f5868133506ddcfa5ce54b4dac231ebdc19be6c6"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34d25aa25c10f966d5415795ed271da84605044dbf436c054966cea5442451b3"}, + {file = "hiredis-3.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4180dc5f646b426e5fa1212e1348c167ee2a864b3a70d56579163d64a847dd1e"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d92144e0cd6e6e841a6ad343e9d58631626eeb4ac96b0322649379b5d4527447"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb91ba42903de637b94a1b64477f381f94ad82c0742c264f9245be76a7a3cbc"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ce71a797b5bc02c51da082428c00251ed6a7a67a03acbda5fbf9e8d028725f6"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e04c7feb9467e3170cd4d5bee381775783d81bbc45d6147c1c0ce3b50dc04f9"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a31806306a60f3565c04c964d6bee0e9d4a5120e1da589e41976b53972edf635"}, + {file = "hiredis-3.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:bc51f594c2c0863ded6501642dc96701ca8bbea9ced4fa3af0a1aeda8aa634cb"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4663a319ab7d22c597b9421e5ea384fd583e044f2f1ca9a1b98d4fef8a0fea2f"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:8060fa256862b0c3de64a73ab45bc1ccf381caca464f2647af9075b200828948"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9445b7f117a9c8c8ccad97cb44daa55ddccff3cbc9079984eac56d982ba01f"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:732cf1c5cf1324f7bf3b6086976fe62a2ca98f0bf6316f31063c2c67be8797bc"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2102a94063d878c40df92f55199637a74f535e3a0b79ceba4a00538853a21be3"}, + {file = "hiredis-3.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d968dde69e3fe903bf9ef00667669dcf04a3e096e33aaf138775106ead138bc8"}, + {file = "hiredis-3.1.0.tar.gz", hash = "sha256:51d40ac3611091020d7dea6b05ed62cb152bff595fa4f931e7b6479d777acf7c"}, ] [[package]] From e46295033819032eb3b50eb6e5a8e9adbff59a84 Mon Sep 17 00:00:00 2001 From: V02460 Date: Mon, 17 Feb 2025 15:25:42 +0100 Subject: [PATCH 147/691] Document consequences of replacing secrets (#18138) Document consequences of replacing secrets. The covered config options are `registration_shared_secret`, `macaroon_secret_key`, `form_secret` and `worker_replication_secret`. Even though I looked at the source code to check the added documentation is right, I would appreciate additional verification of the statements made. In an hand-wavy attempt at classifying how bad the consequences of secret replacement are, I added some explanations as warnings and others as regular paragraphs. Closes #17971 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18138.doc | 1 + .../configuration/config_documentation.md | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 changelog.d/18138.doc diff --git a/changelog.d/18138.doc b/changelog.d/18138.doc new file mode 100644 index 0000000000..5f07cd2763 --- /dev/null +++ b/changelog.d/18138.doc @@ -0,0 +1 @@ +Document consequences of replacing secrets. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 92ab6438c1..e3c06d5371 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2592,6 +2592,14 @@ This is primarily intended for use with the `register_new_matrix_user` script (see [Registering a user](../../setup/installation.md#registering-a-user)); however, the interface is [documented](../../admin_api/register_api.html). +Replacing an existing `registration_shared_secret` with a new one requires users +of the [Shared-Secret Registration API](../../admin_api/register_api.html) to +start using the new secret for requesting any further one-time nonces. + +> ⚠️ **Warning** – The additional consequences of replacing +> [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it delegates +> to `registration_shared_secret`. + See also [`registration_shared_secret_path`](#registration_shared_secret_path). Example configuration: @@ -3168,6 +3176,11 @@ A secret which is used to sign If none is specified, the `registration_shared_secret` is used, if one is given; otherwise, a secret key is derived from the signing key. +> ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new one +> will lead to invalidation of access tokens for all guest users. It will also +> break unsubscribe links in emails sent before the change. An unlucky user +> might encounter a broken SSO login flow and would have to start again. + Example configuration: ```yaml macaroon_secret_key: @@ -3195,6 +3208,9 @@ A secret which is used to calculate HMACs for form values, to stop falsification of values. Must be specified for the User Consent forms to work. +Replacing an existing `form_secret` with a new one might break the user consent +page for an unlucky user and require them to reopen the page from a new link. + Example configuration: ```yaml form_secret: @@ -4443,6 +4459,9 @@ HTTP requests from workers. The default, this value is omitted (equivalently `null`), which means that traffic between the workers and the main process is not authenticated. +Replacing an existing `worker_replication_secret` with a new one will break +communication with all workers that have not yet updated their secret. + Example configuration: ```yaml worker_replication_secret: "secret_secret" From 0c31783b4fa49fdf3b0e5b70f937ea0718114648 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Feb 2025 16:39:26 +0100 Subject: [PATCH 148/691] Limit size of user directory search queries (#18172) If a user search has many words we can end up creating really large queries that take a long time for the database to process. Generally, such searches don't return any results anyway (due to limits on user ID and display name length). We "fix" this by cheating and only searching for the first ten words. --- changelog.d/18172.misc | 1 + synapse/storage/databases/main/user_directory.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18172.misc diff --git a/changelog.d/18172.misc b/changelog.d/18172.misc new file mode 100644 index 0000000000..49b6be263b --- /dev/null +++ b/changelog.d/18172.misc @@ -0,0 +1 @@ +Reduce database load of user search when using large search terms. diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index a51182de55..d6cd0774a8 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -1237,7 +1237,13 @@ def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: search_term = _filter_text_for_index(search_term) escaped_words = [] - for word in _parse_words(search_term): + for index, word in enumerate(_parse_words(search_term)): + if index >= 10: + # We limit how many terms we include, as otherwise it can use + # excessive database time if people accidentally search for large + # strings. + break + # Postgres tsvector and tsquery quoting rules: # words potentially containing punctuation should be quoted # and then existing quotes and backslashes should be doubled From 12dc6b102f071eb2eb84f2cff4cf92903276ffbb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 17 Feb 2025 10:23:04 -0600 Subject: [PATCH 149/691] Add support to proxy outbound requests from Synapse in tests (#18158) Adds new environment variables that can be used with the Docker image (`SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY`) Useful for things like the [Secure Border Gateway](https://element.io/server-suite/secure-border-gateways) ### Why is this necessary? You can already configure the `HTTP_PROXY`/`HTTPS_PROXY` environment variables to proxy outbound requests but setting this globally in the Docker image affects all processes which isn't always desirable or workable in the case where the proxy is running in the Docker image itself (because the Debian packages will fail to download because the proxy isn't up and running yet) . Adding Synapse specific environment variables (`SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY`) makes things much more targetable. --- changelog.d/18158.docker | 1 + docker/README.md | 3 +++ docker/conf-workers/synapse.supervisord.conf.j2 | 3 +++ docker/configure_workers_and_start.py | 7 +++++++ synapse/http/proxyagent.py | 6 ++++++ 5 files changed, 20 insertions(+) create mode 100644 changelog.d/18158.docker diff --git a/changelog.d/18158.docker b/changelog.d/18158.docker new file mode 100644 index 0000000000..60469690ff --- /dev/null +++ b/changelog.d/18158.docker @@ -0,0 +1 @@ +Add `SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY` environment variables to pass through specifically to the Synapse process (instead of needing to apply [`http_proxy`/`https_proxy`/`no_proxy`](https://element-hq.github.io/synapse/latest/setup/forward_proxy.html) globally). diff --git a/docker/README.md b/docker/README.md index 8dba6fdb05..3438e9c441 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,6 +114,9 @@ The following environment variables are supported in `run` mode: is set via `docker run --user`, defaults to `991`, `991`. Note that this user must have permission to read the config files, and write to the data directories. * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`. +* `SYNAPSE_HTTP_PROXY`: Passed through to the Synapse process as the `http_proxy` environment variable. +* `SYNAPSE_HTTPS_PROXY`: Passed through to the Synapse process as the `https_proxy` environment variable. +* `SYNAPSE_NO_PROXY`: Passed through to the Synapse process as `no_proxy` environment variable. For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this: diff --git a/docker/conf-workers/synapse.supervisord.conf.j2 b/docker/conf-workers/synapse.supervisord.conf.j2 index 481eb4fc92..4fb11b259e 100644 --- a/docker/conf-workers/synapse.supervisord.conf.j2 +++ b/docker/conf-workers/synapse.supervisord.conf.j2 @@ -1,5 +1,6 @@ {% if use_forking_launcher %} [program:synapse_fork] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/python -m synapse.app.complement_fork_starter {{ main_config_path }} synapse.app.homeserver @@ -20,6 +21,7 @@ exitcodes=0 {% else %} [program:synapse_main] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml @@ -36,6 +38,7 @@ exitcodes=0 {% for worker in workers %} [program:synapse_{{ worker.name }}] +environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/prefix-log /usr/local/bin/python -m {{ worker.app }} --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 15d8d7b558..6d73e8feaa 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1099,6 +1099,13 @@ def main(args: List[str], environ: MutableMapping[str, str]) -> None: else: log("Could not find %s, will not use" % (jemallocpath,)) + # Empty strings are falsy in Python so this default is fine. We just can't have these + # be undefined because supervisord will complain about our + # `%(ENV_SYNAPSE_HTTP_PROXY)s` usage. + environ.setdefault("SYNAPSE_HTTP_PROXY", "") + environ.setdefault("SYNAPSE_HTTPS_PROXY", "") + environ.setdefault("SYNAPSE_NO_PROXY", "") + # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. log("Starting supervisord") diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index c91cf30fd1..fd16ee42dd 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -150,6 +150,12 @@ class ProxyAgent(_AgentBase): http_proxy = proxies["http"].encode() if "http" in proxies else None https_proxy = proxies["https"].encode() if "https" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None + logger.debug( + "Using proxy settings: http_proxy=%s, https_proxy=%s, no_proxy=%s", + http_proxy, + https_proxy, + no_proxy, + ) self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint( http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs From 2d4f28915eebb7109e8ac444d8f2d0d303d65013 Mon Sep 17 00:00:00 2001 From: qashlan Date: Tue, 18 Feb 2025 12:20:30 +0200 Subject: [PATCH 150/691] Add method to get current server time in milliseconds in ModuleApi (#18144) - Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. - Fixes #18104 Signed-off-by: Ahmed Qashlan Co-authored-by: Eric Eastwood Co-authored-by: Erik Johnston --- changelog.d/18144.feature | 1 + synapse/module_api/__init__.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/18144.feature diff --git a/changelog.d/18144.feature b/changelog.d/18144.feature new file mode 100644 index 0000000000..74a28e5a4e --- /dev/null +++ b/changelog.d/18144.feature @@ -0,0 +1 @@ +Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. \ No newline at end of file diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 2a2f821427..bf9532e891 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1846,6 +1846,10 @@ class ModuleApi: deactivation=deactivation, ) + def get_current_time_msec(self) -> int: + """Returns the current server time in milliseconds.""" + return self._clock.time_msec() + class PublicRoomListManager: """Contains methods for adding to, removing from and querying whether a room From 4aa725a7302e64a45591a7361c96b967596d16cb Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 18 Feb 2025 14:33:34 +0100 Subject: [PATCH 151/691] 1.125.0rc1 --- CHANGES.md | 50 +++++++++++++++++++++++++++++++++++++++ changelog.d/17436.doc | 1 - changelog.d/17616.misc | 1 - changelog.d/17949.feature | 1 - changelog.d/17967.misc | 1 - changelog.d/18004.feature | 1 - changelog.d/18122.doc | 1 - changelog.d/18124.misc | 1 - changelog.d/18125.bugfix | 1 - changelog.d/18134.misc | 1 - changelog.d/18135.bugfix | 1 - changelog.d/18136.misc | 1 - changelog.d/18138.doc | 1 - changelog.d/18139.misc | 1 - changelog.d/18144.feature | 1 - changelog.d/18145.bugfix | 1 - changelog.d/18158.docker | 1 - changelog.d/18159.doc | 1 - changelog.d/18172.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- 21 files changed, 57 insertions(+), 19 deletions(-) delete mode 100644 changelog.d/17436.doc delete mode 100644 changelog.d/17616.misc delete mode 100644 changelog.d/17949.feature delete mode 100644 changelog.d/17967.misc delete mode 100644 changelog.d/18004.feature delete mode 100644 changelog.d/18122.doc delete mode 100644 changelog.d/18124.misc delete mode 100644 changelog.d/18125.bugfix delete mode 100644 changelog.d/18134.misc delete mode 100644 changelog.d/18135.bugfix delete mode 100644 changelog.d/18136.misc delete mode 100644 changelog.d/18138.doc delete mode 100644 changelog.d/18139.misc delete mode 100644 changelog.d/18144.feature delete mode 100644 changelog.d/18145.bugfix delete mode 100644 changelog.d/18158.docker delete mode 100644 changelog.d/18159.doc delete mode 100644 changelog.d/18172.misc diff --git a/CHANGES.md b/CHANGES.md index 58b410decb..d83430088c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,53 @@ +# Synapse 1.125.0rc1 (2025-02-18) + +### Features + +- Add functionality to be able to use multiple values in SSO feature `attribute_requirements`. ([\#17949](https://github.com/element-hq/synapse/issues/17949)) +- Add experimental config options `admin_token_path` and `client_secret_path` for MSC 3861. ([\#18004](https://github.com/element-hq/synapse/issues/18004)) +- Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. ([\#18144](https://github.com/element-hq/synapse/issues/18144)) + +### Bugfixes + +- Fix a bug when updating a user 3pid with invalid returns 500 server error change to 400 with a message. ([\#18125](https://github.com/element-hq/synapse/issues/18125)) +- Fix user directory search when using a legacy module with a `check_username_for_spam` callback. Broke in v1.122.0. ([\#18135](https://github.com/element-hq/synapse/issues/18135)) +- Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. ([\#18145](https://github.com/element-hq/synapse/issues/18145)) + +### Updates to the Docker image + +- Add `SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY` environment variables to pass through specifically to the Synapse process (instead of needing to apply [`http_proxy`/`https_proxy`/`no_proxy`](https://element-hq.github.io/synapse/latest/setup/forward_proxy.html) globally). ([\#18158](https://github.com/element-hq/synapse/issues/18158)) + +### Improved Documentation + +- Add Oracle Linux 8 and 9 installation instructions. ([\#17436](https://github.com/element-hq/synapse/issues/17436)) +- Document missing server config options (`daemonize`, `print_pidfile`, `user_agent_suffix`, `use_frozen_dicts`, `manhole`). ([\#18122](https://github.com/element-hq/synapse/issues/18122)) +- Document consequences of replacing secrets. ([\#18138](https://github.com/element-hq/synapse/issues/18138)) +- Make `burst_count` field an integer in `rc_presence` config documentation example. ([\#18159](https://github.com/element-hq/synapse/issues/18159)) + +### Internal Changes + +- Overload DatabasePool.simple_select_one_txn to return non-None when the allow_none parameter is False. ([\#17616](https://github.com/element-hq/synapse/issues/17616)) +- Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library. ([\#17967](https://github.com/element-hq/synapse/issues/17967)) +- Add log message when worker lock timeouts get large. ([\#18124](https://github.com/element-hq/synapse/issues/18124)) +- Make it explicit that you can buy an AGPL-alternative commercial license from Element. ([\#18134](https://github.com/element-hq/synapse/issues/18134)) +- Fix the 'Fix linting' GitHub Actions workflow. ([\#18136](https://github.com/element-hq/synapse/issues/18136)) +- Do not log at the exception-level when clients provide empty `since` token to `/sync` API. ([\#18139](https://github.com/element-hq/synapse/issues/18139)) +- Reduce database load of user search when using large search terms. ([\#18172](https://github.com/element-hq/synapse/issues/18172)) + + + +### Updates to locked dependencies + +* Bump bcrypt from 4.2.0 to 4.2.1. ([\#18127](https://github.com/element-hq/synapse/issues/18127)) +* Bump bytes from 1.9.0 to 1.10.0. ([\#18149](https://github.com/element-hq/synapse/issues/18149)) +* Bump gitpython from 3.1.43 to 3.1.44. ([\#18128](https://github.com/element-hq/synapse/issues/18128)) +* Bump hiredis from 3.0.0 to 3.1.0. ([\#18169](https://github.com/element-hq/synapse/issues/18169)) +* Bump serde_json from 1.0.137 to 1.0.138. ([\#18129](https://github.com/element-hq/synapse/issues/18129)) +* Bump service-identity from 24.1.0 to 24.2.0. ([\#18171](https://github.com/element-hq/synapse/issues/18171)) +* Bump sigstore/cosign-installer from 3.7.0 to 3.8.0. ([\#18147](https://github.com/element-hq/synapse/issues/18147)) +* Bump twine from 6.0.1 to 6.1.0. ([\#18170](https://github.com/element-hq/synapse/issues/18170)) +* Bump types-pyyaml from 6.0.12.20240917 to 6.0.12.20241230. ([\#18097](https://github.com/element-hq/synapse/issues/18097)) +* Bump ulid from 1.1.4 to 1.2.0. ([\#18148](https://github.com/element-hq/synapse/issues/18148)) + # Synapse 1.124.0 (2025-02-11) No significant changes since 1.124.0rc3. diff --git a/changelog.d/17436.doc b/changelog.d/17436.doc deleted file mode 100644 index 7846d3a8bf..0000000000 --- a/changelog.d/17436.doc +++ /dev/null @@ -1 +0,0 @@ -Add Oracle Linux 8 and 9 installation instructions. diff --git a/changelog.d/17616.misc b/changelog.d/17616.misc deleted file mode 100644 index 8250832dcd..0000000000 --- a/changelog.d/17616.misc +++ /dev/null @@ -1 +0,0 @@ -Overload DatabasePool.simple_select_one_txn to return non-None when the allow_none parameter is False. diff --git a/changelog.d/17949.feature b/changelog.d/17949.feature deleted file mode 100644 index ee9bb51e03..0000000000 --- a/changelog.d/17949.feature +++ /dev/null @@ -1 +0,0 @@ -Add functionality to be able to use multiple values in SSO feature `attribute_requirements`. diff --git a/changelog.d/17967.misc b/changelog.d/17967.misc deleted file mode 100644 index 60e61a6f7d..0000000000 --- a/changelog.d/17967.misc +++ /dev/null @@ -1 +0,0 @@ -Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library. \ No newline at end of file diff --git a/changelog.d/18004.feature b/changelog.d/18004.feature deleted file mode 100644 index 8cacd1a0ef..0000000000 --- a/changelog.d/18004.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental config options `admin_token_path` and `client_secret_path` for MSC 3861. \ No newline at end of file diff --git a/changelog.d/18122.doc b/changelog.d/18122.doc deleted file mode 100644 index b7bb7107e2..0000000000 --- a/changelog.d/18122.doc +++ /dev/null @@ -1 +0,0 @@ -Document missing server config options (`daemonize`, `print_pidfile`, `user_agent_suffix`, `use_frozen_dicts`, `manhole`). diff --git a/changelog.d/18124.misc b/changelog.d/18124.misc deleted file mode 100644 index 8ac6a73a20..0000000000 --- a/changelog.d/18124.misc +++ /dev/null @@ -1 +0,0 @@ -Add log message when worker lock timeouts get large. diff --git a/changelog.d/18125.bugfix b/changelog.d/18125.bugfix deleted file mode 100644 index 0e6aa704ff..0000000000 --- a/changelog.d/18125.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug when updating a user 3pid with invalid returns 500 server error change to 400 with a message. \ No newline at end of file diff --git a/changelog.d/18134.misc b/changelog.d/18134.misc deleted file mode 100644 index be90691516..0000000000 --- a/changelog.d/18134.misc +++ /dev/null @@ -1 +0,0 @@ -Make it explicit that you can buy an AGPL-alternative commercial license from Element. diff --git a/changelog.d/18135.bugfix b/changelog.d/18135.bugfix deleted file mode 100644 index 998914a811..0000000000 --- a/changelog.d/18135.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix user directory search when using a legacy module with a `check_username_for_spam` callback. Broke in v1.122.0. diff --git a/changelog.d/18136.misc b/changelog.d/18136.misc deleted file mode 100644 index ed88c38acf..0000000000 --- a/changelog.d/18136.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the 'Fix linting' GitHub Actions workflow. diff --git a/changelog.d/18138.doc b/changelog.d/18138.doc deleted file mode 100644 index 5f07cd2763..0000000000 --- a/changelog.d/18138.doc +++ /dev/null @@ -1 +0,0 @@ -Document consequences of replacing secrets. \ No newline at end of file diff --git a/changelog.d/18139.misc b/changelog.d/18139.misc deleted file mode 100644 index f753033b8e..0000000000 --- a/changelog.d/18139.misc +++ /dev/null @@ -1 +0,0 @@ -Do not log at the exception-level when clients provide empty `since` token to `/sync` API. diff --git a/changelog.d/18144.feature b/changelog.d/18144.feature deleted file mode 100644 index 74a28e5a4e..0000000000 --- a/changelog.d/18144.feature +++ /dev/null @@ -1 +0,0 @@ -Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. \ No newline at end of file diff --git a/changelog.d/18145.bugfix b/changelog.d/18145.bugfix deleted file mode 100644 index ad9d353834..0000000000 --- a/changelog.d/18145.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. \ No newline at end of file diff --git a/changelog.d/18158.docker b/changelog.d/18158.docker deleted file mode 100644 index 60469690ff..0000000000 --- a/changelog.d/18158.docker +++ /dev/null @@ -1 +0,0 @@ -Add `SYNAPSE_HTTP_PROXY`/`SYNAPSE_HTTPS_PROXY`/`SYNAPSE_NO_PROXY` environment variables to pass through specifically to the Synapse process (instead of needing to apply [`http_proxy`/`https_proxy`/`no_proxy`](https://element-hq.github.io/synapse/latest/setup/forward_proxy.html) globally). diff --git a/changelog.d/18159.doc b/changelog.d/18159.doc deleted file mode 100644 index 031f8a149f..0000000000 --- a/changelog.d/18159.doc +++ /dev/null @@ -1 +0,0 @@ -Make `burst_count` field an integer in `rc_presence` config documentation example. \ No newline at end of file diff --git a/changelog.d/18172.misc b/changelog.d/18172.misc deleted file mode 100644 index 49b6be263b..0000000000 --- a/changelog.d/18172.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce database load of user search when using large search terms. diff --git a/debian/changelog b/debian/changelog index 3e20885c86..a0d5a7614f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.125.0~rc1) stable; urgency=medium + + * New synapse release 1.125.0rc1. + + -- Synapse Packaging team Tue, 18 Feb 2025 13:32:49 +0000 + matrix-synapse-py3 (1.124.0) stable; urgency=medium * New Synapse release 1.124.0. diff --git a/pyproject.toml b/pyproject.toml index 2d982eae70..a40be93d7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.124.0" +version = "1.125.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 30fcd586fe8200dbf3505a6ff84ec5122e91da3c Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 18 Feb 2025 14:47:13 +0100 Subject: [PATCH 152/691] Tweak changelog --- CHANGES.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d83430088c..7db366d1d5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,14 +3,13 @@ ### Features - Add functionality to be able to use multiple values in SSO feature `attribute_requirements`. ([\#17949](https://github.com/element-hq/synapse/issues/17949)) -- Add experimental config options `admin_token_path` and `client_secret_path` for MSC 3861. ([\#18004](https://github.com/element-hq/synapse/issues/18004)) +- Add experimental config options `admin_token_path` and `client_secret_path` for [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861). ([\#18004](https://github.com/element-hq/synapse/issues/18004)) - Add `get_current_time_msec()` method to the [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) for sound time comparisons with Synapse. ([\#18144](https://github.com/element-hq/synapse/issues/18144)) ### Bugfixes -- Fix a bug when updating a user 3pid with invalid returns 500 server error change to 400 with a message. ([\#18125](https://github.com/element-hq/synapse/issues/18125)) +- Update the response when a client attempts to add an invalid email address to the user's account from a 500, to a 400 with error text. ([\#18125](https://github.com/element-hq/synapse/issues/18125)) - Fix user directory search when using a legacy module with a `check_username_for_spam` callback. Broke in v1.122.0. ([\#18135](https://github.com/element-hq/synapse/issues/18135)) -- Add rate limit `rc_presence.per_user`. This prevents load from excessive presence updates sent by clients via sync api. Also rate limit `/_matrix/client/v3/presence` as per the spec. Contributed by @rda0. ([\#18145](https://github.com/element-hq/synapse/issues/18145)) ### Updates to the Docker image @@ -25,7 +24,7 @@ ### Internal Changes -- Overload DatabasePool.simple_select_one_txn to return non-None when the allow_none parameter is False. ([\#17616](https://github.com/element-hq/synapse/issues/17616)) +- Overload `DatabasePool.simple_select_one_txn` to return non-`None` when the `allow_none` parameter is `False`. ([\#17616](https://github.com/element-hq/synapse/issues/17616)) - Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library. ([\#17967](https://github.com/element-hq/synapse/issues/17967)) - Add log message when worker lock timeouts get large. ([\#18124](https://github.com/element-hq/synapse/issues/18124)) - Make it explicit that you can buy an AGPL-alternative commercial license from Element. ([\#18134](https://github.com/element-hq/synapse/issues/18134)) From ecad88f5c5787f6a8b7e72d3cb97b868fb685484 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 18 Feb 2025 14:44:38 +0000 Subject: [PATCH 153/691] Cleanup deleted state group references (#18165) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18165.bugfix | 1 + synapse/storage/databases/state/store.py | 8 ++++++ tests/storage/test_purge.py | 35 +++++++++++++++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18165.bugfix diff --git a/changelog.d/18165.bugfix b/changelog.d/18165.bugfix new file mode 100644 index 0000000000..eba5face52 --- /dev/null +++ b/changelog.d/18165.bugfix @@ -0,0 +1 @@ +Cleanup deleted state group references. diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 8c7980e719..90d7beb92f 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -828,10 +828,18 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): "DELETE FROM state_groups_state WHERE state_group = ?", [(sg,) for sg in state_groups_to_delete], ) + txn.execute_batch( + "DELETE FROM state_group_edges WHERE state_group = ?", + [(sg,) for sg in state_groups_to_delete], + ) txn.execute_batch( "DELETE FROM state_groups WHERE id = ?", [(sg,) for sg in state_groups_to_delete], ) + txn.execute_batch( + "DELETE FROM state_groups_pending_deletion WHERE state_group = ?", + [(sg,) for sg in state_groups_to_delete], + ) return True diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 5d6a8518c0..916e42e731 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -247,7 +247,7 @@ class PurgeTests(HomeserverTestCase): 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 ) - # We expect that the unreferenced state group has been deleted. + # We expect that the unreferenced state group has been deleted from all tables. row = self.get_success( self.state_store.db_pool.simple_select_one_onecol( table="state_groups", @@ -259,6 +259,39 @@ class PurgeTests(HomeserverTestCase): ) self.assertIsNone(row) + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups_state", + keyvalues={"state_group": unreferenced_state_group}, + retcol="state_group", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_group_edges", + keyvalues={"state_group": unreferenced_state_group}, + retcol="state_group", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups_pending_deletion", + keyvalues={"state_group": unreferenced_state_group}, + retcol="state_group", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + # We expect there to now only be one state group for the room, which is # the state group of the last event (as the only outlier). state_groups = self.get_success( From f7bc63ef5787f20b63586bc7d8665cd2ce76c268 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 18 Feb 2025 15:47:35 +0100 Subject: [PATCH 154/691] Make sure we advertise registration as disabled when MSC3861 is enabled (#17661) This has been a problem with Element Web, as it will proble /register with an empty body, which gave this error: ``` curl -d '{}' -HContent-Type:application/json /_matrix/client/v3/register {"errcode": "M_UNKNOWN", "error": "Invalid username"} ``` And Element Web would choke on it. This changes that so we reply instead: ``` {"errcode": "M_FORBIDDEN", "error": "Registration has been disabled. Only m.login.application_service registrations are allowed."} ``` Also adds a test for this. See https://github.com/element-hq/element-web/issues/27993 --------- Co-authored-by: Andrew Morgan --- changelog.d/17661.bugfix | 1 + synapse/rest/client/register.py | 12 +++++--- tests/handlers/test_oauth_delegation.py | 37 ++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 changelog.d/17661.bugfix diff --git a/changelog.d/17661.bugfix b/changelog.d/17661.bugfix new file mode 100644 index 0000000000..33881bbc6a --- /dev/null +++ b/changelog.d/17661.bugfix @@ -0,0 +1 @@ +Make sure we advertise registration as disabled when MSC3861 is enabled. diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index ad76f188ab..58231d2b04 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -908,6 +908,14 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): await self.ratelimiter.ratelimit(None, client_addr, update=False) + # Allow only ASes to use this API. + if body.get("type") != APP_SERVICE_REGISTRATION_TYPE: + raise SynapseError( + 403, + "Registration has been disabled. Only m.login.application_service registrations are allowed.", + errcode=Codes.FORBIDDEN, + ) + kind = parse_string(request, "kind", default="user") if kind == "guest": @@ -923,10 +931,6 @@ class RegisterAppServiceOnlyRestServlet(RestServlet): if not isinstance(desired_username, str) or len(desired_username) > 512: raise SynapseError(400, "Invalid username") - # Allow only ASes to use this API. - if body.get("type") != APP_SERVICE_REGISTRATION_TYPE: - raise SynapseError(403, "Non-application service registration type") - if not self.auth.has_access_token(request): raise SynapseError( 400, diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index ba2f8ff510..b9d66a6c52 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -43,6 +43,7 @@ from synapse.api.errors import ( OAuthInsufficientScopeError, SynapseError, ) +from synapse.appservice import ApplicationService from synapse.http.site import SynapseRequest from synapse.rest import admin from synapse.rest.client import account, devices, keys, login, logout, register @@ -575,6 +576,16 @@ class MSC3861OAuthDelegation(HomeserverTestCase): channel.json_body["errcode"], Codes.UNRECOGNIZED, channel.json_body ) + def expect_forbidden( + self, method: str, path: str, content: Union[bytes, str, JsonDict] = "" + ) -> None: + channel = self.make_request(method, path, content) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["errcode"], Codes.FORBIDDEN, channel.json_body + ) + def test_uia_endpoints(self) -> None: """Test that endpoints that were removed in MSC2964 are no longer available.""" @@ -629,11 +640,35 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_registration_endpoints_removed(self) -> None: """Test that registration endpoints that were removed in MSC2964 are no longer available.""" + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, + sender="@as_main:test", + ) + + self.hs.get_datastores().main.services_cache = [appservice] self.expect_unrecognized( "GET", "/_matrix/client/v1/register/m.login.registration_token/validity" ) + + # Registration is disabled + self.expect_forbidden( + "POST", + "/_matrix/client/v3/register", + {"username": "alice", "password": "hunter2"}, + ) + # This is still available for AS registrations - # self.expect_unrecognized("POST", "/_matrix/client/v3/register") + channel = self.make_request( + "POST", + "/_matrix/client/v3/register", + {"username": "alice", "type": "m.login.application_service"}, + shorthand=False, + access_token="i_am_an_app_service", + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.expect_unrecognized("GET", "/_matrix/client/v3/register/available") self.expect_unrecognized( "POST", "/_matrix/client/v3/register/email/requestToken" From 1525a3b4d48a0f5657d61423e1f205bff9a77948 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 19 Feb 2025 11:55:15 +0100 Subject: [PATCH 155/691] Speedup the building of Docker images (#18038) This is a split off #18033 This uses a few tricks to speed up the building of docker images: - This switches to use `uv pip install` instead of `pip install`. This saves a bunch of time, especially when cross-compiling - I then looked at what packages were not using binary wheels: I upgraded MarkupSafe to have binaries for py3.12, and got back to Python 3.12 because hiredis didn't have builds for py3.13 with the version we were using - The generation of the requirements.txt is arch-agnostic, so I've switched this one to run on the build architecture, so that both arch can share it - The download of runtime depdendencies can be done on the build architecture through manual `apt-get download` plus `dpkg --extract` - We were using -slim images, but still installed a bunch of -dev dependencies. Turns out, all the dev dependencies were already installed in the non-slim image, which saves a bunch of time as well --- changelog.d/18038.docker | 1 + docker/Dockerfile | 169 ++++++++++++++++++++------------------- poetry.lock | 115 ++++++++++++++------------ 3 files changed, 150 insertions(+), 135 deletions(-) create mode 100644 changelog.d/18038.docker diff --git a/changelog.d/18038.docker b/changelog.d/18038.docker new file mode 100644 index 0000000000..accb09be2a --- /dev/null +++ b/changelog.d/18038.docker @@ -0,0 +1 @@ +Speed up the building of the Docker image. diff --git a/docker/Dockerfile b/docker/Dockerfile index a4931011a7..1dd65f2413 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,45 +20,16 @@ # `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in # in `poetry export` in the past. +ARG DEBIAN_VERSION=bookworm ARG PYTHON_VERSION=3.12 +ARG POETRY_VERSION=1.8.3 ### ### Stage 0: generate requirements.txt ### -# We hardcode the use of Debian bookworm here because this could change upstream -# and other Dockerfiles used for testing are expecting bookworm. -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS requirements - -# RUN --mount is specific to buildkit and is documented at -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for apt (and below for pip), to improve -# rebuild speeds on slow connections. -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - build-essential curl git libffi-dev libssl-dev pkg-config \ - && rm -rf /var/lib/apt/lists/* - -# Install rust and ensure its in the PATH. -# (Rust may be needed to compile `cryptography`---which is one of poetry's -# dependencies---on platforms that don't have a `cryptography` wheel. -ENV RUSTUP_HOME=/rust -ENV CARGO_HOME=/cargo -ENV PATH=/cargo/bin:/rust/bin:$PATH -RUN mkdir /rust /cargo - -RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal - -# arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not -# set to true, so we expose it as a build-arg. -ARG CARGO_NET_GIT_FETCH_WITH_CLI=false -ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI - -# We install poetry in its own build stage to avoid its dependencies conflicting with -# synapse's dependencies. -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user "poetry==1.3.2" +### This stage is platform-agnostic, so we can use the build platform in case of cross-compilation. +### +FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS requirements WORKDIR /synapse @@ -75,41 +46,30 @@ ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION # Instead, we'll just install what a regular `pip install` would from PyPI. ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE +# This silences a warning as uv isn't able to do hardlinks between its cache +# (mounted as --mount=type=cache) and the target directory. +ENV UV_LINK_MODE=copy + # Export the dependencies, but only if we're actually going to use the Poetry lockfile. # Otherwise, just create an empty requirements file so that the Dockerfile can # proceed. -RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ +ARG POETRY_VERSION +RUN --mount=type=cache,target=/root/.cache/uv \ + if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ + uvx --with poetry-plugin-export==1.8.0 \ + poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ else \ - touch /synapse/requirements.txt; \ + touch /synapse/requirements.txt; \ fi ### ### Stage 1: builder ### -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS builder - -# install the OS build deps -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - build-essential \ - libffi-dev \ - libjpeg-dev \ - libpq-dev \ - libssl-dev \ - libwebp-dev \ - libxml++2.6-dev \ - libxslt1-dev \ - openssl \ - zlib1g-dev \ - git \ - curl \ - libicu-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS builder +# This silences a warning as uv isn't able to do hardlinks between its cache +# (mounted as --mount=type=cache) and the target directory. +ENV UV_LINK_MODE=copy # Install rust and ensure its in the PATH ENV RUSTUP_HOME=/rust @@ -119,7 +79,6 @@ RUN mkdir /rust /cargo RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal - # arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not # set to true, so we expose it as a build-arg. ARG CARGO_NET_GIT_FETCH_WITH_CLI=false @@ -131,8 +90,8 @@ ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI # # This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. COPY --from=requirements /synapse/requirements.txt /synapse/ -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --prefix="/install" --no-deps -r /synapse/requirements.txt # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ @@ -146,41 +105,85 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE # Install the synapse package itself. # If we have populated requirements.txt, we don't install any dependencies # as we should already have those from the previous `pip install` step. -RUN --mount=type=cache,target=/synapse/target,sharing=locked \ +RUN \ + --mount=type=cache,target=/root/.cache/uv \ + --mount=type=cache,target=/synapse/target,sharing=locked \ --mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \ if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \ + uv pip install --prefix="/install" --no-deps /synapse[all]; \ else \ - pip install --prefix="/install" --no-warn-script-location /synapse[all]; \ + uv pip install --prefix="/install" /synapse[all]; \ fi ### -### Stage 2: runtime +### Stage 2: runtime dependencies download for ARM64 and AMD64 +### +FROM --platform=$BUILDPLATFORM docker.io/library/debian:${DEBIAN_VERSION} AS runtime-deps + +# Tell apt to keep downloaded package files, as we're using cache mounts. +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +# Add both target architectures +RUN dpkg --add-architecture arm64 +RUN dpkg --add-architecture amd64 + +# Fetch the runtime dependencies debs for both architectures +# We do that by building a recursive list of packages we need to download with `apt-cache depends` +# and then downloading them with `apt-get download`. +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -qq && \ + apt-get install -y --no-install-recommends rsync && \ + apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends \ + curl \ + gosu \ + libjpeg62-turbo \ + libpq5 \ + libwebp7 \ + xmlsec1 \ + libjemalloc2 \ + libicu \ + | grep '^\w' > /tmp/pkg-list && \ + for arch in arm64 amd64; do \ + mkdir -p /tmp/debs-${arch} && \ + cd /tmp/debs-${arch} && \ + apt-get download $(sed "s/$/:${arch}/" /tmp/pkg-list); \ + done + +# Extract the debs for each architecture +# On the runtime image, /lib is a symlink to /usr/lib, so we need to copy the +# libraries to the right place, else the `COPY` won't work. +# On amd64, we'll also have a /lib64 folder with ld-linux-x86-64.so.2, which is +# already present in the runtime image. +RUN \ + for arch in arm64 amd64; do \ + mkdir -p /install-${arch}/var/lib/dpkg/status.d/ && \ + for deb in /tmp/debs-${arch}/*.deb; do \ + package_name=$(dpkg-deb -I ${deb} | awk '/^ Package: .*$/ {print $2}'); \ + echo "Extracting: ${package_name}"; \ + dpkg --ctrl-tarfile $deb | tar -Ox ./control > /install-${arch}/var/lib/dpkg/status.d/${package_name}; \ + dpkg --extract $deb /install-${arch}; \ + done; \ + rsync -avr /install-${arch}/lib/ /install-${arch}/usr/lib; \ + rm -rf /install-${arch}/lib /install-${arch}/lib64; \ + done + + +### +### Stage 3: runtime ### -FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm +FROM docker.io/library/python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION} + +ARG TARGETARCH LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' LABEL org.opencontainers.image.documentation='https://github.com/element-hq/synapse/blob/master/docker/README.md' LABEL org.opencontainers.image.source='https://github.com/element-hq/synapse.git' LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later' -RUN \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -qq && apt-get install -yqq \ - curl \ - gosu \ - libjpeg62-turbo \ - libpq5 \ - libwebp7 \ - xmlsec1 \ - libjemalloc2 \ - libicu72 \ - libssl-dev \ - openssl \ - && rm -rf /var/lib/apt/lists/* - +COPY --from=runtime-deps /install-${TARGETARCH} / COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py COPY ./docker/conf /conf diff --git a/poetry.lock b/poetry.lock index 0d388eff6a..427a8de85e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1155,61 +1155,72 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "markupsafe" -version = "2.1.2" +version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] From 6b4cc9f3f60fc1ee7da25c80d1aaba6d38705f3f Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 20 Feb 2025 10:40:30 -0800 Subject: [PATCH 156/691] Document suspension Admin API (#18162) Missed in the transition from experimental to stable. Fixes #18160 --- changelog.d/18162.misc | 1 + docs/admin_api/user_admin_api.md | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 changelog.d/18162.misc diff --git a/changelog.d/18162.misc b/changelog.d/18162.misc new file mode 100644 index 0000000000..4d46979641 --- /dev/null +++ b/changelog.d/18162.misc @@ -0,0 +1 @@ +Document suspension Admin API. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 2742d2d2cd..77ce800507 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -414,6 +414,32 @@ The following actions are **NOT** performed. The list may be incomplete. - Remove from monthly active users - Remove user's consent information (consent version and timestamp) +## Suspend/Unsuspend Account + +This API allows an admin to suspend/unsuspend an account. While an account is suspended, the user is +prohibited from sending invites, joining or knocking on rooms, sending messages, changing profile data, and redacting messages other than their own. + +The api is: + +``` +PUT /_synapse/admin/v1/suspend/ +``` + +with a body of: + +```json +{ + "suspend": true +} +``` + +To unsuspend a user, use the same endpoint with a body of: +```json +{ + "suspend": false +} +``` + ## Reset password **Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) From caa1f9d806945e056261ec6879da3b1a1cc23b17 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 20 Feb 2025 17:56:53 -0600 Subject: [PATCH 157/691] Add support for overriding `id_token_signing_alg_values_supported` for an OpenID identity provider (#18177) Normally, when `discovery` is enabled, `id_token_signing_alg_values_supported` comes from the OpenID Discovery Document (`/.well-known/openid-configuration`). If nothing was specified, we default to supporting `RS256` in the downstream usage. This PR just adds support for adding a default/overriding the the discovered value [just like we do for other things like the `token_endpoint`](https://github.com/element-hq/synapse/blob/1525a3b4d48a0f5657d61423e1f205bff9a77948/docs/usage/configuration/config_documentation.md#oidc_providers), etc. --- changelog.d/18177.feature | 1 + .../configuration/config_documentation.md | 18 ++++++ synapse/config/oidc.py | 35 ++++++++++ synapse/handlers/oidc.py | 5 ++ tests/handlers/test_oidc.py | 64 +++++++++++++++++-- 5 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18177.feature diff --git a/changelog.d/18177.feature b/changelog.d/18177.feature new file mode 100644 index 0000000000..71d568474b --- /dev/null +++ b/changelog.d/18177.feature @@ -0,0 +1 @@ +Add support for specifying/overriding `id_token_signing_alg_values_supported` for an OpenID identity provider. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e3c06d5371..facf60a043 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3579,6 +3579,24 @@ Options for each entry include: to `auto`, which uses PKCE if supported during metadata discovery. Set to `always` to force enable PKCE or `never` to force disable PKCE. +* `id_token_signing_alg_values_supported`: List of the JWS signing algorithms (`alg` + values) that are supported for signing the `id_token`. + + This is *not* required if `discovery` is disabled. We default to supporting `RS256` in + the downstream usage if no algorithms are configured here or in the discovery + document. + + According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid + approach would be to reject this provider as non-compliant if it's not included but we + simply allow whatever and see what happens (you're the one that configured the value + and cooperating with the identity provider). + + The `alg` value `"none"` MAY be supported but can only be used if the Authorization + Endpoint does not include `id_token` in the `response_type` (ex. + `/authorize?response_type=code` where `none` can apply, + `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when + using the Authorization Code Flow). + * `scopes`: list of scopes to request. This should normally include the "openid" scope. Defaults to `["openid"]`. diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py index d0a03baf55..fc4bc35b30 100644 --- a/synapse/config/oidc.py +++ b/synapse/config/oidc.py @@ -125,6 +125,10 @@ OIDC_PROVIDER_CONFIG_SCHEMA = { "enum": ["client_secret_basic", "client_secret_post", "none"], }, "pkce_method": {"type": "string", "enum": ["auto", "always", "never"]}, + "id_token_signing_alg_values_supported": { + "type": "array", + "items": {"type": "string"}, + }, "scopes": {"type": "array", "items": {"type": "string"}}, "authorization_endpoint": {"type": "string"}, "token_endpoint": {"type": "string"}, @@ -326,6 +330,9 @@ def _parse_oidc_config_dict( client_secret_jwt_key=client_secret_jwt_key, client_auth_method=client_auth_method, pkce_method=oidc_config.get("pkce_method", "auto"), + id_token_signing_alg_values_supported=oidc_config.get( + "id_token_signing_alg_values_supported" + ), scopes=oidc_config.get("scopes", ["openid"]), authorization_endpoint=oidc_config.get("authorization_endpoint"), token_endpoint=oidc_config.get("token_endpoint"), @@ -402,6 +409,34 @@ class OidcProviderConfig: # Valid values are 'auto', 'always', and 'never'. pkce_method: str + id_token_signing_alg_values_supported: Optional[List[str]] + """ + List of the JWS signing algorithms (`alg` values) that are supported for signing the + `id_token`. + + This is *not* required if `discovery` is disabled. We default to supporting `RS256` + in the downstream usage if no algorithms are configured here or in the discovery + document. + + According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid + approach would be to reject this provider as non-compliant if it's not included but + we can just allow whatever and see what happens (they're the ones that configured + the value and cooperating with the identity provider). It wouldn't be wise to add it + ourselves because absence of `RS256` might indicate that the provider actually + doesn't support it, despite the spec requirement. Adding it silently could lead to + failed authentication attempts or strange mismatch attacks. + + The `alg` value `"none"` MAY be supported but can only be used if the Authorization + Endpoint does not include `id_token` in the `response_type` (ex. + `/authorize?response_type=code` where `none` can apply, + `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when + using the Authorization Code Flow). + + Spec: + - https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata + - https://openid.net/specs/openid-connect-core-1_0.html#AuthorizationExamples + """ + # list of scopes to request scopes: Collection[str] diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index c9109c9e79..76b692928d 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -640,6 +640,11 @@ class OidcProvider: elif self._config.pkce_method == "never": metadata.pop("code_challenge_methods_supported", None) + if self._config.id_token_signing_alg_values_supported: + metadata["id_token_signing_alg_values_supported"] = ( + self._config.id_token_signing_alg_values_supported + ) + self._validate_metadata(metadata) return metadata diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 1b43ee43c6..5ffc5a90a8 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -70,12 +70,16 @@ DEFAULT_CONFIG = { } # extends the default config with explicit OAuth2 endpoints instead of using discovery +# +# We add "explicit" to things to make them different from the discovered values to make +# sure that the explicit values override the discovered ones. EXPLICIT_ENDPOINT_CONFIG = { **DEFAULT_CONFIG, "discover": False, - "authorization_endpoint": ISSUER + "authorize", - "token_endpoint": ISSUER + "token", - "jwks_uri": ISSUER + "jwks", + "authorization_endpoint": ISSUER + "authorize-explicit", + "token_endpoint": ISSUER + "token-explicit", + "jwks_uri": ISSUER + "jwks-explicit", + "id_token_signing_alg_values_supported": ["RS256", ""], } @@ -259,12 +263,64 @@ class OidcHandlerTestCase(HomeserverTestCase): self.get_success(self.provider.load_metadata()) self.fake_server.get_metadata_handler.assert_not_called() + @override_config({"oidc_config": {**EXPLICIT_ENDPOINT_CONFIG, "discover": True}}) + def test_discovery_with_explicit_config(self) -> None: + """ + The handler should discover the endpoints from OIDC discovery document but + values are overriden by the explicit config. + """ + # This would throw if some metadata were invalid + metadata = self.get_success(self.provider.load_metadata()) + self.fake_server.get_metadata_handler.assert_called_once() + + self.assertEqual(metadata.issuer, self.fake_server.issuer) + # It seems like authlib does not have that defined in its metadata models + self.assertEqual( + metadata.get("userinfo_endpoint"), + self.fake_server.userinfo_endpoint, + ) + + # Ensure the values are overridden correctly since these were configured + # explicitly + self.assertEqual( + metadata.authorization_endpoint, + EXPLICIT_ENDPOINT_CONFIG["authorization_endpoint"], + ) + self.assertEqual( + metadata.token_endpoint, EXPLICIT_ENDPOINT_CONFIG["token_endpoint"] + ) + self.assertEqual(metadata.jwks_uri, EXPLICIT_ENDPOINT_CONFIG["jwks_uri"]) + self.assertEqual( + metadata.id_token_signing_alg_values_supported, + EXPLICIT_ENDPOINT_CONFIG["id_token_signing_alg_values_supported"], + ) + + # subsequent calls should be cached + self.reset_mocks() + self.get_success(self.provider.load_metadata()) + self.fake_server.get_metadata_handler.assert_not_called() + @override_config({"oidc_config": EXPLICIT_ENDPOINT_CONFIG}) def test_no_discovery(self) -> None: """When discovery is disabled, it should not try to load from discovery document.""" - self.get_success(self.provider.load_metadata()) + metadata = self.get_success(self.provider.load_metadata()) self.fake_server.get_metadata_handler.assert_not_called() + # Ensure the values are overridden correctly since these were configured + # explicitly + self.assertEqual( + metadata.authorization_endpoint, + EXPLICIT_ENDPOINT_CONFIG["authorization_endpoint"], + ) + self.assertEqual( + metadata.token_endpoint, EXPLICIT_ENDPOINT_CONFIG["token_endpoint"] + ) + self.assertEqual(metadata.jwks_uri, EXPLICIT_ENDPOINT_CONFIG["jwks_uri"]) + self.assertEqual( + metadata.id_token_signing_alg_values_supported, + EXPLICIT_ENDPOINT_CONFIG["id_token_signing_alg_values_supported"], + ) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_load_jwks(self) -> None: """JWKS loading is done once (then cached) if used.""" From 8fd7148e6ab3c1346dbf6715bf60c50b0d3fa3b8 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 21 Feb 2025 02:06:44 -0800 Subject: [PATCH 158/691] Prevent suspended users from sending encrypted messages (#18157) Missed in the first round. --- changelog.d/18157.bugfix | 1 + synapse/handlers/message.py | 32 ++++++++-- tests/rest/client/test_rooms.py | 102 +++++++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 changelog.d/18157.bugfix diff --git a/changelog.d/18157.bugfix b/changelog.d/18157.bugfix new file mode 100644 index 0000000000..307e9c96ff --- /dev/null +++ b/changelog.d/18157.bugfix @@ -0,0 +1 @@ +Prevent suspended users from sending encrypted messages. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index df3010ecf6..4642b8b578 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -644,11 +644,33 @@ class EventCreationHandler: """ await self.auth_blocking.check_auth_blocking(requester=requester) - if event_dict["type"] == EventTypes.Message: - requester_suspended = await self.store.get_user_suspended_status( - requester.user.to_string() - ) - if requester_suspended: + requester_suspended = await self.store.get_user_suspended_status( + requester.user.to_string() + ) + if requester_suspended: + # We want to allow suspended users to perform "corrective" actions + # asked of them by server admins, such as redact their messages and + # leave rooms. + if event_dict["type"] in ["m.room.redaction", "m.room.member"]: + if event_dict["type"] == "m.room.redaction": + event = await self.store.get_event( + event_dict["content"]["redacts"], allow_none=True + ) + if event: + if event.sender != requester.user.to_string(): + raise SynapseError( + 403, + "You can only redact your own events while account is suspended.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + if event_dict["type"] == "m.room.member": + if event_dict["content"]["membership"] != "leave": + raise SynapseError( + 403, + "Changing membership while account is suspended is not allowed.", + Codes.USER_ACCOUNT_SUSPENDED, + ) + else: raise SynapseError( 403, "Sending messages while account is suspended is not allowed.", diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 604b585150..f6ec8a8b7b 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -1371,6 +1371,23 @@ class RoomJoinTestCase(RoomBase): ) self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + def test_suspended_user_can_leave_room(self) -> None: + channel = self.make_request( + "POST", f"/join/{self.room1}", access_token=self.tok1 + ) + self.assertEqual(channel.code, 200) + + # set the user as suspended + self.get_success(self.store.set_user_suspended_status(self.user1, True)) + + # leave room + channel = self.make_request( + "POST", + f"/rooms/{self.room1}/leave", + access_token=self.tok1, + ) + self.assertEqual(channel.code, 200) + class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): servlets = [ @@ -3989,10 +4006,25 @@ class UserSuspensionTests(unittest.HomeserverTestCase): self.user2 = self.register_user("teresa", "hackme") self.tok2 = self.login("teresa", "hackme") - self.room1 = self.helper.create_room_as(room_creator=self.user1, tok=self.tok1) + self.admin = self.register_user("admin", "pass", True) + self.admin_tok = self.login("admin", "pass") + + self.room1 = self.helper.create_room_as( + room_creator=self.user1, tok=self.tok1, room_version="11" + ) self.store = hs.get_datastores().main - def test_suspended_user_cannot_send_message_to_room(self) -> None: + self.room2 = self.helper.create_room_as( + room_creator=self.user1, is_public=False, tok=self.tok1 + ) + self.helper.send_state( + self.room2, + EventTypes.RoomEncryption, + {EventContentFields.ENCRYPTION_ALGORITHM: "m.megolm.v1.aes-sha2"}, + tok=self.tok1, + ) + + def test_suspended_user_cannot_send_message_to_public_room(self) -> None: # set the user as suspended self.get_success(self.store.set_user_suspended_status(self.user1, True)) @@ -4004,6 +4036,24 @@ class UserSuspensionTests(unittest.HomeserverTestCase): ) self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + def test_suspended_user_cannot_send_message_to_encrypted_room(self) -> None: + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/suspend/{self.user1}", + {"suspend": True}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {f"user_{self.user1}_suspended": True}) + + channel = self.make_request( + "PUT", + f"/rooms/{self.room2}/send/m.room.encrypted/1", + access_token=self.tok1, + content={}, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + def test_suspended_user_cannot_change_profile_data(self) -> None: # set the user as suspended self.get_success(self.store.set_user_suspended_status(self.user1, True)) @@ -4069,3 +4119,51 @@ class UserSuspensionTests(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(channel.code, 200) + + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/rooms/{self.room1}/send/m.room.redaction/3456346", + access_token=self.tok1, + content={"reason": "bogus", "redacts": event_id}, + shorthand=False, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/rooms/{self.room1}/send/m.room.redaction/3456346", + access_token=self.tok1, + content={"reason": "bogus", "redacts": event_id2}, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + def test_suspended_user_cannot_ban_others(self) -> None: + # user to ban joins room user1 created + self.make_request("POST", f"/rooms/{self.room1}/join", access_token=self.tok2) + + # suspend user1 + self.get_success(self.store.set_user_suspended_status(self.user1, True)) + + # user1 tries to ban other user while suspended + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/ban", + access_token=self.tok1, + content={"reason": "spite", "user_id": self.user2}, + shorthand=False, + ) + self.assertEqual(channel.json_body["errcode"], "M_USER_SUSPENDED") + + # un-suspend user1 + self.get_success(self.store.set_user_suspended_status(self.user1, False)) + + # ban now goes through + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{self.room1}/ban", + access_token=self.tok1, + content={"reason": "spite", "user_id": self.user2}, + shorthand=False, + ) + self.assertEqual(channel.code, 200) From b2a187f49b390b9c5afa56cca03b629fe0f169c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 24 Feb 2025 13:06:16 +0100 Subject: [PATCH 159/691] Disable room list publication by default (#18175) This is in line with our general policy of ensuring that the default config is reasonably locked down. SyTest PR to fix tests: https://github.com/matrix-org/sytest/pull/1396 --- changelog.d/18175.misc | 1 + .../conf/workers-shared-extra.yaml.j2 | 5 +++++ docs/upgrade.md | 20 +++++++++++++++++++ .../configuration/config_documentation.md | 8 ++++++-- synapse/config/room_directory.py | 4 +--- tests/handlers/test_directory.py | 1 + tests/handlers/test_room_list.py | 6 ++++++ tests/rest/admin/test_room.py | 3 +++ tests/rest/client/test_rooms.py | 6 ++++++ 9 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18175.misc diff --git a/changelog.d/18175.misc b/changelog.d/18175.misc new file mode 100644 index 0000000000..c12a30b7c1 --- /dev/null +++ b/changelog.d/18175.misc @@ -0,0 +1 @@ +Disable room list publication by default. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 797d58e9b3..ac0c4bb851 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -139,4 +139,9 @@ caches: sync_response_cache_duration: 0 +# Complement assumes that it can publish to the room list by default. +room_list_publication_rules: + - action: allow + + {% include "shared-orig.yaml.j2" %} diff --git a/docs/upgrade.md b/docs/upgrade.md index 6c96cb91a3..7e4cd52e1d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,26 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.126.0 + +## Room list publication rules change + +The default [`room_list_publication_rules`] setting was changed to disallow +anyone (except server admins) from publishing to the room list by default. + +This is in line with Synapse policy of locking down features by default that can +be abused without moderation. + +To keep the previous behavior of allowing publication by default, add the +following to the config: + +```yaml +room_list_publication_rules: + - "action": "allow" +``` + +[`room_list_publication_rules`]: usage/configuration/config_documentation.md#room_list_publication_rules + # Upgrading to v1.122.0 ## Dropping support for PostgreSQL 11 and 12 diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index facf60a043..8d9a71fb5f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4245,8 +4245,8 @@ unwanted entries from being published in the public room list. The format of this option is the same as that for [`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more -rules. By default, no list is provided, meaning that all rooms may be -published to the room list. +rules. By default, no list is provided, meaning that no one may publish to the +room list (except server admins). Otherwise, requests to publish a room are matched against each rule in order. The first rule that matches decides if the request is allowed or denied. If no @@ -4272,6 +4272,10 @@ Note that the patterns match against fully qualified IDs, e.g. against of `alice`, `room` and `abcedgghijk`. +_Changed in Synapse 1.126.0: The default was changed to deny publishing to the +room list by default_ + + Example configuration: ```yaml diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 704895cf9a..f0349b68f2 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -54,9 +54,7 @@ class RoomDirectoryConfig(Config): for rule in room_list_publication_rules ] else: - self._room_list_publication_rules = [ - _RoomDirectoryRule("room_list_publication_rules", {"action": "allow"}) - ] + self._room_list_publication_rules = [] def is_alias_creation_allowed(self, user_id: str, room_id: str, alias: str) -> bool: """Checks if the given user is allowed to create the given alias diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 4a3e36ffde..b7058d8002 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -587,6 +587,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): self.room_list_handler = hs.get_room_list_handler() self.directory_handler = hs.get_directory_handler() + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_disabling_room_list(self) -> None: self.room_list_handler.enable_room_list_search = True self.directory_handler.enable_room_list_search = True diff --git a/tests/handlers/test_room_list.py b/tests/handlers/test_room_list.py index 4d22ef98c2..45cef09b22 100644 --- a/tests/handlers/test_room_list.py +++ b/tests/handlers/test_room_list.py @@ -6,6 +6,7 @@ from synapse.rest.client import directory, login, room from synapse.types import JsonDict from tests import unittest +from tests.utils import default_config class RoomListHandlerTestCase(unittest.HomeserverTestCase): @@ -30,6 +31,11 @@ class RoomListHandlerTestCase(unittest.HomeserverTestCase): assert channel.code == HTTPStatus.OK, f"couldn't publish room: {channel.result}" return room_id + def default_config(self) -> JsonDict: + config = default_config("test") + config["room_list_publication_rules"] = [{"action": "allow"}] + return config + def test_acls_applied_to_room_directory_results(self) -> None: """ Creates 3 rooms. Room 2 has an ACL that only permits the homeservers diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 1817d67a00..1d44106bd7 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -1282,6 +1282,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_list_rooms(self) -> None: """Test that we can list rooms""" # Create 3 test rooms @@ -1795,6 +1796,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertEqual(room_id, channel.json_body["rooms"][0].get("room_id")) self.assertEqual("ж", channel.json_body["rooms"][0].get("name")) + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_filter_public_rooms(self) -> None: self.helper.create_room_as( self.admin_user, tok=self.admin_user_tok, is_public=True @@ -1872,6 +1874,7 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertEqual(1, response.json_body["total_rooms"]) self.assertEqual(1, len(response.json_body["rooms"])) + @unittest.override_config({"room_list_publication_rules": [{"action": "allow"}]}) def test_single_room(self) -> None: """Test that a single room can be requested correctly""" # Create two test rooms diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index f6ec8a8b7b..a7108b905a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -67,6 +67,7 @@ from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase from tests.test_utils.event_injection import create_event from tests.unittest import override_config +from tests.utils import default_config PATH_PREFIX = b"/_matrix/client/api/v1" @@ -2565,6 +2566,11 @@ class PublicRoomsRoomTypeFilterTestCase(unittest.HomeserverTestCase): tok=self.token, ) + def default_config(self) -> JsonDict: + config = default_config("test") + config["room_list_publication_rules"] = [{"action": "allow"}] + return config + def make_public_rooms_request( self, room_types: Optional[List[Union[str, None]]], From 805e8705c7e918377191a18ec442872937ddd172 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 12:27:56 +0000 Subject: [PATCH 160/691] Bump sigstore/cosign-installer from 3.8.0 to 3.8.1 (#18185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sigstore/cosign-installer](https://github.com/sigstore/cosign-installer) from 3.8.0 to 3.8.1.
Release notes

Sourced from sigstore/cosign-installer's releases.

v3.8.1

What's Changed

Full Changelog: https://github.com/sigstore/cosign-installer/compare/v3...v3.8.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sigstore/cosign-installer&package-manager=github_actions&previous-version=3.8.0&new-version=3.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b439cb0915..e8319364dd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@v3.8.0 + uses: sigstore/cosign-installer@v3.8.1 - name: Checkout repository uses: actions/checkout@v4 From a5c3fe6c1e3b341cab9a1b2b9927a4b89bf1492b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 12:56:04 +0000 Subject: [PATCH 161/691] Bump types-psycopg2 from 2.9.21.20241019 to 2.9.21.20250121 (#18188) Bumps [types-psycopg2](https://github.com/python/typeshed) from 2.9.21.20241019 to 2.9.21.20250121.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-psycopg2&package-manager=pip&previous-version=2.9.21.20241019&new-version=2.9.21.20250121)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 427a8de85e..189fa9063d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2832,13 +2832,13 @@ files = [ [[package]] name = "types-psycopg2" -version = "2.9.21.20241019" +version = "2.9.21.20250121" description = "Typing stubs for psycopg2" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "types-psycopg2-2.9.21.20241019.tar.gz", hash = "sha256:bca89b988d2ebd19bcd08b177d22a877ea8b841decb10ed130afcf39404612fa"}, - {file = "types_psycopg2-2.9.21.20241019-py3-none-any.whl", hash = "sha256:44d091e67732d16a941baae48cd7b53bf91911bc36888652447cf1ef0c1fb3f6"}, + {file = "types_psycopg2-2.9.21.20250121-py3-none-any.whl", hash = "sha256:b890dc6f5a08b6433f0ff73a4ec9a834deedad3e914f2a4a6fd43df021f745f1"}, + {file = "types_psycopg2-2.9.21.20250121.tar.gz", hash = "sha256:2b0e2cd0f3747af1ae25a7027898716d80209604770ef3cbf350fe055b9c349b"}, ] [[package]] From b9276e21eee569c244ef40cab0285fc067dbf702 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 25 Feb 2025 11:34:33 +0100 Subject: [PATCH 162/691] Fix MSC4108 'rendez-vous' responses with some reverse proxy in the front of Synapse (#18178) MSC4108 relies on ETag to determine if something has changed on the rendez-vous channel. Strong and correct ETag comparison works if the response body is bit-for-bit identical, which isn't the case if a proxy in the middle compresses the response on the fly. This adds a `no-transform` directive to the `Cache-Control` header, which tells proxies not to transform the response body. Additionally, some proxies (nginx) will switch to `Transfer-Encoding: chunked` if it doesn't know the Content-Length of the response, and 'weakening' the ETag if that's the case. I've added `Content-Length` headers to all responses, to hopefully solve that. This basically fixes QR-code login when nginx or cloudflare is involved, with gzip/zstd/deflate compression enabled. --- changelog.d/18178.bugfix | 1 + rust/src/rendezvous/mod.rs | 6 +++++- tests/rest/client/test_rendezvous.py | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18178.bugfix diff --git a/changelog.d/18178.bugfix b/changelog.d/18178.bugfix new file mode 100644 index 0000000000..d91327803b --- /dev/null +++ b/changelog.d/18178.bugfix @@ -0,0 +1 @@ +Fix MSC4108 QR-code login not working with some reverse-proxy setups. diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index 23de668102..3148e0f67a 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -47,7 +47,7 @@ fn prepare_headers(headers: &mut HeaderMap, session: &Session) { headers.typed_insert(AccessControlAllowOrigin::ANY); headers.typed_insert(AccessControlExposeHeaders::from_iter([ETAG])); headers.typed_insert(Pragma::no_cache()); - headers.typed_insert(CacheControl::new().with_no_store()); + headers.typed_insert(CacheControl::new().with_no_store().with_no_transform()); headers.typed_insert(session.etag()); headers.typed_insert(session.expires()); headers.typed_insert(session.last_modified()); @@ -192,10 +192,12 @@ impl RendezvousHandler { "url": uri, }) .to_string(); + let length = response.len() as _; let mut response = Response::new(response.as_bytes()); *response.status_mut() = StatusCode::CREATED; response.headers_mut().typed_insert(ContentType::json()); + response.headers_mut().typed_insert(ContentLength(length)); prepare_headers(response.headers_mut(), &session); http_response_to_twisted(twisted_request, response)?; @@ -299,6 +301,7 @@ impl RendezvousHandler { // proxy/cache setup which strips the ETag header if there is no Content-Type set. // Specifically, we noticed this behaviour when placing Synapse behind Cloudflare. response.headers_mut().typed_insert(ContentType::text()); + response.headers_mut().typed_insert(ContentLength(0)); http_response_to_twisted(twisted_request, response)?; @@ -316,6 +319,7 @@ impl RendezvousHandler { response .headers_mut() .typed_insert(AccessControlAllowOrigin::ANY); + response.headers_mut().typed_insert(ContentLength(0)); http_response_to_twisted(twisted_request, response)?; Ok(()) diff --git a/tests/rest/client/test_rendezvous.py b/tests/rest/client/test_rendezvous.py index ab701680a6..83a5cbdc15 100644 --- a/tests/rest/client/test_rendezvous.py +++ b/tests/rest/client/test_rendezvous.py @@ -117,10 +117,11 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase): headers = dict(channel.headers.getAllRawHeaders()) self.assertIn(b"ETag", headers) self.assertIn(b"Expires", headers) + self.assertIn(b"Content-Length", headers) self.assertEqual(headers[b"Content-Type"], [b"application/json"]) self.assertEqual(headers[b"Access-Control-Allow-Origin"], [b"*"]) self.assertEqual(headers[b"Access-Control-Expose-Headers"], [b"etag"]) - self.assertEqual(headers[b"Cache-Control"], [b"no-store"]) + self.assertEqual(headers[b"Cache-Control"], [b"no-store, no-transform"]) self.assertEqual(headers[b"Pragma"], [b"no-cache"]) self.assertIn("url", channel.json_body) self.assertTrue(channel.json_body["url"].startswith("https://")) @@ -141,9 +142,10 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase): self.assertEqual(headers[b"ETag"], [etag]) self.assertIn(b"Expires", headers) self.assertEqual(headers[b"Content-Type"], [b"text/plain"]) + self.assertEqual(headers[b"Content-Length"], [b"7"]) self.assertEqual(headers[b"Access-Control-Allow-Origin"], [b"*"]) self.assertEqual(headers[b"Access-Control-Expose-Headers"], [b"etag"]) - self.assertEqual(headers[b"Cache-Control"], [b"no-store"]) + self.assertEqual(headers[b"Cache-Control"], [b"no-store, no-transform"]) self.assertEqual(headers[b"Pragma"], [b"no-cache"]) self.assertEqual(channel.text_body, "foo=bar") From 1246e54d7fbd12cb32bd6fc6f3531517d35bcb45 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 25 Feb 2025 08:10:32 -0700 Subject: [PATCH 163/691] 1.125.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7db366d1d5..df32bc59be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.125.0 (2025-02-25) + +No significant changes since 1.125.0rc1. + + + + # Synapse 1.125.0rc1 (2025-02-18) ### Features diff --git a/debian/changelog b/debian/changelog index a0d5a7614f..1ac1fc6644 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.125.0) stable; urgency=medium + + * New Synapse release 1.125.0. + + -- Synapse Packaging team Tue, 25 Feb 2025 08:10:07 -0700 + matrix-synapse-py3 (1.125.0~rc1) stable; urgency=medium * New synapse release 1.125.0rc1. diff --git a/pyproject.toml b/pyproject.toml index a40be93d7b..abacfa328d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.125.0rc1" +version = "1.125.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 5121f9210c989fcc909e78195133876dff3bc9b9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 25 Feb 2025 16:25:39 +0000 Subject: [PATCH 164/691] Add background job to clear unreferenced state groups (#18154) Fixes #18150 ### Pull Request Checklist * [X] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Erik Johnston --- changelog.d/18154.feature | 1 + docs/development/database_schema.md | 2 +- synapse/_scripts/synapse_port_db.py | 30 +++ synapse/storage/controllers/purge_events.py | 246 +++++++++++++----- synapse/storage/databases/state/bg_updates.py | 10 +- synapse/storage/databases/state/deletion.py | 44 +++- synapse/storage/schema/__init__.py | 1 + .../02_delete_unreferenced_state_groups.sql | 16 ++ synapse/types/storage/__init__.py | 4 + tests/storage/test_purge.py | 97 +++++++ 10 files changed, 376 insertions(+), 75 deletions(-) create mode 100644 changelog.d/18154.feature create mode 100644 synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql diff --git a/changelog.d/18154.feature b/changelog.d/18154.feature new file mode 100644 index 0000000000..62e1b79a15 --- /dev/null +++ b/changelog.d/18154.feature @@ -0,0 +1 @@ +Add background job to clear unreferenced state groups. diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 37a06acc12..620d1c16b0 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -162,7 +162,7 @@ by a unique name, the current status (stored in JSON), and some dependency infor * Whether the update requires a previous update to be complete. * A rough ordering for which to complete updates. -A new background updates needs to be added to the `background_updates` table: +A new background update needs to be added to the `background_updates` table: ```sql INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 3f67a739a0..59065a0504 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -191,6 +191,11 @@ APPEND_ONLY_TABLES = [ IGNORED_TABLES = { + # Porting the auto generated sequence in this table is non-trivial. + # None of the entries in this list are mandatory for Synapse to keep working. + # If state group disk space is an issue after the port, the + # `delete_unreferenced_state_groups_bg_update` background task can be run again. + "state_groups_pending_deletion", # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", @@ -216,6 +221,15 @@ IGNORED_TABLES = { } +# These background updates will not be applied upon creation of the postgres database. +IGNORED_BACKGROUND_UPDATES = { + # Reapplying this background update to the postgres database is unnecessary after + # already having waited for the SQLite database to complete all running background + # updates. + "delete_unreferenced_state_groups_bg_update", +} + + # Error returned by the run function. Used at the top-level part of the script to # handle errors and return codes. end_error: Optional[str] = None @@ -687,6 +701,20 @@ class Porter: # 0 means off. 1 means full. 2 means incremental. return autovacuum_setting != 0 + async def remove_ignored_background_updates_from_database(self) -> None: + def _remove_delete_unreferenced_state_groups_bg_updates( + txn: LoggingTransaction, + ) -> None: + txn.execute( + "DELETE FROM background_updates WHERE update_name = ANY(?)", + (list(IGNORED_BACKGROUND_UPDATES),), + ) + + await self.postgres_store.db_pool.runInteraction( + "remove_delete_unreferenced_state_groups_bg_updates", + _remove_delete_unreferenced_state_groups_bg_updates, + ) + async def run(self) -> None: """Ports the SQLite database to a PostgreSQL database. @@ -732,6 +760,8 @@ class Porter: self.hs_config.database.get_single_database() ) + await self.remove_ignored_background_updates_from_database() + await self.run_background_updates_on_postgres() self.progress.set_state("Creating port tables") diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 47cec8c469..ef30bf2895 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -21,11 +21,18 @@ import itertools import logging -from typing import TYPE_CHECKING, Collection, Mapping, Set +from typing import ( + TYPE_CHECKING, + Collection, + Mapping, + Set, +) from synapse.logging.context import nested_logging_context from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases +from synapse.types.storage import _BackgroundUpdates if TYPE_CHECKING: from synapse.server import HomeServer @@ -44,6 +51,11 @@ class PurgeEventsStorageController: self._delete_state_groups_loop, 60 * 1000 ) + self.stores.state.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, + self._background_delete_unrefereneced_state_groups, + ) + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room""" @@ -80,68 +92,6 @@ class PurgeEventsStorageController: sg_to_delete ) - async def _find_unreferenced_groups( - self, state_groups: Collection[int] - ) -> Set[int]: - """Used when purging history to figure out which state groups can be - deleted. - - Args: - state_groups: Set of state groups referenced by events - that are going to be deleted. - - Returns: - The set of state groups that can be deleted. - """ - # Set of events that we have found to be referenced by events - referenced_groups = set() - - # Set of state groups we've already seen - state_groups_seen = set(state_groups) - - # Set of state groups to handle next. - next_to_search = set(state_groups) - while next_to_search: - # We bound size of groups we're looking up at once, to stop the - # SQL query getting too big - if len(next_to_search) < 100: - current_search = next_to_search - next_to_search = set() - else: - current_search = set(itertools.islice(next_to_search, 100)) - next_to_search -= current_search - - referenced = await self.stores.main.get_referenced_state_groups( - current_search - ) - referenced_groups |= referenced - - # We don't continue iterating up the state group graphs for state - # groups that are referenced. - current_search -= referenced - - edges = await self.stores.state.get_previous_state_groups(current_search) - - prevs = set(edges.values()) - # We don't bother re-handling groups we've already seen - prevs -= state_groups_seen - next_to_search |= prevs - state_groups_seen |= prevs - - # We also check to see if anything referencing the state groups are - # also unreferenced. This helps ensure that we delete unreferenced - # state groups, if we don't then we will de-delta them when we - # delete the other state groups leading to increased DB usage. - next_edges = await self.stores.state.get_next_state_groups(current_search) - nexts = set(next_edges.keys()) - nexts -= state_groups_seen - next_to_search |= nexts - state_groups_seen |= nexts - - to_delete = state_groups_seen - referenced_groups - - return to_delete - @wrap_as_background_process("_delete_state_groups_loop") async def _delete_state_groups_loop(self) -> None: """Background task that deletes any state groups that may be pending @@ -203,3 +153,173 @@ class PurgeEventsStorageController: room_id, groups_to_sequences, ) + + async def _background_delete_unrefereneced_state_groups( + self, progress: dict, batch_size: int + ) -> int: + """This background update will slowly delete any unreferenced state groups""" + + last_checked_state_group = progress.get("last_checked_state_group") + max_state_group = progress.get("max_state_group") + + if last_checked_state_group is None or max_state_group is None: + # This is the first run. + last_checked_state_group = 0 + + max_state_group = await self.stores.state.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={}, + retcol="MAX(id)", + allow_none=True, + desc="get_max_state_group", + ) + if max_state_group is None: + # There are no state groups so the background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE + ) + return batch_size + + ( + last_checked_state_group, + final_batch, + ) = await self._delete_unreferenced_state_groups_batch( + last_checked_state_group, batch_size, max_state_group + ) + + if not final_batch: + # There are more state groups to check. + progress = { + "last_checked_state_group": last_checked_state_group, + "max_state_group": max_state_group, + } + await self.stores.state.db_pool.updates._background_update_progress( + _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, + progress, + ) + else: + # This background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE + ) + + return batch_size + + async def _delete_unreferenced_state_groups_batch( + self, + last_checked_state_group: int, + batch_size: int, + max_state_group: int, + ) -> tuple[int, bool]: + """Looks for unreferenced state groups starting from the last state group + checked, and any state groups which would become unreferenced if a state group + was deleted, and marks them for deletion. + + Args: + last_checked_state_group: The last state group that was checked. + batch_size: How many state groups to process in this iteration. + + Returns: + (last_checked_state_group, final_batch) + """ + + # Look for state groups that can be cleaned up. + def get_next_state_groups_txn(txn: LoggingTransaction) -> Set[int]: + state_group_sql = "SELECT id FROM state_groups WHERE ? < id AND id <= ? ORDER BY id LIMIT ?" + txn.execute( + state_group_sql, (last_checked_state_group, max_state_group, batch_size) + ) + + next_set = {row[0] for row in txn} + + return next_set + + next_set = await self.stores.state.db_pool.runInteraction( + "get_next_state_groups", get_next_state_groups_txn + ) + + final_batch = False + if len(next_set) < batch_size: + final_batch = True + else: + last_checked_state_group = max(next_set) + + if len(next_set) == 0: + return last_checked_state_group, final_batch + + # Find all state groups that can be deleted if the original set is deleted. + # This set includes the original set, as well as any state groups that would + # become unreferenced upon deleting the original set. + to_delete = await self._find_unreferenced_groups(next_set) + + if len(to_delete) == 0: + return last_checked_state_group, final_batch + + await self.stores.state_deletion.mark_state_groups_as_pending_deletion( + to_delete + ) + + return last_checked_state_group, final_batch + + async def _find_unreferenced_groups( + self, + state_groups: Collection[int], + ) -> Set[int]: + """Used when purging history to figure out which state groups can be + deleted. + + Args: + state_groups: Set of state groups referenced by events + that are going to be deleted. + + Returns: + The set of state groups that can be deleted. + """ + # Set of events that we have found to be referenced by events + referenced_groups = set() + + # Set of state groups we've already seen + state_groups_seen = set(state_groups) + + # Set of state groups to handle next. + next_to_search = set(state_groups) + while next_to_search: + # We bound size of groups we're looking up at once, to stop the + # SQL query getting too big + if len(next_to_search) < 100: + current_search = next_to_search + next_to_search = set() + else: + current_search = set(itertools.islice(next_to_search, 100)) + next_to_search -= current_search + + referenced = await self.stores.main.get_referenced_state_groups( + current_search + ) + referenced_groups |= referenced + + # We don't continue iterating up the state group graphs for state + # groups that are referenced. + current_search -= referenced + + edges = await self.stores.state.get_previous_state_groups(current_search) + + prevs = set(edges.values()) + # We don't bother re-handling groups we've already seen + prevs -= state_groups_seen + next_to_search |= prevs + state_groups_seen |= prevs + + # We also check to see if anything referencing the state groups are + # also unreferenced. This helps ensure that we delete unreferenced + # state groups, if we don't then we will de-delta them when we + # delete the other state groups leading to increased DB usage. + next_edges = await self.stores.state.get_next_state_groups(current_search) + nexts = set(next_edges.keys()) + nexts -= state_groups_seen + next_to_search |= nexts + state_groups_seen |= nexts + + to_delete = state_groups_seen - referenced_groups + + return to_delete diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index f7824cba0f..95fd0ae73a 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -20,7 +20,15 @@ # import logging -from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Dict, + List, + Mapping, + Optional, + Tuple, + Union, +) from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index d4b1c20a45..f77c46f6ae 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -321,18 +321,42 @@ class StateDeletionDataStore: async def mark_state_groups_as_pending_deletion( self, state_groups: Collection[int] ) -> None: - """Mark the given state groups as pending deletion""" + """Mark the given state groups as pending deletion. + + If any of the state groups are already pending deletion, then those records are + left as is. + """ + + await self.db_pool.runInteraction( + "mark_state_groups_as_pending_deletion", + self._mark_state_groups_as_pending_deletion_txn, + state_groups, + ) + + def _mark_state_groups_as_pending_deletion_txn( + self, + txn: LoggingTransaction, + state_groups: Collection[int], + ) -> None: + sql = """ + INSERT INTO state_groups_pending_deletion (state_group, insertion_ts) + VALUES %s + ON CONFLICT (state_group) + DO NOTHING + """ now = self._clock.time_msec() - - await self.db_pool.simple_upsert_many( - table="state_groups_pending_deletion", - key_names=("state_group",), - key_values=[(state_group,) for state_group in state_groups], - value_names=("insertion_ts",), - value_values=[(now,) for _ in state_groups], - desc="mark_state_groups_as_pending_deletion", - ) + rows = [ + ( + state_group, + now, + ) + for state_group in state_groups + ] + if isinstance(txn.database_engine, PostgresEngine): + txn.execute_values(sql % ("?",), rows, fetch=False) + else: + txn.execute_batch(sql % ("(?, ?)",), rows) async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None: """Mark the given state groups as now being referenced""" diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 49e648a92f..c90c2c6051 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -158,6 +158,7 @@ Changes in SCHEMA_VERSION = 88 Changes in SCHEMA_VERSION = 89 - Add `state_groups_pending_deletion` and `state_groups_persisting` tables. + - Add background update to delete unreferenced state groups. """ diff --git a/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql b/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql new file mode 100644 index 0000000000..184dc8564c --- /dev/null +++ b/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a background update to delete any unreferenced state groups +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8902, 'delete_unreferenced_state_groups_bg_update', '{}'); diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index b5fa20a41a..d0a85ef208 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -48,3 +48,7 @@ class _BackgroundUpdates: SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = ( "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update" ) + + DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE = ( + "delete_unreferenced_state_groups_bg_update" + ) diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 916e42e731..ecdc893405 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -24,6 +24,7 @@ from synapse.api.errors import NotFoundError, SynapseError from synapse.rest.client import room from synapse.server import HomeServer from synapse.types.state import StateFilter +from synapse.types.storage import _BackgroundUpdates from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -303,3 +304,99 @@ class PurgeTests(HomeserverTestCase): ) ) self.assertEqual(len(state_groups), 1) + + def test_clear_unreferenced_state_groups(self) -> None: + """Test that any unreferenced state groups are automatically cleaned up.""" + + self.helper.send(self.room_id, body="test1") + state1 = self.helper.send_state( + self.room_id, "org.matrix.test", body={"number": 2} + ) + # Create enough state events to require multiple batches of + # delete_unreferenced_state_groups_bg_update to be run. + for i in range(200): + self.helper.send_state(self.room_id, "org.matrix.test", body={"number": i}) + state2 = self.helper.send_state( + self.room_id, "org.matrix.test", body={"number": 3} + ) + self.helper.send(self.room_id, body="test4") + last = self.helper.send(self.room_id, body="test5") + + # Create an unreferenced state group that has a prev group of one of the + # to-be-purged events. + prev_group = self.get_success( + self.store._get_state_group_for_event(state1["event_id"]) + ) + unreferenced_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=prev_group, + delta_ids={("org.matrix.test", ""): state2["event_id"]}, + current_state_ids=None, + ) + ) + + another_unreferenced_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=unreferenced_state_group, + delta_ids={("org.matrix.test", ""): state2["event_id"]}, + current_state_ids=None, + ) + ) + + # Insert and run the background update. + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Advance so that the background job to delete the state groups runs + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We expect that the unreferenced state group has been deleted. + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": unreferenced_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + # We expect that the other unreferenced state group has also been deleted. + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": another_unreferenced_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + # We expect there to now only be one state group for the room, which is + # the state group of the last event (as the only outlier). + state_groups = self.get_success( + self.state_store.db_pool.simple_select_onecol( + table="state_groups", + keyvalues={"room_id": self.room_id}, + retcol="id", + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertEqual(len(state_groups), 207) From 2159b3852e79103c791eda159c8895c3b5cf2ff5 Mon Sep 17 00:00:00 2001 From: V02460 Date: Tue, 25 Feb 2025 17:26:01 +0100 Subject: [PATCH 165/691] Add --no-secrets-in-config command line option (#18092) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the `--no-secrets-in-config` command line option that makes Synapse reject all configurations containing keys with in-line secret values. Currently this rejects - `turn_shared_secret` - `registration_shared_secret` - `macaroon_secret_key` - `recaptcha_private_key` - `recaptcha_public_key` - `experimental_features.msc3861.client_secret` - `experimental_features.msc3861.jwk` - `experimental_features.msc3861.admin_token` - `form_secret` - `redis.password` - `worker_replication_secret` > [!TIP] > Hey, you! Yes, you! 😊 If you think this list is missing an item, please leave a comment below. Thanks :) This PR complements my other PRs[^1] that add the corresponding `_path` variants for this class of config options. It enables admins to enforce a policy of no secrets in configuration files and guards against accident and malice. Because I consider the flag `--no-secrets-in-config` to be security-relevant, I did not add a corresponding `--secrets-in-config` flag; this way, if Synapse command line options are appended at various places, there is no way to weaken the once-set setting with a succeeding flag. [^1]: [#17690](https://github.com/element-hq/synapse/pull/17690), [#17717](https://github.com/element-hq/synapse/pull/17717), [#17983](https://github.com/element-hq/synapse/pull/17983), [#17984](https://github.com/element-hq/synapse/pull/17984), [#18004](https://github.com/element-hq/synapse/pull/18004), [#18090](https://github.com/element-hq/synapse/pull/18090) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18092.feature | 1 + synapse/config/_base.py | 32 +++++++++- synapse/config/_base.pyi | 6 +- synapse/config/captcha.py | 14 ++++- synapse/config/experimental.py | 30 +++++++++- synapse/config/key.py | 16 ++++- synapse/config/redis.py | 9 ++- synapse/config/registration.py | 9 ++- synapse/config/voip.py | 9 ++- synapse/config/workers.py | 9 ++- tests/config/test_load.py | 104 +++++++++++++++++++++++++++++++++ tests/config/test_workers.py | 2 +- 12 files changed, 227 insertions(+), 14 deletions(-) create mode 100644 changelog.d/18092.feature diff --git a/changelog.d/18092.feature b/changelog.d/18092.feature new file mode 100644 index 0000000000..26371cc810 --- /dev/null +++ b/changelog.d/18092.feature @@ -0,0 +1 @@ +Add the `--no-secrets-in-config` command line option. \ No newline at end of file diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 912346a423..132ba26af9 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -589,6 +589,14 @@ class RootConfig: " Defaults to the directory containing the last config file", ) + config_parser.add_argument( + "--no-secrets-in-config", + dest="secrets_in_config", + action="store_false", + default=True, + help="Reject config options that expect an in-line secret as value.", + ) + cls.invoke_all_static("add_arguments", config_parser) @classmethod @@ -626,7 +634,10 @@ class RootConfig: config_dict = read_config_files(config_files) obj.parse_config_dict( - config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path + config_dict, + config_dir_path=config_dir_path, + data_dir_path=data_dir_path, + allow_secrets_in_config=config_args.secrets_in_config, ) obj.invoke_all("read_arguments", config_args) @@ -653,6 +664,13 @@ class RootConfig: help="Specify config file. Can be given multiple times and" " may specify directories containing *.yaml files.", ) + parser.add_argument( + "--no-secrets-in-config", + dest="secrets_in_config", + action="store_false", + default=True, + help="Reject config options that expect an in-line secret as value.", + ) # we nest the mutually-exclusive group inside another group so that the help # text shows them in their own group. @@ -821,14 +839,21 @@ class RootConfig: return None obj.parse_config_dict( - config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path + config_dict, + config_dir_path=config_dir_path, + data_dir_path=data_dir_path, + allow_secrets_in_config=config_args.secrets_in_config, ) obj.invoke_all("read_arguments", config_args) return obj def parse_config_dict( - self, config_dict: Dict[str, Any], config_dir_path: str, data_dir_path: str + self, + config_dict: Dict[str, Any], + config_dir_path: str, + data_dir_path: str, + allow_secrets_in_config: bool = True, ) -> None: """Read the information from the config dict into this Config object. @@ -846,6 +871,7 @@ class RootConfig: config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path, + allow_secrets_in_config=allow_secrets_in_config, ) def generate_missing_files( diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index d9cb0da38b..55b0e2cbf4 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -132,7 +132,11 @@ class RootConfig: @classmethod def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: Any) -> None: ... def parse_config_dict( - self, config_dict: Dict[str, Any], config_dir_path: str, data_dir_path: str + self, + config_dict: Dict[str, Any], + config_dir_path: str, + data_dir_path: str, + allow_secrets_in_config: bool = ..., ) -> None: ... def generate_config( self, diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 84897c09c5..57d67abbc3 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -29,8 +29,15 @@ from ._base import Config, ConfigError class CaptchaConfig(Config): section = "captcha" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: recaptcha_private_key = config.get("recaptcha_private_key") + if recaptcha_private_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("recaptcha_private_key",), + ) if recaptcha_private_key is not None and not isinstance( recaptcha_private_key, str ): @@ -38,6 +45,11 @@ class CaptchaConfig(Config): self.recaptcha_private_key = recaptcha_private_key recaptcha_public_key = config.get("recaptcha_public_key") + if recaptcha_public_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("recaptcha_public_key",), + ) if recaptcha_public_key is not None and not isinstance( recaptcha_public_key, str ): diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 3beaeb8869..0a963b121a 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -250,7 +250,9 @@ class MSC3861: ) return self._admin_token - def check_config_conflicts(self, root: RootConfig) -> None: + def check_config_conflicts( + self, root: RootConfig, allow_secrets_in_config: bool + ) -> None: """Checks for any configuration conflicts with other parts of Synapse. Raises: @@ -260,6 +262,24 @@ class MSC3861: if not self.enabled: return + if self._client_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "client_secret"), + ) + + if self.jwk and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "jwk"), + ) + + if self._admin_token and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("experimental", "msc3861", "admin_token"), + ) + if ( root.auth.password_enabled_for_reauth or root.auth.password_enabled_for_login @@ -350,7 +370,9 @@ class ExperimentalConfig(Config): section = "experimental" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: experimental = config.get("experimental_features") or {} # MSC3026 (busy presence state) @@ -494,7 +516,9 @@ class ExperimentalConfig(Config): ) from exc # Check that none of the other config options conflict with MSC3861 when enabled - self.msc3861.check_config_conflicts(self.root) + self.msc3861.check_config_conflicts( + self.root, allow_secrets_in_config=allow_secrets_in_config + ) self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False diff --git a/synapse/config/key.py b/synapse/config/key.py index 01aae09c13..6f99f39e81 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -112,7 +112,11 @@ class KeyConfig(Config): section = "key" def read_config( - self, config: JsonDict, config_dir_path: str, **kwargs: Any + self, + config: JsonDict, + config_dir_path: str, + allow_secrets_in_config: bool, + **kwargs: Any, ) -> None: # the signing key can be specified inline or in a separate file if "signing_key" in config: @@ -172,6 +176,11 @@ class KeyConfig(Config): ) macaroon_secret_key = config.get("macaroon_secret_key") + if macaroon_secret_key and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("macaroon_secret_key",), + ) macaroon_secret_key_path = config.get("macaroon_secret_key_path") if macaroon_secret_key_path: if macaroon_secret_key: @@ -193,6 +202,11 @@ class KeyConfig(Config): # a secret which is used to calculate HMACs for form values, to stop # falsification of values self.form_secret = config.get("form_secret", None) + if self.form_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("form_secret",), + ) def generate_config_section( self, diff --git a/synapse/config/redis.py b/synapse/config/redis.py index 3f38fa11b0..948c95eef7 100644 --- a/synapse/config/redis.py +++ b/synapse/config/redis.py @@ -34,7 +34,9 @@ These are mutually incompatible. class RedisConfig(Config): section = "redis" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: redis_config = config.get("redis") or {} self.redis_enabled = redis_config.get("enabled", False) @@ -48,6 +50,11 @@ class RedisConfig(Config): self.redis_path = redis_config.get("path", None) self.redis_dbid = redis_config.get("dbid", None) self.redis_password = redis_config.get("password") + if self.redis_password and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("redis", "password"), + ) redis_password_path = redis_config.get("password_path") if redis_password_path: if self.redis_password: diff --git a/synapse/config/registration.py b/synapse/config/registration.py index c7f3e6d35e..3cf7031656 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -43,7 +43,9 @@ You have configured both `registration_shared_secret` and class RegistrationConfig(Config): section = "registration" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.enable_registration = strtobool( str(config.get("enable_registration", False)) ) @@ -68,6 +70,11 @@ class RegistrationConfig(Config): # read the shared secret, either inline or from an external file self.registration_shared_secret = config.get("registration_shared_secret") + if self.registration_shared_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("registration_shared_secret",), + ) registration_shared_secret_path = config.get("registration_shared_secret_path") if registration_shared_secret_path: if self.registration_shared_secret: diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 8614a41dd4..f33602d975 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -34,9 +34,16 @@ These are mutually incompatible. class VoipConfig(Config): section = "voip" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.turn_uris = config.get("turn_uris", []) self.turn_shared_secret = config.get("turn_shared_secret") + if self.turn_shared_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("turn_shared_secret",), + ) turn_shared_secret_path = config.get("turn_shared_secret_path") if turn_shared_secret_path: if self.turn_shared_secret: diff --git a/synapse/config/workers.py b/synapse/config/workers.py index ab896be307..632cef46ce 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -218,7 +218,9 @@ class WorkerConfig(Config): section = "worker" - def read_config(self, config: JsonDict, **kwargs: Any) -> None: + def read_config( + self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any + ) -> None: self.worker_app = config.get("worker_app") # Canonicalise worker_app so that master always has None @@ -243,6 +245,11 @@ class WorkerConfig(Config): # The shared secret used for authentication when connecting to the main synapse. self.worker_replication_secret = config.get("worker_replication_secret", None) + if self.worker_replication_secret and not allow_secrets_in_config: + raise ConfigError( + "Config options that expect an in-line secret as value are disabled", + ("worker_replication_secret",), + ) self.worker_name = config.get("worker_name", self.worker_app) self.instance_name = self.worker_name or MAIN_PROCESS_INSTANCE_NAME diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 220ca23aa7..18fb2e0c2c 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -21,6 +21,7 @@ # import tempfile from typing import Callable +from unittest import mock import yaml from parameterized import parameterized @@ -31,6 +32,11 @@ from synapse.config.homeserver import HomeServerConfig from tests.config.utils import ConfigFileTestCase +try: + import authlib +except ImportError: + authlib = None + try: import hiredis except ImportError: @@ -189,3 +195,101 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): config = HomeServerConfig.load_config("", ["-c", self.config_file]) self.assertEqual(get_secret(config), b"53C237") + + @parameterized.expand( + [ + "turn_shared_secret: 53C237", + "registration_shared_secret: 53C237", + "macaroon_secret_key: 53C237", + "recaptcha_private_key: 53C237", + "recaptcha_public_key: ¬53C237", + "form_secret: 53C237", + "worker_replication_secret: 53C237", + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " client_secret: 53C237" + ] + * (authlib is not None), + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " client_auth_method: private_key_jwt\n" + ' jwk: {{"mock": "mock"}}' + ] + * (authlib is not None), + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + " admin_token: 53C237\n" + " client_secret_path: {secret_file}" + ] + * (authlib is not None), + *["redis:\n enabled: true\n password: 53C237"] * (hiredis is not None), + ] + ) + def test_no_secrets_in_config(self, config_line: str) -> None: + if authlib is not None: + patcher = mock.patch("authlib.jose.rfc7517.JsonWebKey.import_key") + self.addCleanup(patcher.stop) + patcher.start() + + with tempfile.NamedTemporaryFile(buffering=0) as secret_file: + # Only used for less mocking with admin_token + secret_file.write(b"53C237") + + self.generate_config_and_remove_lines_containing( + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] + ) + # Check strict mode with no offenders. + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) + self.add_lines_to_config( + ["", config_line.format(secret_file=secret_file.name)] + ) + # Check strict mode with a single offender. + with self.assertRaises(ConfigError): + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) + + # Check lenient mode with a single offender. + HomeServerConfig.load_config("", ["-c", self.config_file]) + + def test_no_secrets_in_config_but_in_files(self) -> None: + with tempfile.NamedTemporaryFile(buffering=0) as secret_file: + secret_file.write(b"53C237") + + self.generate_config_and_remove_lines_containing( + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] + ) + self.add_lines_to_config( + [ + "", + f"turn_shared_secret_path: {secret_file.name}", + f"registration_shared_secret_path: {secret_file.name}", + f"macaroon_secret_key_path: {secret_file.name}", + f"recaptcha_private_key_path: {secret_file.name}", + f"recaptcha_public_key_path: {secret_file.name}", + f"form_secret_path: {secret_file.name}", + f"worker_replication_secret_path: {secret_file.name}", + *[ + "experimental_features:\n" + " msc3861:\n" + " enabled: true\n" + f" admin_token_path: {secret_file.name}\n" + f" client_secret_path: {secret_file.name}\n" + # f" jwk_path: {secret_file.name}" + ] + * (authlib is not None), + *[f"redis:\n enabled: true\n password_path: {secret_file.name}"] + * (hiredis is not None), + ] + ) + HomeServerConfig.load_config( + "", ["-c", self.config_file, "--no-secrets-in-config"] + ) diff --git a/tests/config/test_workers.py b/tests/config/test_workers.py index 64c0285d01..3a21975b89 100644 --- a/tests/config/test_workers.py +++ b/tests/config/test_workers.py @@ -47,7 +47,7 @@ class WorkerDutyConfigTestCase(TestCase): "worker_app": worker_app, **extras, } - worker_config.read_config(worker_config_dict) + worker_config.read_config(worker_config_dict, allow_secrets_in_config=True) return worker_config def test_old_configs_master(self) -> None: From d901dff9e0168e885650e89fd55f90cfc84e9a90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:07:33 +0000 Subject: [PATCH 166/691] Bump serde_json from 1.0.138 to 1.0.139 (#18186) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.138 to 1.0.139.
Release notes

Sourced from serde_json's releases.

v1.0.139

  • Documentation improvements
Commits
  • 4d4f53c Release 1.0.139
  • 5d6b32f Merge pull request #1242 from dtolnay/writefloat
  • e5bb8bd Document behavior of write_f32/f64 on non-finite floats
  • 7a79781 Merge pull request #1241 from dtolnay/doclink
  • 13591f1 Convert html links to intra-doc links
  • 1d7378e Unset doc-scrape-examples for lib target
  • 1174c5f Resolve unnecessary_semicolon pedantic clippy lint
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_json&package-manager=cargo&previous-version=1.0.138&new-version=1.0.139)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8831f7e6fd..0ed9c7bee1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,9 +457,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", From 5cf9f762a8ff431136b3969bc70101efaa8b696a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:08:01 +0000 Subject: [PATCH 167/691] Bump log from 0.4.25 to 0.4.26 (#18184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [log](https://github.com/rust-lang/log) from 0.4.25 to 0.4.26.
Release notes

Sourced from log's releases.

0.4.26

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.25...0.4.26

Changelog

Sourced from log's changelog.

[0.4.26] - 2025-02-18

What's Changed

Full Changelog: https://github.com/rust-lang/log/compare/0.4.25...0.4.26

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=log&package-manager=cargo&previous-version=0.4.25&new-version=0.4.26)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ed9c7bee1..3308e9e370 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,9 +223,9 @@ checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "log" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "memchr" From 2a6b9e9cbcc12945a21b839e157599a477e2748b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:12:27 +0000 Subject: [PATCH 168/691] Bump authlib from 1.4.0 to 1.4.1 (#18190) Bumps [authlib](https://github.com/lepture/authlib) from 1.4.0 to 1.4.1.
Release notes

Sourced from authlib's releases.

Version 1.4.1

  • Improve garbage collection on OAuth clients. #698
  • Fix client parameters for httpx. #694
Changelog

Sourced from authlib's changelog.

Version 1.4.1

Released on Jan 28, 2025

  • Improve garbage collection on OAuth clients. :issue:698
  • Fix client parameters for httpx. :issue:694
Commits
  • 0e8f480 chore: release 1.4.1
  • c46e939 fix(client): improve garbage collection for oauth clients
  • 9188e21 fix(httpx): remove compact code for httpx
  • c7e2d9f fix(httpx): update test cases for httpx
  • ce1405d fix: improve garbage collection via #698
  • 532cce6 fix: update httpx client kwargs #694
  • fe12a57 chore: update readme
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=authlib&package-manager=pip&previous-version=1.4.0&new-version=1.4.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 189fa9063d..62da2e2f76 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,13 +32,13 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.4.0" +version = "1.4.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" files = [ - {file = "Authlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"}, - {file = "authlib-1.4.0.tar.gz", hash = "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2"}, + {file = "Authlib-1.4.1-py2.py3-none-any.whl", hash = "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"}, + {file = "authlib-1.4.1.tar.gz", hash = "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d"}, ] [package.dependencies] From f155eaa05fcf19fab01231932b1641ebc4607425 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:14:56 +0000 Subject: [PATCH 169/691] Bump click from 8.1.7 to 8.1.8 (#18189) Bumps [click](https://github.com/pallets/click) from 8.1.7 to 8.1.8.
Release notes

Sourced from click's releases.

8.1.8

This is the Click 8.1.8 fix release, which fixes bugs but does not otherwise change behavior and should not result in breaking changes compared to the latest feature release.

PyPI: https://pypi.org/project/click/8.1.8/ Changes: https://click.palletsprojects.com/en/stable/changes/#version-8-1-8 Milestone https://github.com/pallets/click/milestones/23?closed=1

  • Fix an issue with type hints for click.open_file(). #2717
  • Fix issue where error message for invalid click.Path displays on multiple lines. #2697
  • Fixed issue that prevented a default value of "" from being displayed in the help for an option. #2500
  • The test runner handles stripping color consistently on Windows. #2705
  • Show correct value for flag default when using default_map. #2632
  • Fix click.echo(color=...) passing color to coloroma so it can be forced on Windows. #2606.
  • More robust bash version check, fixing problem on Windows with git-bash. #2638
  • Cache the help option generated by the help_option_names setting to respect its eagerness. #2811
  • Replace uses of os.system with subprocess.Popen. #1476
  • Exceptions generated during a command will use the context's color setting when being displayed. #2193
  • Error message when defining option with invalid name is more descriptive. #2452
  • Refactor code generating default --help option to deduplicate code. #2563
  • Test CLIRunner resets patched _compat.should_strip_ansi. #2732
Changelog

Sourced from click's changelog.

Version 8.1.8

Unreleased

  • Fix an issue with type hints for click.open_file(). :issue:2717
  • Fix issue where error message for invalid click.Path displays on multiple lines. :issue:2697
  • Fixed issue that prevented a default value of "" from being displayed in the help for an option. :issue:2500
  • The test runner handles stripping color consistently on Windows. :issue:2705
  • Show correct value for flag default when using default_map. :issue:2632
  • Fix click.echo(color=...) passing color to coloroma so it can be forced on Windows. :issue:2606.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=click&package-manager=pip&previous-version=8.1.7&new-version=8.1.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 62da2e2f76..1ad631199a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -304,13 +304,13 @@ files = [ [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] From 5e1d8f657da8c13791a3e3e4648b375b6ed02558 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:15:41 +0000 Subject: [PATCH 170/691] Bump anyhow from 1.0.95 to 1.0.96 (#18187) Bumps [anyhow](https://github.com/dtolnay/anyhow) from 1.0.95 to 1.0.96.
Release notes

Sourced from anyhow's releases.

1.0.96

  • Documentation improvements
Commits
  • f0aa0d3 Release 1.0.96
  • bc33c24 Convert html links to intra-doc links
  • 1cff785 Unset doc-scrape-examples for lib target
  • d71c806 More precise gitignore patterns
  • 3e40975 Remove **/*.rs.bk from project-specific gitignore
  • b880dd0 Ignore Cargo-generated tests/crate/target directory
  • 8891ce3 Merge pull request #404 from dtolnay/missingabi
  • 51a173e Ignore missing_abi lint in nightly-2025-01-16
  • 4d71a84 Ignore double_ended_iterator_last clippy lint
  • af0937e Update ui test suite to nightly-2025-01-02
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=anyhow&package-manager=cargo&previous-version=1.0.95&new-version=1.0.96)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3308e9e370..56d8654479 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "arc-swap" From 0fa7ffd76f16498fc35b066e49933e9019ca9253 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 25 Feb 2025 18:18:15 +0100 Subject: [PATCH 171/691] Move User Event Redaction Admin API version indicator to the correct place (#18152) Previously it was in the middle of the parameter definitions. --- changelog.d/18152.doc | 1 + docs/admin_api/user_admin_api.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18152.doc diff --git a/changelog.d/18152.doc b/changelog.d/18152.doc new file mode 100644 index 0000000000..5a02f26751 --- /dev/null +++ b/changelog.d/18152.doc @@ -0,0 +1 @@ +Move incorrectly placed version indicator in User Event Redaction Admin API docs. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 77ce800507..875876081f 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -1494,13 +1494,13 @@ The following JSON body parameter must be provided: - `rooms` - A list of rooms to redact the user's events in. If an empty list is provided all events in all rooms the user is a member of will be redacted -_Added in Synapse 1.116.0._ - The following JSON body parameters are optional: - `reason` - Reason the redaction is being requested, ie "spam", "abuse", etc. This will be included in each redaction event, and be visible to users. - `limit` - a limit on the number of the user's events to search for ones that can be redacted (events are redacted newest to oldest) in each room, defaults to 1000 if not provided +_Added in Synapse 1.116.0._ + ## Check the status of a redaction process From fbb21b29bb3e7942c4cd051d3db7316a668cdaef Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 25 Feb 2025 12:22:01 -0500 Subject: [PATCH 172/691] Define delayed event ratelimit category (#18019) Apply ratelimiting on delayed event management separately from messages. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18019.feature | 1 + demo/start.sh | 3 + .../conf/workers-shared-extra.yaml.j2 | 4 + .../configuration/config_documentation.md | 23 +++ synapse/config/ratelimiting.py | 6 + synapse/handlers/delayed_events.py | 32 +++- tests/rest/client/test_delayed_events.py | 143 ++++++++++++++++++ tests/rest/client/test_rooms.py | 35 +++++ 8 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18019.feature diff --git a/changelog.d/18019.feature b/changelog.d/18019.feature new file mode 100644 index 0000000000..74e22df74a --- /dev/null +++ b/changelog.d/18019.feature @@ -0,0 +1 @@ +Define ratelimit configuration for delayed event management. diff --git a/demo/start.sh b/demo/start.sh index 7636c41f1f..e010302bf4 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -142,6 +142,9 @@ for port in 8080 8081 8082; do per_user: per_second: 1000 burst_count: 1000 + rc_delayed_event_mgmt: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index ac0c4bb851..9ab8fedcae 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -94,6 +94,10 @@ rc_presence: per_second: 9999 burst_count: 9999 +rc_delayed_event_mgmt: + per_second: 9999 + burst_count: 9999 + federation_rr_transactions_per_room_per_second: 9999 allow_device_name_lookup_over_federation: true diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 8d9a71fb5f..8523c5f65f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1947,6 +1947,29 @@ rc_presence: burst_count: 1 ``` --- +### `rc_delayed_event_mgmt` + +Ratelimiting settings for delayed event management. + +This is a ratelimiting option that ratelimits +attempts to restart, cancel, or view delayed events +based on the sending client's account and device ID. +It defaults to: `per_second: 1`, `burst_count: 5`. + +Attempts to create or send delayed events are ratelimited not by this setting, but by `rc_message`. + +Setting this to a high value allows clients to make delayed event management requests often +(such as repeatedly restarting a delayed event with a short timeout, +or restarting several different delayed events all at once) +without the risk of being ratelimited. + +Example configuration: +```yaml +rc_delayed_event_mgmt: + per_second: 2 + burst_count: 20 +``` +--- ### `federation_rr_transactions_per_room_per_second` Sets outgoing federation transaction frequency for sending read-receipts, diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 06af4da3c5..eb1dc2dacb 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -234,3 +234,9 @@ class RatelimitConfig(Config): "rc_presence.per_user", defaults={"per_second": 0.1, "burst_count": 1}, ) + + self.rc_delayed_event_mgmt = RatelimitSettings.parse( + config, + "rc_delayed_event_mgmt", + defaults={"per_second": 1, "burst_count": 5}, + ) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 3c88a96fd3..b3f40809a1 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -19,6 +19,7 @@ from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes from synapse.api.errors import ShadowBanError +from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.logging.opentracing import set_tag from synapse.metrics import event_processing_positions @@ -57,10 +58,19 @@ class DelayedEventsHandler: self._storage_controllers = hs.get_storage_controllers() self._config = hs.config self._clock = hs.get_clock() - self._request_ratelimiter = hs.get_request_ratelimiter() self._event_creation_handler = hs.get_event_creation_handler() self._room_member_handler = hs.get_room_member_handler() + self._request_ratelimiter = hs.get_request_ratelimiter() + + # Ratelimiter for management of existing delayed events, + # keyed by the sending user ID & device ID. + self._delayed_event_mgmt_ratelimiter = Ratelimiter( + store=self._store, + clock=self._clock, + cfg=self._config.ratelimiting.rc_delayed_event_mgmt, + ) + self._next_delayed_event_call: Optional[IDelayedCall] = None # The current position in the current_state_delta stream @@ -227,6 +237,9 @@ class DelayedEventsHandler: Raises: SynapseError: if the delayed event fails validation checks. """ + # Use standard request limiter for scheduling new delayed events. + # TODO: Instead apply ratelimiting based on the scheduled send time. + # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) self._event_creation_handler.validator.validate_builder( @@ -285,7 +298,10 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) await self._initialized_from_db next_send_ts = await self._store.cancel_delayed_event( @@ -308,7 +324,10 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) await self._initialized_from_db next_send_ts = await self._store.restart_delayed_event( @@ -332,6 +351,8 @@ class DelayedEventsHandler: NotFoundError: if no matching delayed event could be found. """ assert self._is_master + # Use standard request limiter for sending delayed events on-demand, + # as an on-demand send is similar to sending a regular event. await self._request_ratelimiter.ratelimit(requester) await self._initialized_from_db @@ -415,7 +436,10 @@ class DelayedEventsHandler: async def get_all_for_user(self, requester: Requester) -> List[JsonDict]: """Return all pending delayed events requested by the given user.""" - await self._request_ratelimiter.ratelimit(requester) + await self._delayed_event_mgmt_ratelimiter.ratelimit( + requester, + (requester.user.to_string(), requester.device_id), + ) return await self._store.get_all_delayed_events_for_user( requester.user.localpart ) diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 1793b38c4a..2c938390c8 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -109,6 +109,27 @@ class DelayedEventsTestCase(HomeserverTestCase): ) self.assertEqual(setter_expected, content.get(setter_key), content) + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_get_delayed_events_ratelimit(self) -> None: + args = ("GET", PATH_PREFIX) + + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_update_delayed_event_without_id(self) -> None: channel = self.make_request( "POST", @@ -206,6 +227,46 @@ class DelayedEventsTestCase(HomeserverTestCase): expect_code=HTTPStatus.NOT_FOUND, ) + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_cancel_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "cancel"}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "cancel"}, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_send_delayed_state_event(self) -> None: state_key = "to_send_on_request" @@ -250,6 +311,44 @@ class DelayedEventsTestCase(HomeserverTestCase): ) self.assertEqual(setter_expected, content.get(setter_key), content) + @unittest.override_config({"rc_message": {"per_second": 3.5, "burst_count": 4}}) + def test_send_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "send"}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "send"}, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_restart_delayed_state_event(self) -> None: state_key = "to_send_on_restarted_timeout" @@ -309,6 +408,46 @@ class DelayedEventsTestCase(HomeserverTestCase): ) self.assertEqual(setter_expected, content.get(setter_key), content) + @unittest.override_config( + {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} + ) + def test_restart_delayed_event_ratelimit(self) -> None: + delay_ids = [] + for _ in range(2): + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), + {}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = channel.json_body.get("delay_id") + self.assertIsNotNone(delay_id) + delay_ids.append(delay_id) + + channel = self.make_request( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "restart"}, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + args = ( + "POST", + f"{PATH_PREFIX}/{delay_ids.pop(0)}", + {"action": "restart"}, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_delayed_state_events_are_cancelled_by_more_recent_state(self) -> None: state_key = "to_be_cancelled" @@ -374,3 +513,7 @@ def _get_path_for_delayed_state( room_id: str, event_type: str, state_key: str, delay_ms: int ) -> str: return f"rooms/{room_id}/state/{event_type}/{state_key}?org.matrix.msc4140.delay={delay_ms}" + + +def _get_path_for_delayed_send(room_id: str, event_type: str, delay_ms: int) -> str: + return f"rooms/{room_id}/send/{event_type}?org.matrix.msc4140.delay={delay_ms}" diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index a7108b905a..dd8350ddd1 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2399,6 +2399,41 @@ class RoomDelayedEventTestCase(RoomBase): ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "rc_message": {"per_second": 1, "burst_count": 2}, + } + ) + def test_add_delayed_event_ratelimit(self) -> None: + """Test that requests to schedule new delayed events are ratelimited by a RateLimiter, + which ratelimits them correctly, including by not limiting when the requester is + exempt from ratelimiting. + """ + + # Test that new delayed events are correctly ratelimited. + args = ( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + # Test that the new delayed events aren't ratelimited anymore. + channel = self.make_request(*args) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + class RoomSearchTestCase(unittest.HomeserverTestCase): servlets = [ From 5219a9a21490c6300870179519323b5e5f844d50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:55:23 +0000 Subject: [PATCH 173/691] Bump serde from 1.0.217 to 1.0.218 (#18183) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.217 to 1.0.218.
Release notes

Sourced from serde's releases.

v1.0.218

  • Documentation improvements
Commits
  • 7bfd518 Release 1.0.218
  • 723a949 Merge pull request #2895 from dtolnay/stabledoc
  • 2b44efb Point standard library links to stable
  • 03dc0fc Merge pull request #2894 from dtolnay/doclink
  • 85cb0c4 Convert html links to intra-doc links
  • abe7194 Update ui test suite to nightly-2025-02-12
  • aaccac7 Unset doc-scrape-examples for lib target
  • 7cd4d84 Update ui test suite to nightly-2025-02-07
  • 04ff3e8 More precise gitignore patterns
  • dc3031b Remove *.sw[po] from gitignore
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde&package-manager=cargo&previous-version=1.0.217&new-version=1.0.218)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56d8654479..845ff0ae9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -437,18 +437,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", From c4e5a582fbb06391d721ce625d7f9817064f26f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 18:03:26 +0000 Subject: [PATCH 174/691] Bump pyo3-log from 0.12.0 to 0.12.1 (#18046) Bumps [pyo3-log](https://github.com/vorner/pyo3-log) from 0.12.0 to 0.12.1.
Changelog

Sourced from pyo3-log's changelog.

0.12.1

  • Pass-through exceptions (#57)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pyo3-log&package-manager=cargo&previous-version=0.12.0&new-version=0.12.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Devon Hudson Co-authored-by: Devon Hudson --- Cargo.lock | 4 ++-- changelog.d/18046.misc | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18046.misc diff --git a/Cargo.lock b/Cargo.lock index 845ff0ae9b..b9aa1c8a6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb421dc86d38d08e04b927b02424db480be71b777fa3a56f32e2f2a3a1a3b08" +checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" dependencies = [ "arc-swap", "log", diff --git a/changelog.d/18046.misc b/changelog.d/18046.misc new file mode 100644 index 0000000000..8acbe6a3db --- /dev/null +++ b/changelog.d/18046.misc @@ -0,0 +1 @@ +Bump pyo3-log from 0.12.0 to 0.12.1. From 131607ee51bd5e922fe4aacd167557bd20659a82 Mon Sep 17 00:00:00 2001 From: V02460 Date: Tue, 25 Feb 2025 22:35:06 +0100 Subject: [PATCH 175/691] Add form_secret_path config option (#18090) I [was told](https://github.com/element-hq/synapse/pull/17983#issuecomment-2593370897) about another config option with a secret, so I got `form_secret` a companion: `form_secret_path` This PR makes NixOS and Kubernetes users a little bit happy. Includes docs and tests. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18090.feature | 1 + docs/usage/configuration/config_documentation.md | 16 ++++++++++++++++ synapse/config/key.py | 16 ++++++++++++++-- tests/config/test_load.py | 7 ++++++- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18090.feature diff --git a/changelog.d/18090.feature b/changelog.d/18090.feature new file mode 100644 index 0000000000..343e2f45eb --- /dev/null +++ b/changelog.d/18090.feature @@ -0,0 +1 @@ +Add `form_secret_path` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 8523c5f65f..f5a5aa2eb4 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3238,6 +3238,22 @@ Example configuration: ```yaml form_secret: ``` +--- +### `form_secret_path` + +An alternative to [`form_secret`](#form_secret): +allows the secret to be specified in an external file. + +The file should be a plain text file, containing only the secret. +Synapse reads the secret from the given file once at startup. + +Example configuration: +```yaml +form_secret_path: /path/to/secrets/file +``` + +_Added in Synapse 1.125.0._ + --- ## Signing Keys Config options relating to signing keys diff --git a/synapse/config/key.py b/synapse/config/key.py index 6f99f39e81..337f98dbc1 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -96,6 +96,11 @@ Conflicting options 'macaroon_secret_key' and 'macaroon_secret_key_path' are both defined in config file. """ +CONFLICTING_FORM_SECRET_OPTS_ERROR = """\ +Conflicting options 'form_secret' and 'form_secret_path' are both defined in +config file. +""" + logger = logging.getLogger(__name__) @@ -201,12 +206,19 @@ class KeyConfig(Config): # a secret which is used to calculate HMACs for form values, to stop # falsification of values - self.form_secret = config.get("form_secret", None) - if self.form_secret and not allow_secrets_in_config: + form_secret = config.get("form_secret", None) + if form_secret and not allow_secrets_in_config: raise ConfigError( "Config options that expect an in-line secret as value are disabled", ("form_secret",), ) + form_secret_path = config.get("form_secret_path", None) + if form_secret_path: + if form_secret: + raise ConfigError(CONFLICTING_FORM_SECRET_OPTS_ERROR) + self.form_secret = read_file(form_secret_path, "form_secret_path").strip() + else: + self.form_secret = form_secret def generate_config_section( self, diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 18fb2e0c2c..14dfa6e59d 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -138,6 +138,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "turn_shared_secret_path: /does/not/exist", "registration_shared_secret_path: /does/not/exist", "macaroon_secret_key_path: /does/not/exist", + "form_secret_path: /does/not/exist", "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] @@ -165,6 +166,10 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "macaroon_secret_key_path: {}", lambda c: c.key.macaroon_secret_key, ), + ( + "form_secret_path: {}", + lambda c: c.key.form_secret.encode("utf-8"), + ), ( "experimental_features:\n msc3861:\n client_secret_path: {}", lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), @@ -186,7 +191,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): self, config_line: str, get_secret: Callable[[RootConfig], str] ) -> None: self.generate_config_and_remove_lines_containing( - ["registration_shared_secret", "macaroon_secret_key"] + ["form_secret", "macaroon_secret_key", "registration_shared_secret"] ) with tempfile.NamedTemporaryFile(buffering=0) as secret_file: secret_file.write(b"53C237") From c360da0f8b642d28601a06f2b5c3e072a5dbfca7 Mon Sep 17 00:00:00 2001 From: V02460 Date: Wed, 26 Feb 2025 16:55:10 +0100 Subject: [PATCH 176/691] Add worker_replication_secret_path config option (#18191) Workers now get their secrets from files, too! There are not many config options left to pathify :) Includes documentation and unit tests. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Devon Hudson --- changelog.d/18191.feature | 1 + .../configuration/config_documentation.md | 18 +++++++++++++++- synapse/config/workers.py | 21 +++++++++++++++++-- tests/config/test_load.py | 5 +++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18191.feature diff --git a/changelog.d/18191.feature b/changelog.d/18191.feature new file mode 100644 index 0000000000..f47c9e2275 --- /dev/null +++ b/changelog.d/18191.feature @@ -0,0 +1 @@ +Add `worker_replication_secret_path` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index f5a5aa2eb4..ffee089304 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3252,7 +3252,7 @@ Example configuration: form_secret_path: /path/to/secrets/file ``` -_Added in Synapse 1.125.0._ +_Added in Synapse 1.126.0._ --- ## Signing Keys @@ -4527,6 +4527,22 @@ Example configuration: ```yaml worker_replication_secret: "secret_secret" ``` +--- +### `worker_replication_secret_path` + +An alternative to [`worker_replication_secret`](#worker_replication_secret): +allows the secret to be specified in an external file. + +The file should be a plain text file, containing only the secret. +Synapse reads the secret from the given file once at startup. + +Example configuration: +```yaml +worker_replication_secret_path: /path/to/secrets/file +``` + +_Added in Synapse 1.126.0._ + --- ### `start_pushers` diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 632cef46ce..5af50ee952 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -38,6 +38,7 @@ from synapse.config._base import ( ConfigError, RoutableShardedWorkerHandlingConfig, ShardedWorkerHandlingConfig, + read_file, ) from synapse.config._util import parse_and_validate_mapping from synapse.config.server import ( @@ -65,6 +66,11 @@ configuration under `main` inside the `instance_map`. See workers documentation `https://element-hq.github.io/synapse/latest/workers.html#worker-configuration` """ +CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR = """\ +Conflicting options 'worker_replication_secret' and +'worker_replication_secret_path' are both defined in config file. +""" + # This allows for a handy knob when it's time to change from 'master' to # something with less 'history' MAIN_PROCESS_INSTANCE_NAME = "master" @@ -244,12 +250,23 @@ class WorkerConfig(Config): raise ConfigError(DIRECT_TCP_ERROR, ("worker_replication_port",)) # The shared secret used for authentication when connecting to the main synapse. - self.worker_replication_secret = config.get("worker_replication_secret", None) - if self.worker_replication_secret and not allow_secrets_in_config: + worker_replication_secret = config.get("worker_replication_secret", None) + if worker_replication_secret and not allow_secrets_in_config: raise ConfigError( "Config options that expect an in-line secret as value are disabled", ("worker_replication_secret",), ) + worker_replication_secret_path = config.get( + "worker_replication_secret_path", None + ) + if worker_replication_secret_path: + if worker_replication_secret: + raise ConfigError(CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR) + self.worker_replication_secret = read_file( + worker_replication_secret_path, "worker_replication_secret_path" + ).strip() + else: + self.worker_replication_secret = worker_replication_secret self.worker_name = config.get("worker_name", self.worker_app) self.instance_name = self.worker_name or MAIN_PROCESS_INSTANCE_NAME diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 14dfa6e59d..a5456ac6f8 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -139,6 +139,7 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "registration_shared_secret_path: /does/not/exist", "macaroon_secret_key_path: /does/not/exist", "form_secret_path: /does/not/exist", + "worker_replication_secret_path: /does/not/exist", "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] @@ -170,6 +171,10 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "form_secret_path: {}", lambda c: c.key.form_secret.encode("utf-8"), ), + ( + "worker_replication_secret_path: {}", + lambda c: c.worker.worker_replication_secret.encode("utf-8"), + ), ( "experimental_features:\n msc3861:\n client_secret_path: {}", lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), From 154e23f6d76277cc8012dc7a5dfa5f22d62b9133 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 3 Mar 2025 09:40:48 +0000 Subject: [PATCH 177/691] Add `redirect_uri` option to `oidc_providers` entries (#18197) Allows overriding the `redirect_uri` parameter sent to both the authorization and token endpoints of the IdP. Typically this parameter is hardcoded to `/_synapse/client/oidc/callback`. Yet it can be useful in certain contexts to allow a different callback URL. For instance, if you would like to intercept the authorization code returned from the IdP and do something with it, before eventually calling Synapse's OIDC callback URL yourself. This change enables enterprise use cases but does not change the default behaviour. --- Best reviewed commit-by-commit. --------- Co-authored-by: Eric Eastwood --- changelog.d/18197.feature | 1 + .../configuration/config_documentation.md | 7 +++ synapse/config/oidc.py | 16 ++++++ synapse/handlers/oidc.py | 7 ++- tests/handlers/test_oidc.py | 50 +++++++++++++++++++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18197.feature diff --git a/changelog.d/18197.feature b/changelog.d/18197.feature new file mode 100644 index 0000000000..4572ac3bdb --- /dev/null +++ b/changelog.d/18197.feature @@ -0,0 +1 @@ +Add support for specifying/overriding `redirect_uri` in the authorization and token requests against an OpenID identity provider. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ffee089304..d2d282f203 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3662,6 +3662,13 @@ Options for each entry include: not included in `scopes`. Set to `userinfo_endpoint` to always use the userinfo endpoint. +* `redirect_uri`: An optional string, that if set will override the `redirect_uri` + parameter sent in the requests to the authorization and token endpoints. + Useful if you want to redirect the client to another endpoint as part of the + OIDC login. Be aware that the client must then call Synapse's OIDC callback + URL (`/_synapse/client/oidc/callback`) manually afterwards. + Must be a valid URL including scheme and path. + * `additional_authorization_parameters`: String to string dictionary that will be passed as additional parameters to the authorization grant URL. diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py index fc4bc35b30..8ba0ba2c36 100644 --- a/synapse/config/oidc.py +++ b/synapse/config/oidc.py @@ -141,6 +141,9 @@ OIDC_PROVIDER_CONFIG_SCHEMA = { "type": "string", "enum": ["auto", "userinfo_endpoint"], }, + "redirect_uri": { + "type": ["string", "null"], + }, "allow_existing_users": {"type": "boolean"}, "user_mapping_provider": {"type": ["object", "null"]}, "attribute_requirements": { @@ -344,6 +347,7 @@ def _parse_oidc_config_dict( ), skip_verification=oidc_config.get("skip_verification", False), user_profile_method=oidc_config.get("user_profile_method", "auto"), + redirect_uri=oidc_config.get("redirect_uri"), allow_existing_users=oidc_config.get("allow_existing_users", False), user_mapping_provider_class=user_mapping_provider_class, user_mapping_provider_config=user_mapping_provider_config, @@ -467,6 +471,18 @@ class OidcProviderConfig: # values are: "auto" or "userinfo_endpoint". user_profile_method: str + redirect_uri: Optional[str] + """ + An optional replacement for Synapse's hardcoded `redirect_uri` URL + (`/_synapse/client/oidc/callback`). This can be used to send + the client to a different URL after it receives a response from the + `authorization_endpoint`. + + If this is set, the client is expected to call Synapse's OIDC callback URL + reproduced above itself with the necessary parameters and session cookie, in + order to complete OIDC login. + """ + # whether to allow a user logging in via OIDC to match a pre-existing account # instead of failing allow_existing_users: bool diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 76b692928d..18efdd9f6e 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -382,7 +382,12 @@ class OidcProvider: self._macaroon_generaton = macaroon_generator self._config = provider - self._callback_url: str = hs.config.oidc.oidc_callback_url + + self._callback_url: str + if provider.redirect_uri is not None: + self._callback_url = provider.redirect_uri + else: + self._callback_url = hs.config.oidc.oidc_callback_url # Calculate the prefix for OIDC callback paths based on the public_baseurl. # We'll insert this into the Path= parameter of any session cookies we set. diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 5ffc5a90a8..cfd9969563 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -57,6 +57,7 @@ CLIENT_ID = "test-client-id" CLIENT_SECRET = "test-client-secret" BASE_URL = "https://synapse/" CALLBACK_URL = BASE_URL + "_synapse/client/oidc/callback" +TEST_REDIRECT_URI = "https://test/oidc/callback" SCOPES = ["openid"] # config for common cases @@ -586,6 +587,24 @@ class OidcHandlerTestCase(HomeserverTestCase): code_verifier = get_value_from_macaroon(macaroon, "code_verifier") self.assertEqual(code_verifier, "") + @override_config( + {"oidc_config": {**DEFAULT_CONFIG, "redirect_uri": TEST_REDIRECT_URI}} + ) + def test_redirect_request_with_overridden_redirect_uri(self) -> None: + """The authorization endpoint redirect has the overridden `redirect_uri` value.""" + req = Mock(spec=["cookies"]) + req.cookies = [] + + url = urlparse( + self.get_success( + self.provider.handle_redirect_request(req, b"http://client/redirect") + ) + ) + + # Ensure that the redirect_uri in the returned url has been overridden. + params = parse_qs(url.query) + self.assertEqual(params["redirect_uri"], [TEST_REDIRECT_URI]) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_callback_error(self) -> None: """Errors from the provider returned in the callback are displayed.""" @@ -953,6 +972,37 @@ class OidcHandlerTestCase(HomeserverTestCase): self.assertEqual(args["client_id"], [CLIENT_ID]) self.assertEqual(args["redirect_uri"], [CALLBACK_URL]) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "redirect_uri": TEST_REDIRECT_URI, + } + } + ) + def test_code_exchange_with_overridden_redirect_uri(self) -> None: + """Code exchange behaves correctly and handles various error scenarios.""" + # Set up a fake IdP with a token endpoint handler. + token = { + "type": "Bearer", + "access_token": "aabbcc", + } + + self.fake_server.post_token_handler.side_effect = None + self.fake_server.post_token_handler.return_value = FakeResponse.json( + payload=token + ) + code = "code" + + # Exchange the code against the fake IdP. + self.get_success(self.provider._exchange_code(code, code_verifier="")) + + # Check that the `redirect_uri` parameter provided matches our + # overridden config value. + kwargs = self.fake_server.request.call_args[1] + args = parse_qs(kwargs["data"].decode("utf-8")) + self.assertEqual(args["redirect_uri"], [TEST_REDIRECT_URI]) + @override_config( { "oidc_config": { From 08c56c3acca4b3a4207c4a28b2b76b98a07403de Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 4 Mar 2025 14:08:44 +0100 Subject: [PATCH 178/691] Support getting the device ID explicitly from MAS (#18174) The context for this is that the Matrix spec allows basically anything in the device ID. With MSC3861, we're restricting this to strings that can be represented as scopes. Whilst this works well for next-gen auth sessions, compatibility/legacy sessions still can have characters that can't be encoded (mainly spaces) in them. To work around that, we added in MAS a behaviour where the device_id is given as an explicit property of the token introspection response, and remove it from the scope. Because we don't expect users to rollout new Synapse and MAS versions in sync, we needed a way to 'advertise' support for this behaviour: the easiest way to do that was through an extra header in the introspection response. On the longer term, I expect MAS and Synapse to move away from the introspection endpoint, and instead define a specific API for Synapse -> MAS communication. PR on the MAS side: https://github.com/element-hq/matrix-authentication-service/pull/4067 --- changelog.d/18174.misc | 1 + synapse/api/auth/msc3861_delegated.py | 47 ++++++++++++++++--------- tests/handlers/test_oauth_delegation.py | 38 ++++++++++++++++++++ 3 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 changelog.d/18174.misc diff --git a/changelog.d/18174.misc b/changelog.d/18174.misc new file mode 100644 index 0000000000..b7c7c4db53 --- /dev/null +++ b/changelog.d/18174.misc @@ -0,0 +1 @@ +Support device IDs that can't be represented in a scope when delegating auth to Matrix Authentication Service 0.15.0+. diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index f825b5c95e..e6bf271a1f 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -214,6 +214,9 @@ class MSC3861DelegatedAuth(BaseAuth): "Content-Type": "application/x-www-form-urlencoded", "User-Agent": str(self._http_client.user_agent, "utf-8"), "Accept": "application/json", + # Tell MAS that we support reading the device ID as an explicit + # value, not encoded in the scope. This is supported by MAS 0.15+ + "X-MAS-Supports-Device-Id": "1", } args = {"token": token, "token_type_hint": "access_token"} @@ -409,29 +412,41 @@ class MSC3861DelegatedAuth(BaseAuth): else: user_id = UserID.from_string(user_id_str) - # Find device_ids in scope - # We only allow a single device_id in the scope, so we find them all in the - # scope list, and raise if there are more than one. The OIDC server should be - # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. - device_ids = [ - tok[len(SCOPE_MATRIX_DEVICE_PREFIX) :] - for tok in scope - if tok.startswith(SCOPE_MATRIX_DEVICE_PREFIX) - ] + # MAS 0.15+ will give us the device ID as an explicit value for compatibility sessions + # If present, we get it from here, if not we get it in thee scope + device_id = introspection_result.get("device_id") + if device_id is not None: + # We got the device ID explicitly, just sanity check that it's a string + if not isinstance(device_id, str): + raise AuthError( + 500, + "Invalid device ID in introspection result", + ) + else: + # Find device_ids in scope + # We only allow a single device_id in the scope, so we find them all in the + # scope list, and raise if there are more than one. The OIDC server should be + # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. + device_ids = [ + tok[len(SCOPE_MATRIX_DEVICE_PREFIX) :] + for tok in scope + if tok.startswith(SCOPE_MATRIX_DEVICE_PREFIX) + ] - if len(device_ids) > 1: - raise AuthError( - 500, - "Multiple device IDs in scope", - ) + if len(device_ids) > 1: + raise AuthError( + 500, + "Multiple device IDs in scope", + ) + + device_id = device_ids[0] if device_ids else None - device_id = device_ids[0] if device_ids else None if device_id is not None: # Sanity check the device_id if len(device_id) > 255 or len(device_id) < 1: raise AuthError( 500, - "Invalid device ID in scope", + "Invalid device ID in introspection result", ) # Create the device on the fly if it does not exist diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index b9d66a6c52..5f8c25557a 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -380,6 +380,44 @@ class MSC3861OAuthDelegation(HomeserverTestCase): ) self.assertEqual(requester.device_id, DEVICE) + def test_active_user_with_device_explicit_device_id(self) -> None: + """The handler should return a requester with normal user rights and a device ID, given explicitly, as supported by MAS 0.15+""" + + self.http_client.request = AsyncMock( + return_value=FakeResponse.json( + code=200, + payload={ + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE]), + "device_id": DEVICE, + "username": USERNAME, + }, + ) + ) + request = Mock(args={}) + request.args[b"access_token"] = [b"mockAccessToken"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_user_by_req(request)) + self.http_client.get_json.assert_called_once_with(WELL_KNOWN) + self.http_client.request.assert_called_once_with( + method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + ) + # It should have called with the 'X-MAS-Supports-Device-Id: 1' header + self.assertEqual( + self.http_client.request.call_args[1]["headers"].getRawHeaders( + b"X-MAS-Supports-Device-Id", + ), + [b"1"], + ) + self._assertParams() + self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) + self.assertEqual(requester.is_guest, False) + self.assertEqual( + get_awaitable_result(self.auth.is_server_admin(requester)), False + ) + self.assertEqual(requester.device_id, DEVICE) + def test_multiple_devices(self) -> None: """The handler should raise an error if multiple devices are found in the scope.""" From aa4a7b75d731d64712e0edf28a6b904362e98dad Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 4 Mar 2025 13:29:36 +0000 Subject: [PATCH 179/691] 1.126.0rc1 --- CHANGES.md | 60 +++++++++++++++++++++++++++++++++++++-- changelog.d/17661.bugfix | 1 - changelog.d/18019.feature | 1 - changelog.d/18038.docker | 1 - changelog.d/18046.misc | 1 - changelog.d/18090.feature | 1 - changelog.d/18092.feature | 1 - changelog.d/18152.doc | 1 - changelog.d/18154.feature | 1 - changelog.d/18157.bugfix | 1 - changelog.d/18162.misc | 1 - changelog.d/18165.bugfix | 1 - changelog.d/18174.misc | 1 - changelog.d/18175.misc | 1 - changelog.d/18177.feature | 1 - changelog.d/18178.bugfix | 1 - changelog.d/18191.feature | 1 - changelog.d/18197.feature | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- 20 files changed, 65 insertions(+), 20 deletions(-) delete mode 100644 changelog.d/17661.bugfix delete mode 100644 changelog.d/18019.feature delete mode 100644 changelog.d/18038.docker delete mode 100644 changelog.d/18046.misc delete mode 100644 changelog.d/18090.feature delete mode 100644 changelog.d/18092.feature delete mode 100644 changelog.d/18152.doc delete mode 100644 changelog.d/18154.feature delete mode 100644 changelog.d/18157.bugfix delete mode 100644 changelog.d/18162.misc delete mode 100644 changelog.d/18165.bugfix delete mode 100644 changelog.d/18174.misc delete mode 100644 changelog.d/18175.misc delete mode 100644 changelog.d/18177.feature delete mode 100644 changelog.d/18178.bugfix delete mode 100644 changelog.d/18191.feature delete mode 100644 changelog.d/18197.feature diff --git a/CHANGES.md b/CHANGES.md index df32bc59be..992c099d6e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,66 @@ +# Synapse 1.126.0rc1 (2025-03-04) + +Installations using the Debian/Ubuntu packages from `packages.matrix.org`: +Please be aware that we have recently updated the expiry date on the repository's GPG signing key, but this change +must be imported into your keyring. +If you have the `matrix-org-archive-keyring` package installed and update before the current key expires, this should +happen automatically. +Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +### Features + +- Define ratelimit configuration for delayed event management. ([\#18019](https://github.com/element-hq/synapse/issues/18019)) +- Add `form_secret_path` config option. ([\#18090](https://github.com/element-hq/synapse/issues/18090)) +- Add the `--no-secrets-in-config` command line option. ([\#18092](https://github.com/element-hq/synapse/issues/18092)) +- Add background job to clear unreferenced state groups. ([\#18154](https://github.com/element-hq/synapse/issues/18154)) +- Add support for specifying/overriding `id_token_signing_alg_values_supported` for an OpenID identity provider. ([\#18177](https://github.com/element-hq/synapse/issues/18177)) +- Add `worker_replication_secret_path` config option. ([\#18191](https://github.com/element-hq/synapse/issues/18191)) +- Add support for specifying/overriding `redirect_uri` in the authorization and token requests against an OpenID identity provider. ([\#18197](https://github.com/element-hq/synapse/issues/18197)) + +### Bugfixes + +- Make sure we advertise registration as disabled when MSC3861 is enabled. ([\#17661](https://github.com/element-hq/synapse/issues/17661)) +- Prevent suspended users from sending encrypted messages. ([\#18157](https://github.com/element-hq/synapse/issues/18157)) +- Cleanup deleted state group references. ([\#18165](https://github.com/element-hq/synapse/issues/18165)) +- Fix MSC4108 QR-code login not working with some reverse-proxy setups. ([\#18178](https://github.com/element-hq/synapse/issues/18178)) +- Support device IDs that can't be represented in a scope when delegating auth to Matrix Authentication Service 0.15.0+. ([\#18174](https://github.com/element-hq/synapse/issues/18174)) + +### Updates to the Docker image + +- Speed up the building of the Docker image. ([\#18038](https://github.com/element-hq/synapse/issues/18038)) + +### Improved Documentation + +- Move incorrectly placed version indicator in User Event Redaction Admin API docs. ([\#18152](https://github.com/element-hq/synapse/issues/18152)) +- Document suspension Admin API. ([\#18162](https://github.com/element-hq/synapse/issues/18162)) + +### Deprecations and Removals + +- Disable room list publication by default. ([\#18175](https://github.com/element-hq/synapse/issues/18175)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.95 to 1.0.96. ([\#18187](https://github.com/element-hq/synapse/issues/18187)) +* Bump authlib from 1.4.0 to 1.4.1. ([\#18190](https://github.com/element-hq/synapse/issues/18190)) +* Bump click from 8.1.7 to 8.1.8. ([\#18189](https://github.com/element-hq/synapse/issues/18189)) +* Bump log from 0.4.25 to 0.4.26. ([\#18184](https://github.com/element-hq/synapse/issues/18184)) +* Bump pyo3-log from 0.12.0 to 0.12.1. ([\#18046](https://github.com/element-hq/synapse/issues/18046)) +* Bump serde from 1.0.217 to 1.0.218. ([\#18183](https://github.com/element-hq/synapse/issues/18183)) +* Bump serde_json from 1.0.138 to 1.0.139. ([\#18186](https://github.com/element-hq/synapse/issues/18186)) +* Bump sigstore/cosign-installer from 3.8.0 to 3.8.1. ([\#18185](https://github.com/element-hq/synapse/issues/18185)) +* Bump types-psycopg2 from 2.9.21.20241019 to 2.9.21.20250121. ([\#18188](https://github.com/element-hq/synapse/issues/18188)) + + # Synapse 1.125.0 (2025-02-25) No significant changes since 1.125.0rc1. - - # Synapse 1.125.0rc1 (2025-02-18) ### Features diff --git a/changelog.d/17661.bugfix b/changelog.d/17661.bugfix deleted file mode 100644 index 33881bbc6a..0000000000 --- a/changelog.d/17661.bugfix +++ /dev/null @@ -1 +0,0 @@ -Make sure we advertise registration as disabled when MSC3861 is enabled. diff --git a/changelog.d/18019.feature b/changelog.d/18019.feature deleted file mode 100644 index 74e22df74a..0000000000 --- a/changelog.d/18019.feature +++ /dev/null @@ -1 +0,0 @@ -Define ratelimit configuration for delayed event management. diff --git a/changelog.d/18038.docker b/changelog.d/18038.docker deleted file mode 100644 index accb09be2a..0000000000 --- a/changelog.d/18038.docker +++ /dev/null @@ -1 +0,0 @@ -Speed up the building of the Docker image. diff --git a/changelog.d/18046.misc b/changelog.d/18046.misc deleted file mode 100644 index 8acbe6a3db..0000000000 --- a/changelog.d/18046.misc +++ /dev/null @@ -1 +0,0 @@ -Bump pyo3-log from 0.12.0 to 0.12.1. diff --git a/changelog.d/18090.feature b/changelog.d/18090.feature deleted file mode 100644 index 343e2f45eb..0000000000 --- a/changelog.d/18090.feature +++ /dev/null @@ -1 +0,0 @@ -Add `form_secret_path` config option. \ No newline at end of file diff --git a/changelog.d/18092.feature b/changelog.d/18092.feature deleted file mode 100644 index 26371cc810..0000000000 --- a/changelog.d/18092.feature +++ /dev/null @@ -1 +0,0 @@ -Add the `--no-secrets-in-config` command line option. \ No newline at end of file diff --git a/changelog.d/18152.doc b/changelog.d/18152.doc deleted file mode 100644 index 5a02f26751..0000000000 --- a/changelog.d/18152.doc +++ /dev/null @@ -1 +0,0 @@ -Move incorrectly placed version indicator in User Event Redaction Admin API docs. \ No newline at end of file diff --git a/changelog.d/18154.feature b/changelog.d/18154.feature deleted file mode 100644 index 62e1b79a15..0000000000 --- a/changelog.d/18154.feature +++ /dev/null @@ -1 +0,0 @@ -Add background job to clear unreferenced state groups. diff --git a/changelog.d/18157.bugfix b/changelog.d/18157.bugfix deleted file mode 100644 index 307e9c96ff..0000000000 --- a/changelog.d/18157.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent suspended users from sending encrypted messages. diff --git a/changelog.d/18162.misc b/changelog.d/18162.misc deleted file mode 100644 index 4d46979641..0000000000 --- a/changelog.d/18162.misc +++ /dev/null @@ -1 +0,0 @@ -Document suspension Admin API. \ No newline at end of file diff --git a/changelog.d/18165.bugfix b/changelog.d/18165.bugfix deleted file mode 100644 index eba5face52..0000000000 --- a/changelog.d/18165.bugfix +++ /dev/null @@ -1 +0,0 @@ -Cleanup deleted state group references. diff --git a/changelog.d/18174.misc b/changelog.d/18174.misc deleted file mode 100644 index b7c7c4db53..0000000000 --- a/changelog.d/18174.misc +++ /dev/null @@ -1 +0,0 @@ -Support device IDs that can't be represented in a scope when delegating auth to Matrix Authentication Service 0.15.0+. diff --git a/changelog.d/18175.misc b/changelog.d/18175.misc deleted file mode 100644 index c12a30b7c1..0000000000 --- a/changelog.d/18175.misc +++ /dev/null @@ -1 +0,0 @@ -Disable room list publication by default. diff --git a/changelog.d/18177.feature b/changelog.d/18177.feature deleted file mode 100644 index 71d568474b..0000000000 --- a/changelog.d/18177.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for specifying/overriding `id_token_signing_alg_values_supported` for an OpenID identity provider. diff --git a/changelog.d/18178.bugfix b/changelog.d/18178.bugfix deleted file mode 100644 index d91327803b..0000000000 --- a/changelog.d/18178.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix MSC4108 QR-code login not working with some reverse-proxy setups. diff --git a/changelog.d/18191.feature b/changelog.d/18191.feature deleted file mode 100644 index f47c9e2275..0000000000 --- a/changelog.d/18191.feature +++ /dev/null @@ -1 +0,0 @@ -Add `worker_replication_secret_path` config option. \ No newline at end of file diff --git a/changelog.d/18197.feature b/changelog.d/18197.feature deleted file mode 100644 index 4572ac3bdb..0000000000 --- a/changelog.d/18197.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for specifying/overriding `redirect_uri` in the authorization and token requests against an OpenID identity provider. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 1ac1fc6644..d7a3909224 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.126.0~rc1) stable; urgency=medium + + * New Synapse release 1.126.0rc1. + + -- Synapse Packaging team Tue, 04 Mar 2025 13:11:51 +0000 + matrix-synapse-py3 (1.125.0) stable; urgency=medium * New Synapse release 1.125.0. diff --git a/pyproject.toml b/pyproject.toml index abacfa328d..5f18bd0768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.125.0" +version = "1.126.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 26d9ce80c528879b803c2fffb0b6e75d840ee936 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 4 Mar 2025 14:29:38 +0000 Subject: [PATCH 180/691] Add upgrade notes for the debian repo signing key expiry change --- docs/upgrade.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 7e4cd52e1d..07a9641fdd 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -137,6 +137,24 @@ room_list_publication_rules: [`room_list_publication_rules`]: usage/configuration/config_documentation.md#room_list_publication_rules +## Change of signing key expiry date for the Debian/Ubuntu package repository + +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, +please be aware that we have recently updated the expiry date on the repository's GPG signing key, +but this change must be imported into your keyring. + +If you have the `matrix-org-archive-keyring` package installed and it updates before the current key expires, this should +happen automatically. + +Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +The old version of the key will expire on `2025-03-15`. + # Upgrading to v1.122.0 ## Dropping support for PostgreSQL 11 and 12 From 70f41c4541cf443df55c9bad916b23d8e2a3e85c Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 4 Mar 2025 14:31:13 +0000 Subject: [PATCH 181/691] Tweak changelog notice for debian repo signing key expiry change --- CHANGES.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 992c099d6e..95bea7a4f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,16 +1,9 @@ # Synapse 1.126.0rc1 (2025-03-04) -Installations using the Debian/Ubuntu packages from `packages.matrix.org`: -Please be aware that we have recently updated the expiry date on the repository's GPG signing key, but this change -must be imported into your keyring. -If you have the `matrix-org-archive-keyring` package installed and update before the current key expires, this should -happen automatically. -Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you -will need to get a fresh copy of the keys. You can do so with: +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check +[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) +as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2025-03-15`. -```sh -sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg -``` ### Features From d7e238c8eed6d78d44bff6c8dc155eaf5c5359c3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 4 Mar 2025 14:31:47 +0000 Subject: [PATCH 182/691] Tweak changelog to linkify MSCs --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 95bea7a4f2..6643344125 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,10 +17,10 @@ as we have recently updated the expiry date on the repository's GPG signing key. ### Bugfixes -- Make sure we advertise registration as disabled when MSC3861 is enabled. ([\#17661](https://github.com/element-hq/synapse/issues/17661)) +- Make sure we advertise registration as disabled when [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) is enabled. ([\#17661](https://github.com/element-hq/synapse/issues/17661)) - Prevent suspended users from sending encrypted messages. ([\#18157](https://github.com/element-hq/synapse/issues/18157)) - Cleanup deleted state group references. ([\#18165](https://github.com/element-hq/synapse/issues/18165)) -- Fix MSC4108 QR-code login not working with some reverse-proxy setups. ([\#18178](https://github.com/element-hq/synapse/issues/18178)) +- Fix [MSC4108 QR-code login](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) not working with some reverse-proxy setups. ([\#18178](https://github.com/element-hq/synapse/issues/18178)) - Support device IDs that can't be represented in a scope when delegating auth to Matrix Authentication Service 0.15.0+. ([\#18174](https://github.com/element-hq/synapse/issues/18174)) ### Updates to the Docker image From b7946c29be510855bcf4f834a118d62992e9df0d Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 4 Mar 2025 17:37:28 +0000 Subject: [PATCH 183/691] Fix wheel building configuration in CI by installing libatomic1. (#18212) Signed-off-by: Olivier 'reivilibre --- .ci/before_build_wheel.sh | 11 +++++++++++ changelog.d/18212.misc | 1 + pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .ci/before_build_wheel.sh create mode 100644 changelog.d/18212.misc diff --git a/.ci/before_build_wheel.sh b/.ci/before_build_wheel.sh new file mode 100644 index 0000000000..4356dbc7e7 --- /dev/null +++ b/.ci/before_build_wheel.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +# On 32-bit Linux platforms, we need libatomic1 to use rustup +if command -v apt-get &> /dev/null; then + apt-get update + apt-get install libatomic1 +fi + +# Install a Rust toolchain +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.82.0 -y --profile minimal diff --git a/changelog.d/18212.misc b/changelog.d/18212.misc new file mode 100644 index 0000000000..0e8516a323 --- /dev/null +++ b/changelog.d/18212.misc @@ -0,0 +1 @@ +Fix wheel building configuration in CI by installing libatomic1. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5f18bd0768..80482f469a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -390,7 +390,7 @@ skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aa # # We temporarily pin Rust to 1.82.0 to work around # https://github.com/element-hq/synapse/issues/17988 -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.82.0 -y --profile minimal" +before-all = "sh .ci/before_build_wheel.sh" environment= { PATH = "$PATH:$HOME/.cargo/bin" } # For some reason if we don't manually clean the build directory we From 69aceef8f6ff5690e0e92bc8aac7dfded02363bf Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 5 Mar 2025 14:20:17 +0000 Subject: [PATCH 184/691] Actually fix CI build wheels. (#18213) Follows: #18212 --------- Signed-off-by: Olivier 'reivilibre --- .ci/before_build_wheel.sh | 7 +++---- .github/workflows/release-artifacts.yml | 2 +- Cargo.lock | 20 ++++++++++---------- changelog.d/18213.misc | 1 + rust/Cargo.toml | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 changelog.d/18213.misc diff --git a/.ci/before_build_wheel.sh b/.ci/before_build_wheel.sh index 4356dbc7e7..56108dcd60 100644 --- a/.ci/before_build_wheel.sh +++ b/.ci/before_build_wheel.sh @@ -1,10 +1,9 @@ #!/bin/sh -set -eu +set -xeu # On 32-bit Linux platforms, we need libatomic1 to use rustup -if command -v apt-get &> /dev/null; then - apt-get update - apt-get install libatomic1 +if command -v yum &> /dev/null; then + yum install -y libatomic fi # Install a Rust toolchain diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 10583bc060..3311f09b2d 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -139,7 +139,7 @@ jobs: python-version: "3.x" - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.19.1 + run: python -m pip install cibuildwheel==2.23.0 - name: Set up QEMU to emulate aarch64 if: matrix.arch == 'aarch64' diff --git a/Cargo.lock b/Cargo.lock index b9aa1c8a6b..b8247e1315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" dependencies = [ "anyhow", "cfg-if", @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" dependencies = [ "once_cell", "target-lexicon", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" dependencies = [ "libc", "pyo3-build-config", @@ -327,9 +327,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ "heck", "proc-macro2", diff --git a/changelog.d/18213.misc b/changelog.d/18213.misc new file mode 100644 index 0000000000..0e8516a323 --- /dev/null +++ b/changelog.d/18213.misc @@ -0,0 +1 @@ +Fix wheel building configuration in CI by installing libatomic1. \ No newline at end of file diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a8a24ee1c7..651b268f86 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.23.2", features = [ +pyo3 = { version = "0.23.5", features = [ "macros", "anyhow", "abi3", From 350e84a8a4041821a6fb4e8f6cd42436862275ff Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 5 Mar 2025 14:35:21 +0000 Subject: [PATCH 185/691] 1.126.0rc2 --- CHANGES.md | 9 ++++++++- changelog.d/18212.misc | 1 - changelog.d/18213.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/18212.misc delete mode 100644 changelog.d/18213.misc diff --git a/CHANGES.md b/CHANGES.md index 6643344125..40df704d68 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,16 @@ -# Synapse 1.126.0rc1 (2025-03-04) +# Synapse 1.126.0rc2 (2025-03-05) Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2025-03-15`. +### Internal Changes + +- Fix wheel building configuration in CI by installing libatomic1. ([\#18212](https://github.com/element-hq/synapse/issues/18212), [\#18213](https://github.com/element-hq/synapse/issues/18213)) + +# Synapse 1.126.0rc1 (2025-03-04) + +Synapse 1.126.0rc1 was not fully released due to an error in CI. ### Features diff --git a/changelog.d/18212.misc b/changelog.d/18212.misc deleted file mode 100644 index 0e8516a323..0000000000 --- a/changelog.d/18212.misc +++ /dev/null @@ -1 +0,0 @@ -Fix wheel building configuration in CI by installing libatomic1. \ No newline at end of file diff --git a/changelog.d/18213.misc b/changelog.d/18213.misc deleted file mode 100644 index 0e8516a323..0000000000 --- a/changelog.d/18213.misc +++ /dev/null @@ -1 +0,0 @@ -Fix wheel building configuration in CI by installing libatomic1. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index d7a3909224..b66326d79b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.126.0~rc2) stable; urgency=medium + + * New Synapse release 1.126.0rc2. + + -- Synapse Packaging team Wed, 05 Mar 2025 14:29:12 +0000 + matrix-synapse-py3 (1.126.0~rc1) stable; urgency=medium * New Synapse release 1.126.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 80482f469a..f95c8e0e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.126.0rc1" +version = "1.126.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 8295de87a7fb5dcf975d6b3d8f899686f3a564e6 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Fri, 7 Mar 2025 15:44:13 +0000 Subject: [PATCH 186/691] Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to a suspected issue that causes increased disk usage. (#18222) Revert "Add background job to clear unreferenced state groups (#18154)" This mechanism is suspected of inserting large numbers of rows into `state_groups_state`, thus unreasonably increasing disk usage. See: https://github.com/element-hq/synapse/issues/18217 This reverts commit 5121f9210c989fcc909e78195133876dff3bc9b9 (#18154). --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18222.bugfix | 1 + docs/development/database_schema.md | 2 +- synapse/_scripts/synapse_port_db.py | 30 --- synapse/storage/controllers/purge_events.py | 246 +++++------------- synapse/storage/databases/state/bg_updates.py | 10 +- synapse/storage/databases/state/deletion.py | 44 +--- synapse/storage/schema/__init__.py | 1 - .../02_delete_unreferenced_state_groups.sql | 16 -- synapse/types/storage/__init__.py | 4 - tests/storage/test_purge.py | 97 ------- 10 files changed, 76 insertions(+), 375 deletions(-) create mode 100644 changelog.d/18222.bugfix delete mode 100644 synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql diff --git a/changelog.d/18222.bugfix b/changelog.d/18222.bugfix new file mode 100644 index 0000000000..16160c5651 --- /dev/null +++ b/changelog.d/18222.bugfix @@ -0,0 +1 @@ +Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to a suspected issue that causes increased disk usage. \ No newline at end of file diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 620d1c16b0..37a06acc12 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -162,7 +162,7 @@ by a unique name, the current status (stored in JSON), and some dependency infor * Whether the update requires a previous update to be complete. * A rough ordering for which to complete updates. -A new background update needs to be added to the `background_updates` table: +A new background updates needs to be added to the `background_updates` table: ```sql INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 59065a0504..3f67a739a0 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -191,11 +191,6 @@ APPEND_ONLY_TABLES = [ IGNORED_TABLES = { - # Porting the auto generated sequence in this table is non-trivial. - # None of the entries in this list are mandatory for Synapse to keep working. - # If state group disk space is an issue after the port, the - # `delete_unreferenced_state_groups_bg_update` background task can be run again. - "state_groups_pending_deletion", # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", @@ -221,15 +216,6 @@ IGNORED_TABLES = { } -# These background updates will not be applied upon creation of the postgres database. -IGNORED_BACKGROUND_UPDATES = { - # Reapplying this background update to the postgres database is unnecessary after - # already having waited for the SQLite database to complete all running background - # updates. - "delete_unreferenced_state_groups_bg_update", -} - - # Error returned by the run function. Used at the top-level part of the script to # handle errors and return codes. end_error: Optional[str] = None @@ -701,20 +687,6 @@ class Porter: # 0 means off. 1 means full. 2 means incremental. return autovacuum_setting != 0 - async def remove_ignored_background_updates_from_database(self) -> None: - def _remove_delete_unreferenced_state_groups_bg_updates( - txn: LoggingTransaction, - ) -> None: - txn.execute( - "DELETE FROM background_updates WHERE update_name = ANY(?)", - (list(IGNORED_BACKGROUND_UPDATES),), - ) - - await self.postgres_store.db_pool.runInteraction( - "remove_delete_unreferenced_state_groups_bg_updates", - _remove_delete_unreferenced_state_groups_bg_updates, - ) - async def run(self) -> None: """Ports the SQLite database to a PostgreSQL database. @@ -760,8 +732,6 @@ class Porter: self.hs_config.database.get_single_database() ) - await self.remove_ignored_background_updates_from_database() - await self.run_background_updates_on_postgres() self.progress.set_state("Creating port tables") diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index ef30bf2895..47cec8c469 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -21,18 +21,11 @@ import itertools import logging -from typing import ( - TYPE_CHECKING, - Collection, - Mapping, - Set, -) +from typing import TYPE_CHECKING, Collection, Mapping, Set from synapse.logging.context import nested_logging_context from synapse.metrics.background_process_metrics import wrap_as_background_process -from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases -from synapse.types.storage import _BackgroundUpdates if TYPE_CHECKING: from synapse.server import HomeServer @@ -51,11 +44,6 @@ class PurgeEventsStorageController: self._delete_state_groups_loop, 60 * 1000 ) - self.stores.state.db_pool.updates.register_background_update_handler( - _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, - self._background_delete_unrefereneced_state_groups, - ) - async def purge_room(self, room_id: str) -> None: """Deletes all record of a room""" @@ -92,6 +80,68 @@ class PurgeEventsStorageController: sg_to_delete ) + async def _find_unreferenced_groups( + self, state_groups: Collection[int] + ) -> Set[int]: + """Used when purging history to figure out which state groups can be + deleted. + + Args: + state_groups: Set of state groups referenced by events + that are going to be deleted. + + Returns: + The set of state groups that can be deleted. + """ + # Set of events that we have found to be referenced by events + referenced_groups = set() + + # Set of state groups we've already seen + state_groups_seen = set(state_groups) + + # Set of state groups to handle next. + next_to_search = set(state_groups) + while next_to_search: + # We bound size of groups we're looking up at once, to stop the + # SQL query getting too big + if len(next_to_search) < 100: + current_search = next_to_search + next_to_search = set() + else: + current_search = set(itertools.islice(next_to_search, 100)) + next_to_search -= current_search + + referenced = await self.stores.main.get_referenced_state_groups( + current_search + ) + referenced_groups |= referenced + + # We don't continue iterating up the state group graphs for state + # groups that are referenced. + current_search -= referenced + + edges = await self.stores.state.get_previous_state_groups(current_search) + + prevs = set(edges.values()) + # We don't bother re-handling groups we've already seen + prevs -= state_groups_seen + next_to_search |= prevs + state_groups_seen |= prevs + + # We also check to see if anything referencing the state groups are + # also unreferenced. This helps ensure that we delete unreferenced + # state groups, if we don't then we will de-delta them when we + # delete the other state groups leading to increased DB usage. + next_edges = await self.stores.state.get_next_state_groups(current_search) + nexts = set(next_edges.keys()) + nexts -= state_groups_seen + next_to_search |= nexts + state_groups_seen |= nexts + + to_delete = state_groups_seen - referenced_groups + + return to_delete + @wrap_as_background_process("_delete_state_groups_loop") async def _delete_state_groups_loop(self) -> None: """Background task that deletes any state groups that may be pending @@ -153,173 +203,3 @@ class PurgeEventsStorageController: room_id, groups_to_sequences, ) - - async def _background_delete_unrefereneced_state_groups( - self, progress: dict, batch_size: int - ) -> int: - """This background update will slowly delete any unreferenced state groups""" - - last_checked_state_group = progress.get("last_checked_state_group") - max_state_group = progress.get("max_state_group") - - if last_checked_state_group is None or max_state_group is None: - # This is the first run. - last_checked_state_group = 0 - - max_state_group = await self.stores.state.db_pool.simple_select_one_onecol( - table="state_groups", - keyvalues={}, - retcol="MAX(id)", - allow_none=True, - desc="get_max_state_group", - ) - if max_state_group is None: - # There are no state groups so the background process is finished. - await self.stores.state.db_pool.updates._end_background_update( - _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE - ) - return batch_size - - ( - last_checked_state_group, - final_batch, - ) = await self._delete_unreferenced_state_groups_batch( - last_checked_state_group, batch_size, max_state_group - ) - - if not final_batch: - # There are more state groups to check. - progress = { - "last_checked_state_group": last_checked_state_group, - "max_state_group": max_state_group, - } - await self.stores.state.db_pool.updates._background_update_progress( - _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, - progress, - ) - else: - # This background process is finished. - await self.stores.state.db_pool.updates._end_background_update( - _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE - ) - - return batch_size - - async def _delete_unreferenced_state_groups_batch( - self, - last_checked_state_group: int, - batch_size: int, - max_state_group: int, - ) -> tuple[int, bool]: - """Looks for unreferenced state groups starting from the last state group - checked, and any state groups which would become unreferenced if a state group - was deleted, and marks them for deletion. - - Args: - last_checked_state_group: The last state group that was checked. - batch_size: How many state groups to process in this iteration. - - Returns: - (last_checked_state_group, final_batch) - """ - - # Look for state groups that can be cleaned up. - def get_next_state_groups_txn(txn: LoggingTransaction) -> Set[int]: - state_group_sql = "SELECT id FROM state_groups WHERE ? < id AND id <= ? ORDER BY id LIMIT ?" - txn.execute( - state_group_sql, (last_checked_state_group, max_state_group, batch_size) - ) - - next_set = {row[0] for row in txn} - - return next_set - - next_set = await self.stores.state.db_pool.runInteraction( - "get_next_state_groups", get_next_state_groups_txn - ) - - final_batch = False - if len(next_set) < batch_size: - final_batch = True - else: - last_checked_state_group = max(next_set) - - if len(next_set) == 0: - return last_checked_state_group, final_batch - - # Find all state groups that can be deleted if the original set is deleted. - # This set includes the original set, as well as any state groups that would - # become unreferenced upon deleting the original set. - to_delete = await self._find_unreferenced_groups(next_set) - - if len(to_delete) == 0: - return last_checked_state_group, final_batch - - await self.stores.state_deletion.mark_state_groups_as_pending_deletion( - to_delete - ) - - return last_checked_state_group, final_batch - - async def _find_unreferenced_groups( - self, - state_groups: Collection[int], - ) -> Set[int]: - """Used when purging history to figure out which state groups can be - deleted. - - Args: - state_groups: Set of state groups referenced by events - that are going to be deleted. - - Returns: - The set of state groups that can be deleted. - """ - # Set of events that we have found to be referenced by events - referenced_groups = set() - - # Set of state groups we've already seen - state_groups_seen = set(state_groups) - - # Set of state groups to handle next. - next_to_search = set(state_groups) - while next_to_search: - # We bound size of groups we're looking up at once, to stop the - # SQL query getting too big - if len(next_to_search) < 100: - current_search = next_to_search - next_to_search = set() - else: - current_search = set(itertools.islice(next_to_search, 100)) - next_to_search -= current_search - - referenced = await self.stores.main.get_referenced_state_groups( - current_search - ) - referenced_groups |= referenced - - # We don't continue iterating up the state group graphs for state - # groups that are referenced. - current_search -= referenced - - edges = await self.stores.state.get_previous_state_groups(current_search) - - prevs = set(edges.values()) - # We don't bother re-handling groups we've already seen - prevs -= state_groups_seen - next_to_search |= prevs - state_groups_seen |= prevs - - # We also check to see if anything referencing the state groups are - # also unreferenced. This helps ensure that we delete unreferenced - # state groups, if we don't then we will de-delta them when we - # delete the other state groups leading to increased DB usage. - next_edges = await self.stores.state.get_next_state_groups(current_search) - nexts = set(next_edges.keys()) - nexts -= state_groups_seen - next_to_search |= nexts - state_groups_seen |= nexts - - to_delete = state_groups_seen - referenced_groups - - return to_delete diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 95fd0ae73a..f7824cba0f 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -20,15 +20,7 @@ # import logging -from typing import ( - TYPE_CHECKING, - Dict, - List, - Mapping, - Optional, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index f77c46f6ae..d4b1c20a45 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -321,42 +321,18 @@ class StateDeletionDataStore: async def mark_state_groups_as_pending_deletion( self, state_groups: Collection[int] ) -> None: - """Mark the given state groups as pending deletion. - - If any of the state groups are already pending deletion, then those records are - left as is. - """ - - await self.db_pool.runInteraction( - "mark_state_groups_as_pending_deletion", - self._mark_state_groups_as_pending_deletion_txn, - state_groups, - ) - - def _mark_state_groups_as_pending_deletion_txn( - self, - txn: LoggingTransaction, - state_groups: Collection[int], - ) -> None: - sql = """ - INSERT INTO state_groups_pending_deletion (state_group, insertion_ts) - VALUES %s - ON CONFLICT (state_group) - DO NOTHING - """ + """Mark the given state groups as pending deletion""" now = self._clock.time_msec() - rows = [ - ( - state_group, - now, - ) - for state_group in state_groups - ] - if isinstance(txn.database_engine, PostgresEngine): - txn.execute_values(sql % ("?",), rows, fetch=False) - else: - txn.execute_batch(sql % ("(?, ?)",), rows) + + await self.db_pool.simple_upsert_many( + table="state_groups_pending_deletion", + key_names=("state_group",), + key_values=[(state_group,) for state_group in state_groups], + value_names=("insertion_ts",), + value_values=[(now,) for _ in state_groups], + desc="mark_state_groups_as_pending_deletion", + ) async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None: """Mark the given state groups as now being referenced""" diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index c90c2c6051..49e648a92f 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -158,7 +158,6 @@ Changes in SCHEMA_VERSION = 88 Changes in SCHEMA_VERSION = 89 - Add `state_groups_pending_deletion` and `state_groups_persisting` tables. - - Add background update to delete unreferenced state groups. """ diff --git a/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql b/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql deleted file mode 100644 index 184dc8564c..0000000000 --- a/synapse/storage/schema/state/delta/89/02_delete_unreferenced_state_groups.sql +++ /dev/null @@ -1,16 +0,0 @@ --- --- This file is licensed under the Affero General Public License (AGPL) version 3. --- --- Copyright (C) 2025 New Vector, Ltd --- --- This program is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation, either version 3 of the --- License, or (at your option) any later version. --- --- See the GNU Affero General Public License for more details: --- . - --- Add a background update to delete any unreferenced state groups -INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (8902, 'delete_unreferenced_state_groups_bg_update', '{}'); diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index d0a85ef208..b5fa20a41a 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -48,7 +48,3 @@ class _BackgroundUpdates: SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = ( "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update" ) - - DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE = ( - "delete_unreferenced_state_groups_bg_update" - ) diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index ecdc893405..916e42e731 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -24,7 +24,6 @@ from synapse.api.errors import NotFoundError, SynapseError from synapse.rest.client import room from synapse.server import HomeServer from synapse.types.state import StateFilter -from synapse.types.storage import _BackgroundUpdates from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -304,99 +303,3 @@ class PurgeTests(HomeserverTestCase): ) ) self.assertEqual(len(state_groups), 1) - - def test_clear_unreferenced_state_groups(self) -> None: - """Test that any unreferenced state groups are automatically cleaned up.""" - - self.helper.send(self.room_id, body="test1") - state1 = self.helper.send_state( - self.room_id, "org.matrix.test", body={"number": 2} - ) - # Create enough state events to require multiple batches of - # delete_unreferenced_state_groups_bg_update to be run. - for i in range(200): - self.helper.send_state(self.room_id, "org.matrix.test", body={"number": i}) - state2 = self.helper.send_state( - self.room_id, "org.matrix.test", body={"number": 3} - ) - self.helper.send(self.room_id, body="test4") - last = self.helper.send(self.room_id, body="test5") - - # Create an unreferenced state group that has a prev group of one of the - # to-be-purged events. - prev_group = self.get_success( - self.store._get_state_group_for_event(state1["event_id"]) - ) - unreferenced_state_group = self.get_success( - self.state_store.store_state_group( - event_id=last["event_id"], - room_id=self.room_id, - prev_group=prev_group, - delta_ids={("org.matrix.test", ""): state2["event_id"]}, - current_state_ids=None, - ) - ) - - another_unreferenced_state_group = self.get_success( - self.state_store.store_state_group( - event_id=last["event_id"], - room_id=self.room_id, - prev_group=unreferenced_state_group, - delta_ids={("org.matrix.test", ""): state2["event_id"]}, - current_state_ids=None, - ) - ) - - # Insert and run the background update. - self.get_success( - self.store.db_pool.simple_insert( - "background_updates", - { - "update_name": _BackgroundUpdates.DELETE_UNREFERENCED_STATE_GROUPS_BG_UPDATE, - "progress_json": "{}", - }, - ) - ) - self.store.db_pool.updates._all_done = False - self.wait_for_background_updates() - - # Advance so that the background job to delete the state groups runs - self.reactor.advance( - 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 - ) - - # We expect that the unreferenced state group has been deleted. - row = self.get_success( - self.state_store.db_pool.simple_select_one_onecol( - table="state_groups", - keyvalues={"id": unreferenced_state_group}, - retcol="id", - allow_none=True, - desc="test_purge_unreferenced_state_group", - ) - ) - self.assertIsNone(row) - - # We expect that the other unreferenced state group has also been deleted. - row = self.get_success( - self.state_store.db_pool.simple_select_one_onecol( - table="state_groups", - keyvalues={"id": another_unreferenced_state_group}, - retcol="id", - allow_none=True, - desc="test_purge_unreferenced_state_group", - ) - ) - self.assertIsNone(row) - - # We expect there to now only be one state group for the room, which is - # the state group of the last event (as the only outlier). - state_groups = self.get_success( - self.state_store.db_pool.simple_select_onecol( - table="state_groups", - keyvalues={"room_id": self.room_id}, - retcol="id", - desc="test_purge_unreferenced_state_group", - ) - ) - self.assertEqual(len(state_groups), 207) From 1803a62db41a9301b336f82257514255e22566a9 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 7 Mar 2025 15:45:11 +0000 Subject: [PATCH 187/691] 1.126.0rc3 --- CHANGES.md | 9 +++++++++ changelog.d/18222.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18222.bugfix diff --git a/CHANGES.md b/CHANGES.md index 40df704d68..d080be07ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.126.0rc3 (2025-03-07) + +### Bugfixes + +- Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to a suspected issue that causes increased disk usage. ([\#18222](https://github.com/element-hq/synapse/issues/18222)) + + + + # Synapse 1.126.0rc2 (2025-03-05) Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check diff --git a/changelog.d/18222.bugfix b/changelog.d/18222.bugfix deleted file mode 100644 index 16160c5651..0000000000 --- a/changelog.d/18222.bugfix +++ /dev/null @@ -1 +0,0 @@ -Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to a suspected issue that causes increased disk usage. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index b66326d79b..8104933cf1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.126.0~rc3) stable; urgency=medium + + * New Synapse release 1.126.0rc3. + + -- Synapse Packaging team Fri, 07 Mar 2025 15:45:05 +0000 + matrix-synapse-py3 (1.126.0~rc2) stable; urgency=medium * New Synapse release 1.126.0rc2. diff --git a/pyproject.toml b/pyproject.toml index f95c8e0e80..7db8fa08ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.126.0rc2" +version = "1.126.0rc3" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From a4c476305e20609a71770818c8fd3eb38eff704a Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 7 Mar 2025 16:03:18 +0000 Subject: [PATCH 188/691] Tweak changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d080be07ee..8d1ba899e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ### Bugfixes -- Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to a suspected issue that causes increased disk usage. ([\#18222](https://github.com/element-hq/synapse/issues/18222)) +- Revert the background job to clear unreferenced state groups (that was introduced in v1.126.0rc1), due to [a suspected issue](https://github.com/element-hq/synapse/issues/18217) that causes increased disk usage. ([\#18222](https://github.com/element-hq/synapse/issues/18222)) From fda1ffe5b8ca44243a52ffc1fcf25bf274734251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:27:19 +0000 Subject: [PATCH 189/691] Bump serde_json from 1.0.139 to 1.0.140 (#18202) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8247e1315..19d104e91a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,9 +457,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", From c30157b3cbc68c15f4367b20b6324b40fa421bdc Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 11 Mar 2025 13:11:45 +0000 Subject: [PATCH 190/691] 1.126.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8d1ba899e7..7ed24a0559 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.126.0 (2025-03-11) + +No significant changes since 1.126.0rc3. + + + + # Synapse 1.126.0rc3 (2025-03-07) ### Bugfixes diff --git a/debian/changelog b/debian/changelog index 8104933cf1..69d19fa048 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.126.0) stable; urgency=medium + + * New Synapse release 1.126.0. + + -- Synapse Packaging team Tue, 11 Mar 2025 13:11:29 +0000 + matrix-synapse-py3 (1.126.0~rc3) stable; urgency=medium * New Synapse release 1.126.0rc3. diff --git a/pyproject.toml b/pyproject.toml index 7db8fa08ad..27a4da7e72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.126.0rc3" +version = "1.126.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 5064f35958a92bf4678af70ccb9d2762cab543f1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 11 Mar 2025 13:15:44 +0000 Subject: [PATCH 191/691] Move debian signing key expiry notice to top of 1.126.0 notes --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7ed24a0559..1cb3e7da13 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,7 @@ # Synapse 1.126.0 (2025-03-11) +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check +[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) +as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2025-03-15`. No significant changes since 1.126.0rc3. @@ -16,9 +19,6 @@ No significant changes since 1.126.0rc3. # Synapse 1.126.0rc2 (2025-03-05) -Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check -[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) -as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2025-03-15`. ### Internal Changes From 74cc353961197804042a1cba52e93c89c4d30aa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:39:57 +0000 Subject: [PATCH 192/691] Bump serde from 1.0.218 to 1.0.219 (#18228) --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19d104e91a..7ccdba30eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -437,18 +437,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", From 698278ba5089b4b9f823fe57f4befd8b309f636f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:40:09 +0000 Subject: [PATCH 193/691] Bump bytes from 1.10.0 to 1.10.1 (#18227) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ccdba30eb..477620978c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,9 +67,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cfg-if" From 04814a48de6cf5c791967e615e369619fd376bcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:44:39 +0000 Subject: [PATCH 194/691] Bump sentry-sdk from 2.19.2 to 2.22.0 (#18205) --- poetry.lock | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1ad631199a..5ead1cd295 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2348,13 +2348,13 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.19.2" +version = "2.22.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" files = [ - {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"}, - {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"}, + {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, + {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, ] [package.dependencies] @@ -2398,7 +2398,9 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] [[package]] name = "service-identity" From df044a36678530aef419760b3439b183ec0676f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:44:49 +0000 Subject: [PATCH 195/691] Bump bcrypt from 4.2.1 to 4.3.0 (#18207) --- poetry.lock | 80 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5ead1cd295..cbf22d9806 100644 --- a/poetry.lock +++ b/poetry.lock @@ -64,36 +64,62 @@ visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] [[package]] name = "bcrypt" -version = "4.2.1" +version = "4.3.0" description = "Modern password hashing for your software and your servers" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "bcrypt-4.2.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:1340411a0894b7d3ef562fb233e4b6ed58add185228650942bdc885362f32c17"}, - {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ee315739bc8387aa36ff127afc99120ee452924e0df517a8f3e4c0187a0f5f"}, - {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dbd0747208912b1e4ce730c6725cb56c07ac734b3629b60d4398f082ea718ad"}, - {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:aaa2e285be097050dba798d537b6efd9b698aa88eef52ec98d23dcd6d7cf6fea"}, - {file = "bcrypt-4.2.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:76d3e352b32f4eeb34703370e370997065d28a561e4a18afe4fef07249cb4396"}, - {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b7703ede632dc945ed1172d6f24e9f30f27b1b1a067f32f68bf169c5f08d0425"}, - {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:89df2aea2c43be1e1fa066df5f86c8ce822ab70a30e4c210968669565c0f4685"}, - {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:04e56e3fe8308a88b77e0afd20bec516f74aecf391cdd6e374f15cbed32783d6"}, - {file = "bcrypt-4.2.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cfdf3d7530c790432046c40cda41dfee8c83e29482e6a604f8930b9930e94139"}, - {file = "bcrypt-4.2.1-cp37-abi3-win32.whl", hash = "sha256:adadd36274510a01f33e6dc08f5824b97c9580583bd4487c564fc4617b328005"}, - {file = "bcrypt-4.2.1-cp37-abi3-win_amd64.whl", hash = "sha256:8c458cd103e6c5d1d85cf600e546a639f234964d0228909d8f8dbeebff82d526"}, - {file = "bcrypt-4.2.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8ad2f4528cbf0febe80e5a3a57d7a74e6635e41af1ea5675282a33d769fba413"}, - {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:909faa1027900f2252a9ca5dfebd25fc0ef1417943824783d1c8418dd7d6df4a"}, - {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cde78d385d5e93ece5479a0a87f73cd6fa26b171c786a884f955e165032b262c"}, - {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:533e7f3bcf2f07caee7ad98124fab7499cb3333ba2274f7a36cf1daee7409d99"}, - {file = "bcrypt-4.2.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:687cf30e6681eeda39548a93ce9bfbb300e48b4d445a43db4298d2474d2a1e54"}, - {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:041fa0155c9004eb98a232d54da05c0b41d4b8e66b6fc3cb71b4b3f6144ba837"}, - {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f85b1ffa09240c89aa2e1ae9f3b1c687104f7b2b9d2098da4e923f1b7082d331"}, - {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c6f5fa3775966cca251848d4d5393ab016b3afed251163c1436fefdec3b02c84"}, - {file = "bcrypt-4.2.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:807261df60a8b1ccd13e6599c779014a362ae4e795f5c59747f60208daddd96d"}, - {file = "bcrypt-4.2.1-cp39-abi3-win32.whl", hash = "sha256:b588af02b89d9fad33e5f98f7838bf590d6d692df7153647724a7f20c186f6bf"}, - {file = "bcrypt-4.2.1-cp39-abi3-win_amd64.whl", hash = "sha256:e84e0e6f8e40a242b11bce56c313edc2be121cec3e0ec2d76fce01f6af33c07c"}, - {file = "bcrypt-4.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76132c176a6d9953cdc83c296aeaed65e1a708485fd55abf163e0d9f8f16ce0e"}, - {file = "bcrypt-4.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e158009a54c4c8bc91d5e0da80920d048f918c61a581f0a63e4e93bb556d362f"}, - {file = "bcrypt-4.2.1.tar.gz", hash = "sha256:6765386e3ab87f569b276988742039baab087b2cdb01e809d74e74503c2faafe"}, + {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59e1aa0e2cd871b08ca146ed08445038f42ff75968c7ae50d2fdd7860ade2180"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:0042b2e342e9ae3d2ed22727c1262f76cc4f345683b5c1715f0250cf4277294f"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74a8d21a09f5e025a9a23e7c0fd2c7fe8e7503e4d356c0a2c1486ba010619f09"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:0142b2cb84a009f8452c8c5a33ace5e3dfec4159e7735f5afe9a4d50a8ea722d"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:12fa6ce40cde3f0b899729dbd7d5e8811cb892d31b6f7d0334a1f37748b789fd"}, + {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:5bd3cca1f2aa5dbcf39e2aa13dd094ea181f48959e1071265de49cc2b82525af"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:335a420cfd63fc5bc27308e929bee231c15c85cc4c496610ffb17923abf7f231"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:0e30e5e67aed0187a1764911af023043b4542e70a7461ad20e837e94d23e1d6c"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3b8d62290ebefd49ee0b3ce7500f5dbdcf13b81402c05f6dafab9a1e1b27212f"}, + {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef6630e0ec01376f59a006dc72918b1bf436c3b571b80fa1968d775fa02fe7d"}, + {file = "bcrypt-4.3.0-cp313-cp313t-win32.whl", hash = "sha256:7a4be4cbf241afee43f1c3969b9103a41b40bcb3a3f467ab19f891d9bc4642e4"}, + {file = "bcrypt-4.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c1949bf259a388863ced887c7861da1df681cb2388645766c89fdfd9004c669"}, + {file = "bcrypt-4.3.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:f81b0ed2639568bf14749112298f9e4e2b28853dab50a8b357e31798686a036d"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:864f8f19adbe13b7de11ba15d85d4a428c7e2f344bac110f667676a0ff84924b"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e36506d001e93bffe59754397572f21bb5dc7c83f54454c990c74a468cd589e"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:842d08d75d9fe9fb94b18b071090220697f9f184d4547179b60734846461ed59"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7c03296b85cb87db865d91da79bf63d5609284fc0cab9472fdd8367bbd830753"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:62f26585e8b219cdc909b6a0069efc5e4267e25d4a3770a364ac58024f62a761"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:beeefe437218a65322fbd0069eb437e7c98137e08f22c4660ac2dc795c31f8bb"}, + {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:97eea7408db3a5bcce4a55d13245ab3fa566e23b4c67cd227062bb49e26c585d"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:191354ebfe305e84f344c5964c7cd5f924a3bfc5d405c75ad07f232b6dffb49f"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:41261d64150858eeb5ff43c753c4b216991e0ae16614a308a15d909503617732"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:33752b1ba962ee793fa2b6321404bf20011fe45b9afd2a842139de3011898fef"}, + {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:50e6e80a4bfd23a25f5c05b90167c19030cf9f87930f7cb2eacb99f45d1c3304"}, + {file = "bcrypt-4.3.0-cp38-abi3-win32.whl", hash = "sha256:67a561c4d9fb9465ec866177e7aebcad08fe23aaf6fbd692a6fab69088abfc51"}, + {file = "bcrypt-4.3.0-cp38-abi3-win_amd64.whl", hash = "sha256:584027857bc2843772114717a7490a37f68da563b3620f78a849bcb54dc11e62"}, + {file = "bcrypt-4.3.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0d3efb1157edebfd9128e4e46e2ac1a64e0c1fe46fb023158a407c7892b0f8c3"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08bacc884fd302b611226c01014eca277d48f0a05187666bca23aac0dad6fe24"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6746e6fec103fcd509b96bacdfdaa2fbde9a553245dbada284435173a6f1aef"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:afe327968aaf13fc143a56a3360cb27d4ad0345e34da12c7290f1b00b8fe9a8b"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d9af79d322e735b1fc33404b5765108ae0ff232d4b54666d46730f8ac1a43676"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f1e3ffa1365e8702dc48c8b360fef8d7afeca482809c5e45e653af82ccd088c1"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3004df1b323d10021fda07a813fd33e0fd57bef0e9a480bb143877f6cba996fe"}, + {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:531457e5c839d8caea9b589a1bcfe3756b0547d7814e9ce3d437f17da75c32b0"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:17a854d9a7a476a89dcef6c8bd119ad23e0f82557afbd2c442777a16408e614f"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6fb1fd3ab08c0cbc6826a2e0447610c6f09e983a281b919ed721ad32236b8b23"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e965a9c1e9a393b8005031ff52583cedc15b7884fce7deb8b0346388837d6cfe"}, + {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:79e70b8342a33b52b55d93b3a59223a844962bef479f6a0ea318ebbcadf71505"}, + {file = "bcrypt-4.3.0-cp39-abi3-win32.whl", hash = "sha256:b4d4e57f0a63fd0b358eb765063ff661328f69a04494427265950c71b992a39a"}, + {file = "bcrypt-4.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c950d682f0952bafcceaf709761da0a32a942272fad381081b51096ffa46cea1"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:107d53b5c67e0bbc3f03ebf5b030e0403d24dda980f8e244795335ba7b4a027d"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b693dbb82b3c27a1604a3dff5bfc5418a7e6a781bb795288141e5f80cf3a3492"}, + {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:b6354d3760fcd31994a14c89659dee887f1351a06e5dac3c1142307172a79f90"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a839320bf27d474e52ef8cb16449bb2ce0ba03ca9f44daba6d93fa1d8828e48a"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bdc6a24e754a555d7316fa4774e64c6c3997d27ed2d1964d55920c7c227bc4ce"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:55a935b8e9a1d2def0626c4269db3fcd26728cbff1e84f0341465c31c4ee56d8"}, + {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:57967b7a28d855313a963aaea51bf6df89f833db4320da458e5b3c5ab6d4c938"}, + {file = "bcrypt-4.3.0.tar.gz", hash = "sha256:3a3fd2204178b6d2adcf09cb4f6426ffef54762577a7c9b54c159008cb288c18"}, ] [package.extras] From 60b3cd06507efff71ee60f68a866c43d2f8927ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:48:37 +0000 Subject: [PATCH 196/691] Bump anyhow from 1.0.96 to 1.0.97 (#18201) Bumps [anyhow](https://github.com/dtolnay/anyhow) from 1.0.96 to 1.0.97.
Release notes

Sourced from anyhow's releases.

1.0.97

  • Documentation improvements
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=anyhow&package-manager=cargo&previous-version=1.0.96&new-version=1.0.97)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 477620978c..3b2b932d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "arc-swap" From 929f19b472d87c6c2e15849b557d5647a07fd253 Mon Sep 17 00:00:00 2001 From: karuto Date: Thu, 13 Mar 2025 17:26:22 +0530 Subject: [PATCH 197/691] Fix: corrected routing path for workers doc (#18224) Closes: https://github.com/element-hq/synapse/issues/17926 --- changelog.d/18224.doc | 2 ++ docs/workers.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18224.doc diff --git a/changelog.d/18224.doc b/changelog.d/18224.doc new file mode 100644 index 0000000000..2560d40c26 --- /dev/null +++ b/changelog.d/18224.doc @@ -0,0 +1,2 @@ +Fixed a minor typo in the Synapse documentation. +Contributed by @karuto12. diff --git a/docs/workers.md b/docs/workers.md index 6c614bd6d0..08ee493da9 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -255,7 +255,7 @@ information. ^/_matrix/client/(r0|v3|unstable)/keys/changes$ ^/_matrix/client/(r0|v3|unstable)/keys/claim$ ^/_matrix/client/(r0|v3|unstable)/room_keys/ - ^/_matrix/client/(r0|v3|unstable)/keys/upload/ + ^/_matrix/client/(r0|v3|unstable)/keys/upload$ # Registration/login requests ^/_matrix/client/(api/v1|r0|v3|unstable)/login$ From a278c0d852a01f345936480de0dcfa1292fb4dc7 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 13 Mar 2025 14:52:00 +0000 Subject: [PATCH 198/691] Fix detection of workflow failures in the release script. (#18211) If one workflow is successful and one fails, currently that is reported as success. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18211.misc | 1 + scripts-dev/release.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18211.misc diff --git a/changelog.d/18211.misc b/changelog.d/18211.misc new file mode 100644 index 0000000000..9429b34f62 --- /dev/null +++ b/changelog.d/18211.misc @@ -0,0 +1 @@ +Fix detection of workflow failures in the release script. \ No newline at end of file diff --git a/scripts-dev/release.py b/scripts-dev/release.py index b14b61c705..c556ccaf86 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -592,7 +592,7 @@ def _wait_for_actions(gh_token: Optional[str]) -> None: if all( workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] ): - success = ( + success = all( workflow["status"] == "completed" for workflow in resp["workflow_runs"] ) if success: From 59a15da43364c6b3319983e68f686750a918ec9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 13 Mar 2025 16:28:19 +0000 Subject: [PATCH 199/691] Add caching support to media endpoints (#18235) We do a few things in this PR to better support caching: 1. Change `Cache-Control` header to allow intermediary proxies to cache media *only* if they revalidate on every request. This means that the intermediary cache will still send the request to Synapse but with a `If-None-Match` header, at which point Synapse can check auth and respond with a 304 and empty content. 2. Add `ETag` response header to all media responses. We hardcode this to `1` since all media is immutable (beyond being deleted). 3. Check for `If-None-Match` header (after checking for auth), and if it matches then respond with a 304 and empty body. --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18235.misc | 1 + synapse/media/_base.py | 74 +++++++++++++-- synapse/media/media_repository.py | 17 ++++ synapse/media/thumbnailer.py | 19 ++++ tests/federation/test_federation_media.py | 39 ++++++++ tests/rest/client/test_media.py | 110 ++++++++++++++++++++++ 6 files changed, 253 insertions(+), 7 deletions(-) create mode 100644 changelog.d/18235.misc diff --git a/changelog.d/18235.misc b/changelog.d/18235.misc new file mode 100644 index 0000000000..1c3c42a73c --- /dev/null +++ b/changelog.d/18235.misc @@ -0,0 +1 @@ +Add caching support to media endpoints. diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 7877df62fa..2e48d2fdc7 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -118,6 +118,9 @@ DEFAULT_MAX_TIMEOUT_MS = 20_000 # Maximum allowed timeout_ms for download and thumbnail requests MAXIMUM_ALLOWED_MAX_TIMEOUT_MS = 60_000 +# The ETag header value to use for immutable media. This can be anything. +_IMMUTABLE_ETAG = "1" + def respond_404(request: SynapseRequest) -> None: assert request.path is not None @@ -224,12 +227,7 @@ def add_file_headers( request.setHeader(b"Content-Disposition", disposition.encode("ascii")) - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our - # clients are smart enough to be happy with Cache-Control - request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400") + _add_cache_headers(request) if file_size is not None: request.setHeader(b"Content-Length", b"%d" % (file_size,)) @@ -240,6 +238,26 @@ def add_file_headers( request.setHeader(b"X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex") +def _add_cache_headers(request: Request) -> None: + """Adds the appropriate cache headers to the response""" + + # Cache on the client for at least a day. + # + # We set this to "public,s-maxage=0,proxy-revalidate" to allow CDNs to cache + # the media, so long as they "revalidate" the media on every request. By + # revalidate, we mean send the request to Synapse with a `If-None-Match` + # header, to which Synapse can either respond with a 304 if the user is + # authenticated/authorized, or a 401/403 if they're not. + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=0,proxy-revalidate" + ) + + # Set an ETag header to allow requesters to use it in requests to check if + # the cache is still valid. Since media is immutable (though may be + # deleted), we just set this to a constant. + request.setHeader(b"ETag", _IMMUTABLE_ETAG) + + # separators as defined in RFC2616. SP and HT are handled separately. # see _can_encode_filename_as_token. _FILENAME_SEPARATOR_CHARS = { @@ -336,13 +354,15 @@ async def respond_with_multipart_responder( from synapse.media.media_storage import MultipartFileConsumer + _add_cache_headers(request) + # note that currently the json_object is just {}, this will change when linked media # is implemented multipart_consumer = MultipartFileConsumer( clock, request, media_type, - {}, + {}, # Note: if we change this we need to change the returned ETag. disposition, media_length, ) @@ -419,6 +439,46 @@ async def respond_with_responder( finish_request(request) +def respond_with_304(request: SynapseRequest) -> None: + request.setResponseCode(304) + + # could alternatively use request.notifyFinish() and flip a flag when + # the Deferred fires, but since the flag is RIGHT THERE it seems like + # a waste. + if request._disconnected: + logger.warning( + "Not sending response to request %s, already disconnected.", request + ) + return None + + _add_cache_headers(request) + + request.finish() + + +def check_for_cached_entry_and_respond(request: SynapseRequest) -> bool: + """Check if the request has a conditional header that allows us to return a + 304 Not Modified response, and if it does, return a 304 response. + + This handles clients and intermediary proxies caching media. + This method assumes that the user has already been + authorised to request the media. + + Returns True if we have responded.""" + + # We've checked the user has access to the media, so we now check if it + # is a "conditional request" and we can just return a `304 Not Modified` + # response. Since media is immutable (though may be deleted), we just + # check this is the expected constant. + etag = request.getHeader("If-None-Match") + if etag == _IMMUTABLE_ETAG: + # Return a `304 Not modified`. + respond_with_304(request) + return True + + return False + + class Responder(ABC): """Represents a response that can be streamed to the requester. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index f4d25a7b8b..cf4cba722a 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -52,6 +52,7 @@ from synapse.media._base import ( FileInfo, Responder, ThumbnailInfo, + check_for_cached_entry_and_respond, get_filename_from_headers, respond_404, respond_with_multipart_responder, @@ -459,6 +460,11 @@ class MediaRepository: self.mark_recently_accessed(None, media_id) + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + media_type = media_info.media_type if not media_type: media_type = "application/octet-stream" @@ -538,6 +544,17 @@ class MediaRepository: allow_authenticated, ) + # Check if the media is cached on the client, if so return 304. We need + # to do this after we have fetched remote media, as we need it to do the + # auth. + if check_for_cached_entry_and_respond(request): + # We always need to use the responder. + if responder: + with responder: + pass + + return + # We deliberately stream the file outside the lock if responder and media_info: upload_name = name if name else media_info.upload_name diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index d6b8ce4a09..5d9afda322 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -34,6 +34,7 @@ from synapse.logging.opentracing import trace from synapse.media._base import ( FileInfo, ThumbnailInfo, + check_for_cached_entry_and_respond, respond_404, respond_with_file, respond_with_multipart_responder, @@ -294,6 +295,11 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) await self._select_and_respond_with_thumbnail( request, @@ -334,6 +340,11 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Once we've checked auth we can return early if the media is cached on + # the client + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) for info in thumbnail_infos: t_w = info.width == desired_width @@ -431,6 +442,10 @@ class ThumbnailProvider: respond_404(request) return + # Check if the media is cached on the client, if so return 304. + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id ) @@ -510,6 +525,10 @@ class ThumbnailProvider: if media_info.authenticated: raise NotFoundError() + # Check if the media is cached on the client, if so return 304. + if check_for_cached_entry_and_respond(request): + return + thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id ) diff --git a/tests/federation/test_federation_media.py b/tests/federation/test_federation_media.py index e66aae499b..9c92003ce5 100644 --- a/tests/federation/test_federation_media.py +++ b/tests/federation/test_federation_media.py @@ -147,6 +147,45 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): found_file = any(SMALL_PNG in field for field in stripped_bytes) self.assertTrue(found_file) + def test_federation_etag(self) -> None: + """Test that federation ETags work""" + + content = io.BytesIO(b"file_to_stream") + content_uri = self.get_success( + self.media_repo.create_content( + "text/plain", + "test_upload", + content, + 46, + UserID.from_string("@user_id:whatever.org"), + ) + ) + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/download/{content_uri.media_id}", + ) + self.pump() + self.assertEqual(200, channel.code) + + # We expect exactly one ETag header. + etags = channel.headers.getRawHeaders("ETag") + self.assertIsNotNone(etags) + assert etags is not None # For mypy + self.assertEqual(len(etags), 1) + etag = etags[0] + + # Refetching with the etag should result in 304 and empty body. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/download/{content_uri.media_id}", + custom_headers=[("If-None-Match", etag)], + ) + self.pump() + self.assertEqual(channel.code, 304) + self.assertEqual(channel.is_finished(), True) + self.assertNotIn("body", channel.result) + class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 7d7dbd342b..0e3e370ee8 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -2676,3 +2676,113 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): access_token=self.tok, ) self.assertEqual(channel10.code, 200) + + def test_authenticated_media_etag(self) -> None: + """Test that ETag works correctly with authenticated media over client + APIs""" + + # upload some local media with authentication on + channel = self.make_request( + "POST", + "_matrix/media/v3/upload?filename=test_png_upload", + SMALL_PNG, + self.tok, + shorthand=False, + content_type=b"image/png", + custom_headers=[("Content-Length", str(67))], + ) + self.assertEqual(channel.code, 200) + res = channel.json_body.get("content_uri") + assert res is not None + uri = res.split("mxc://")[1] + + # Check standard media endpoint + self._check_caching(f"/download/{uri}") + + # check thumbnails as well + params = "?width=32&height=32&method=crop" + self._check_caching(f"/thumbnail/{uri}{params}") + + # Inject a piece of remote media. + file_id = "abcdefg12345" + file_info = FileInfo(server_name="lonelyIsland", file_id=file_id) + + media_storage = self.hs.get_media_repository().media_storage + + ctx = media_storage.store_into_file(file_info) + (f, fname) = self.get_success(ctx.__aenter__()) + f.write(SMALL_PNG) + self.get_success(ctx.__aexit__(None, None, None)) + + # we write the authenticated status when storing media, so this should pick up + # config and authenticate the media + self.get_success( + self.store.store_cached_remote_media( + origin="lonelyIsland", + media_id="52", + media_type="image/png", + media_length=1, + time_now_ms=self.clock.time_msec(), + upload_name="remote_test.png", + filesystem_id=file_id, + ) + ) + + # ensure we have thumbnails for the non-dynamic code path + if self.extra_config == {"dynamic_thumbnails": False}: + self.get_success( + self.repo._generate_thumbnails( + "lonelyIsland", "52", file_id, "image/png" + ) + ) + + self._check_caching("/download/lonelyIsland/52") + + params = "?width=32&height=32&method=crop" + self._check_caching(f"/thumbnail/lonelyIsland/52{params}") + + def _check_caching(self, path: str) -> None: + """ + Checks that: + 1. fetching the path returns an ETag header + 2. refetching with the ETag returns a 304 without a body + 3. refetching with the ETag but through unauthenticated endpoint + returns 404 + """ + + # Request media over authenticated endpoint, should be found + channel1 = self.make_request( + "GET", + f"/_matrix/client/v1/media{path}", + access_token=self.tok, + shorthand=False, + ) + self.assertEqual(channel1.code, 200) + + # Should have a single ETag field + etags = channel1.headers.getRawHeaders("ETag") + self.assertIsNotNone(etags) + assert etags is not None # For mypy + self.assertEqual(len(etags), 1) + etag = etags[0] + + # Refetching with the etag should result in 304 and empty body. + channel2 = self.make_request( + "GET", + f"/_matrix/client/v1/media{path}", + access_token=self.tok, + shorthand=False, + custom_headers=[("If-None-Match", etag)], + ) + self.assertEqual(channel2.code, 304) + self.assertEqual(channel2.is_finished(), True) + self.assertNotIn("body", channel2.result) + + # Refetching with the etag but no access token should result in 404. + channel3 = self.make_request( + "GET", + f"/_matrix/media/r0{path}", + shorthand=False, + custom_headers=[("If-None-Match", etag)], + ) + self.assertEqual(channel3.code, 404) From 51df675c054369576ed9bde8d1865c904fa6056c Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Mon, 17 Mar 2025 12:21:45 -0400 Subject: [PATCH 200/691] MSC4140: don't cancel delayed state on own state (#17810) When a user sends a state event, do not cancel their own delayed events for the same piece of state. For context, see [the relevant section in the MSC](https://github.com/matrix-org/matrix-spec-proposals/blob/a09a883d9a013ac4b6ffddebd7ea87a827d211b9/proposals/4140-delayed-events-futures.md#delayed-state-events-are-cancelled-by-a-more-recent-state-event). --- changelog.d/17810.feature | 1 + synapse/handlers/delayed_events.py | 20 ++- .../storage/databases/main/delayed_events.py | 30 ++-- tests/rest/client/test_delayed_events.py | 143 ++++++++++++++---- 4 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 changelog.d/17810.feature diff --git a/changelog.d/17810.feature b/changelog.d/17810.feature new file mode 100644 index 0000000000..5c65e54ceb --- /dev/null +++ b/changelog.d/17810.feature @@ -0,0 +1 @@ +Update MSC4140 implementation to no longer cancel a user's own delayed state events with an event type & state key that match a more recent state event sent by that user. diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index b3f40809a1..80cb1cec9b 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -191,18 +191,36 @@ class DelayedEventsHandler: async def _handle_state_deltas(self, deltas: List[StateDelta]) -> None: """ - Process current state deltas to cancel pending delayed events + Process current state deltas to cancel other users' pending delayed events that target the same state. """ for delta in deltas: + if delta.event_id is None: + logger.debug( + "Not handling delta for deleted state: %r %r", + delta.event_type, + delta.state_key, + ) + continue + logger.debug( "Handling: %r %r, %s", delta.event_type, delta.state_key, delta.event_id ) + event = await self._store.get_event( + delta.event_id, check_room_id=delta.room_id + ) + sender = UserID.from_string(event.sender) + next_send_ts = await self._store.cancel_delayed_state_events( room_id=delta.room_id, event_type=delta.event_type, state_key=delta.state_key, + not_from_localpart=( + sender.localpart + if sender.domain == self._config.server.server_name + else "" + ), ) if self._next_send_ts_changed(next_send_ts): diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1616e30e22..c88682d55c 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -424,25 +424,37 @@ class DelayedEventsStore(SQLBaseStore): room_id: str, event_type: str, state_key: str, + not_from_localpart: str, ) -> Optional[Timestamp]: """ Cancels all matching delayed state events, i.e. remove them as long as they haven't been processed. + Args: + room_id: The room ID to match against. + event_type: The event type to match against. + state_key: The state key to match against. + not_from_localpart: The localpart of a user whose delayed events to not cancel. + If set to the empty string, any users' delayed events may be cancelled. + Returns: The send time of the next delayed event to be sent, if any. """ def cancel_delayed_state_events_txn( txn: LoggingTransaction, ) -> Optional[Timestamp]: - self.db_pool.simple_delete_txn( - txn, - table="delayed_events", - keyvalues={ - "room_id": room_id, - "event_type": event_type, - "state_key": state_key, - "is_processed": False, - }, + txn.execute( + """ + DELETE FROM delayed_events + WHERE room_id = ? AND event_type = ? AND state_key = ? + AND user_localpart <> ? + AND NOT is_processed + """, + ( + room_id, + event_type, + state_key, + not_from_localpart, + ), ) return self._get_next_delayed_event_send_ts_txn(txn) diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 2c938390c8..9f9d241f12 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -22,7 +22,8 @@ from parameterized import parameterized from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import Codes -from synapse.rest.client import delayed_events, room, versions +from synapse.rest import admin +from synapse.rest.client import delayed_events, login, room, versions from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util import Clock @@ -32,7 +33,6 @@ from tests.unittest import HomeserverTestCase PATH_PREFIX = "/_matrix/client/unstable/org.matrix.msc4140/delayed_events" -_HS_NAME = "red" _EVENT_TYPE = "com.example.test" @@ -54,23 +54,41 @@ class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): class DelayedEventsTestCase(HomeserverTestCase): """Tests getting and managing delayed events.""" - servlets = [delayed_events.register_servlets, room.register_servlets] - user_id = f"@sid1:{_HS_NAME}" + servlets = [ + admin.register_servlets, + delayed_events.register_servlets, + login.register_servlets, + room.register_servlets, + ] def default_config(self) -> JsonDict: config = super().default_config() - config["server_name"] = _HS_NAME config["max_event_delay_duration"] = "24h" return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user1_user_id = self.register_user("user1", "pass") + self.user1_access_token = self.login("user1", "pass") + self.user2_user_id = self.register_user("user2", "pass") + self.user2_access_token = self.login("user2", "pass") + self.room_id = self.helper.create_room_as( - self.user_id, + self.user1_user_id, + tok=self.user1_access_token, extra_content={ - "preset": "trusted_private_chat", + "preset": "public_chat", + "power_level_content_override": { + "events": { + _EVENT_TYPE: 0, + } + }, }, ) + self.helper.join( + room=self.room_id, user=self.user2_user_id, tok=self.user2_access_token + ) + def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) @@ -85,6 +103,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) events = self._get_delayed_events() @@ -94,7 +113,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -104,7 +123,7 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) @@ -113,7 +132,7 @@ class DelayedEventsTestCase(HomeserverTestCase): {"rc_delayed_event_mgmt": {"per_second": 0.5, "burst_count": 1}} ) def test_get_delayed_events_ratelimit(self) -> None: - args = ("GET", PATH_PREFIX) + args = ("GET", PATH_PREFIX, b"", self.user1_access_token) channel = self.make_request(*args) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -123,7 +142,9 @@ class DelayedEventsTestCase(HomeserverTestCase): # Add the current user to the ratelimit overrides, allowing them no ratelimiting. self.get_success( - self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) ) # Test that the request isn't ratelimited anymore. @@ -134,6 +155,7 @@ class DelayedEventsTestCase(HomeserverTestCase): channel = self.make_request( "POST", f"{PATH_PREFIX}/", + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, channel.result) @@ -141,6 +163,7 @@ class DelayedEventsTestCase(HomeserverTestCase): channel = self.make_request( "POST", f"{PATH_PREFIX}/abc", + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -153,6 +176,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -165,6 +189,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {"action": "oops"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( @@ -178,6 +203,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/abc", {"action": action}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, channel.result) @@ -192,6 +218,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -205,7 +232,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -214,6 +241,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "cancel"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertListEqual([], self._get_delayed_events()) @@ -222,7 +250,7 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -237,6 +265,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), {}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -247,6 +276,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "cancel"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -254,13 +284,16 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "cancel"}, + self.user1_access_token, ) channel = self.make_request(*args) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. self.get_success( - self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) ) # Test that the request isn't ratelimited anymore. @@ -278,6 +311,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -291,7 +325,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -300,13 +334,14 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "send"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) self.assertListEqual([], self._get_delayed_events()) content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) @@ -319,6 +354,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), {}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -329,6 +365,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "send"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -336,13 +373,16 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "send"}, + self.user1_access_token, ) channel = self.make_request(*args) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. self.get_success( - self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) ) # Test that the request isn't ratelimited anymore. @@ -360,6 +400,7 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: setter_expected, }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -373,7 +414,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -382,6 +423,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_id}", {"action": "restart"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -393,7 +435,7 @@ class DelayedEventsTestCase(HomeserverTestCase): self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, expect_code=HTTPStatus.NOT_FOUND, ) @@ -403,7 +445,7 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) @@ -418,6 +460,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), {}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) delay_id = channel.json_body.get("delay_id") @@ -428,6 +471,7 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "restart"}, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -435,21 +479,66 @@ class DelayedEventsTestCase(HomeserverTestCase): "POST", f"{PATH_PREFIX}/{delay_ids.pop(0)}", {"action": "restart"}, + self.user1_access_token, ) channel = self.make_request(*args) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. self.get_success( - self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) ) # Test that the request isn't ratelimited anymore. channel = self.make_request(*args) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - def test_delayed_state_events_are_cancelled_by_more_recent_state(self) -> None: - state_key = "to_be_cancelled" + def test_delayed_state_is_not_cancelled_by_new_state_from_same_user( + self, + ) -> None: + state_key = "to_not_be_cancelled_by_same_user" + + setter_key = "setter" + setter_expected = "on_timeout" + channel = self.make_request( + "PUT", + _get_path_for_delayed_state(self.room_id, _EVENT_TYPE, state_key, 900), + { + setter_key: setter_expected, + }, + self.user1_access_token, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + events = self._get_delayed_events() + self.assertEqual(1, len(events), events) + + self.helper.send_state( + self.room_id, + _EVENT_TYPE, + { + setter_key: "manual", + }, + self.user1_access_token, + state_key=state_key, + ) + events = self._get_delayed_events() + self.assertEqual(1, len(events), events) + + self.reactor.advance(1) + content = self.helper.get_state( + self.room_id, + _EVENT_TYPE, + self.user1_access_token, + state_key=state_key, + ) + self.assertEqual(setter_expected, content.get(setter_key), content) + + def test_delayed_state_is_cancelled_by_new_state_from_other_user( + self, + ) -> None: + state_key = "to_be_cancelled_by_other_user" setter_key = "setter" channel = self.make_request( @@ -458,19 +547,20 @@ class DelayedEventsTestCase(HomeserverTestCase): { setter_key: "on_timeout", }, + self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) events = self._get_delayed_events() self.assertEqual(1, len(events), events) - setter_expected = "manual" + setter_expected = "other_user" self.helper.send_state( self.room_id, _EVENT_TYPE, { setter_key: setter_expected, }, - None, + self.user2_access_token, state_key=state_key, ) self.assertListEqual([], self._get_delayed_events()) @@ -479,7 +569,7 @@ class DelayedEventsTestCase(HomeserverTestCase): content = self.helper.get_state( self.room_id, _EVENT_TYPE, - "", + self.user1_access_token, state_key=state_key, ) self.assertEqual(setter_expected, content.get(setter_key), content) @@ -488,6 +578,7 @@ class DelayedEventsTestCase(HomeserverTestCase): channel = self.make_request( "GET", PATH_PREFIX, + access_token=self.user1_access_token, ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) From 79081e1be5fb314e7a2f2f0ed606cb9ac30927e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:00:57 +0000 Subject: [PATCH 201/691] Bump http from 1.2.0 to 1.3.1 (#18245) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b2b932d89..c6414a76c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,9 +173,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", From 4d2c4ce92b83f97b939d3ca5d8599f750c238671 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:01:09 +0000 Subject: [PATCH 202/691] Bump ulid from 1.2.0 to 1.2.1 (#18246) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6414a76c5..f53d029cbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,9 +544,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ulid" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab82fc73182c29b02e2926a6df32f2241dbadb5cfc111fd595515b3598f46bb3" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ "rand", "web-time", From d61bdff7a445e6c9d345c23917ba920eb737717d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Mar 2025 05:53:21 -0500 Subject: [PATCH 203/691] Remove `SYNAPSE_USE_FROZEN_DICTS` environment variable (#18123) I got rid of the `SYNAPSE_USE_FROZEN_DICTS` environment variable because it will be overridden by the Synapse worker apps anyway and if we want to support `SYNAPSE_USE_FROZEN_DICTS`, it should be in `synapse/config/server.py`. It's also not documented so I'm assuming no one is using it anyway. Spawning from looking at the frozen dict stuff during the review of https://github.com/element-hq/synapse/pull/18103#discussion_r1935876168 --- changelog.d/18123.misc | 1 + synapse/events/__init__.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 changelog.d/18123.misc diff --git a/changelog.d/18123.misc b/changelog.d/18123.misc new file mode 100644 index 0000000000..5ae23eb07d --- /dev/null +++ b/changelog.d/18123.misc @@ -0,0 +1 @@ +Remove undocumented `SYNAPSE_USE_FROZEN_DICTS` environment variable. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 13d41592b3..a85e66d6bf 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -22,7 +22,6 @@ import abc import collections.abc -import os from typing import ( TYPE_CHECKING, Any, @@ -48,21 +47,21 @@ from synapse.synapse_rust.events import EventInternalMetadata from synapse.types import JsonDict, StrCollection from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze -from synapse.util.stringutils import strtobool if TYPE_CHECKING: from synapse.events.builder import EventBuilder -# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents -# bugs where we accidentally share e.g. signature dicts. However, converting a -# dict to frozen_dicts is expensive. -# -# NOTE: This is overridden by the configuration by the Synapse worker apps, but -# for the sake of tests, it is set here while it cannot be configured on the -# homeserver object itself. -USE_FROZEN_DICTS = strtobool(os.environ.get("SYNAPSE_USE_FROZEN_DICTS", "0")) +USE_FROZEN_DICTS = False +""" +Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents +bugs where we accidentally share e.g. signature dicts. However, converting a +dict to frozen_dicts is expensive. +NOTE: This is overridden by the configuration by the Synapse worker apps, but +for the sake of tests, it is set here because it cannot be configured on the +homeserver object itself. +""" T = TypeVar("T") From bfafd0f2c7125f63e4ab7ca994b3249d757290dc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 18 Mar 2025 13:30:45 +0000 Subject: [PATCH 204/691] 1.127.0rc1 --- CHANGES.md | 29 +++++++++++++++++++++++++++++ changelog.d/17810.feature | 1 - changelog.d/18123.misc | 1 - changelog.d/18211.misc | 1 - changelog.d/18224.doc | 2 -- changelog.d/18235.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 8 files changed, 36 insertions(+), 7 deletions(-) delete mode 100644 changelog.d/17810.feature delete mode 100644 changelog.d/18123.misc delete mode 100644 changelog.d/18211.misc delete mode 100644 changelog.d/18224.doc delete mode 100644 changelog.d/18235.misc diff --git a/CHANGES.md b/CHANGES.md index 1cb3e7da13..3053724c9d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,32 @@ +# Synapse 1.127.0rc1 (2025-03-18) + +### Features + +- Update [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140) implementation to no longer cancel a user's own delayed state events with an event type & state key that match a more recent state event sent by that user. ([\#17810](https://github.com/element-hq/synapse/issues/17810)) + +### Improved Documentation + +- Fixed a minor typo in the Synapse documentation. Contributed by @karuto12. ([\#18224](https://github.com/element-hq/synapse/issues/18224)) + +### Internal Changes + +- Remove undocumented `SYNAPSE_USE_FROZEN_DICTS` environment variable. ([\#18123](https://github.com/element-hq/synapse/issues/18123)) +- Fix detection of workflow failures in the release script. ([\#18211](https://github.com/element-hq/synapse/issues/18211)) +- Add caching support to media endpoints. ([\#18235](https://github.com/element-hq/synapse/issues/18235)) + + + +### Updates to locked dependencies + +* Bump anyhow from 1.0.96 to 1.0.97. ([\#18201](https://github.com/element-hq/synapse/issues/18201)) +* Bump bcrypt from 4.2.1 to 4.3.0. ([\#18207](https://github.com/element-hq/synapse/issues/18207)) +* Bump bytes from 1.10.0 to 1.10.1. ([\#18227](https://github.com/element-hq/synapse/issues/18227)) +* Bump http from 1.2.0 to 1.3.1. ([\#18245](https://github.com/element-hq/synapse/issues/18245)) +* Bump sentry-sdk from 2.19.2 to 2.22.0. ([\#18205](https://github.com/element-hq/synapse/issues/18205)) +* Bump serde from 1.0.218 to 1.0.219. ([\#18228](https://github.com/element-hq/synapse/issues/18228)) +* Bump serde_json from 1.0.139 to 1.0.140. ([\#18202](https://github.com/element-hq/synapse/issues/18202)) +* Bump ulid from 1.2.0 to 1.2.1. ([\#18246](https://github.com/element-hq/synapse/issues/18246)) + # Synapse 1.126.0 (2025-03-11) Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.126/docs/upgrade.md#change-of-signing-key-expiry-date-for-the-debianubuntu-package-repository) diff --git a/changelog.d/17810.feature b/changelog.d/17810.feature deleted file mode 100644 index 5c65e54ceb..0000000000 --- a/changelog.d/17810.feature +++ /dev/null @@ -1 +0,0 @@ -Update MSC4140 implementation to no longer cancel a user's own delayed state events with an event type & state key that match a more recent state event sent by that user. diff --git a/changelog.d/18123.misc b/changelog.d/18123.misc deleted file mode 100644 index 5ae23eb07d..0000000000 --- a/changelog.d/18123.misc +++ /dev/null @@ -1 +0,0 @@ -Remove undocumented `SYNAPSE_USE_FROZEN_DICTS` environment variable. diff --git a/changelog.d/18211.misc b/changelog.d/18211.misc deleted file mode 100644 index 9429b34f62..0000000000 --- a/changelog.d/18211.misc +++ /dev/null @@ -1 +0,0 @@ -Fix detection of workflow failures in the release script. \ No newline at end of file diff --git a/changelog.d/18224.doc b/changelog.d/18224.doc deleted file mode 100644 index 2560d40c26..0000000000 --- a/changelog.d/18224.doc +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a minor typo in the Synapse documentation. -Contributed by @karuto12. diff --git a/changelog.d/18235.misc b/changelog.d/18235.misc deleted file mode 100644 index 1c3c42a73c..0000000000 --- a/changelog.d/18235.misc +++ /dev/null @@ -1 +0,0 @@ -Add caching support to media endpoints. diff --git a/debian/changelog b/debian/changelog index 69d19fa048..c50249f081 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.127.0~rc1) stable; urgency=medium + + * New Synapse release 1.127.0rc1. + + -- Synapse Packaging team Tue, 18 Mar 2025 13:30:05 +0000 + matrix-synapse-py3 (1.126.0) stable; urgency=medium * New Synapse release 1.126.0. diff --git a/pyproject.toml b/pyproject.toml index 27a4da7e72..bc0f526390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.126.0" +version = "1.127.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 4b8dbe22c0c75a3ed3e0630e438c97edb4d4ca86 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 18 Mar 2025 10:59:57 -0700 Subject: [PATCH 205/691] Add a column `participant` to `room_memberships` table (#18068) --- changelog.d/18068.misc | 1 + synapse/_scripts/synapse_port_db.py | 1 + synapse/handlers/message.py | 6 + synapse/storage/databases/main/roommember.py | 148 ++++++++++++++ synapse/storage/schema/__init__.py | 5 +- ...umn_participant_room_memberships_table.sql | 20 ++ tests/rest/client/test_rooms.py | 193 ++++++++++++++++++ 7 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18068.misc create mode 100644 synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql diff --git a/changelog.d/18068.misc b/changelog.d/18068.misc new file mode 100644 index 0000000000..af6f78f549 --- /dev/null +++ b/changelog.d/18068.misc @@ -0,0 +1 @@ +Add a column `participant` to `room_memberships` table. \ No newline at end of file diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 3f67a739a0..1bb9940180 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -128,6 +128,7 @@ BOOLEAN_COLUMNS = { "pushers": ["enabled"], "redactions": ["have_censored"], "remote_media_cache": ["authenticated"], + "room_memberships": ["participant"], "room_stats_state": ["is_federatable"], "rooms": ["is_public", "has_auth_chain_index"], "sliding_sync_joined_rooms": ["is_encrypted"], diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4642b8b578..52c61cfa54 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1462,6 +1462,12 @@ class EventCreationHandler: ) return prev_event + if not event.is_state() and event.type in [ + EventTypes.Message, + EventTypes.Encrypted, + ]: + await self.store.set_room_participation(event.user_id, event.room_id) + if event.internal_metadata.is_out_of_band_membership(): # the only sort of out-of-band-membership events we expect to see here are # invite rejections and rescinded knocks that we have generated ourselves. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 50ed6a28bf..a0a6dcd04e 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -79,6 +79,7 @@ logger = logging.getLogger(__name__) _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update" _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership" +_POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE = 1000 @attr.s(frozen=True, slots=True, auto_attribs=True) @@ -1606,6 +1607,66 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): from_ts, ) + async def set_room_participation(self, user_id: str, room_id: str) -> None: + """ + Record the provided user as participating in the given room + + Args: + user_id: the user ID of the user + room_id: ID of the room to set the participant in + """ + + def _set_room_participation_txn( + txn: LoggingTransaction, user_id: str, room_id: str + ) -> None: + sql = """ + UPDATE room_memberships + SET participant = true + WHERE (user_id, room_id) IN ( + SELECT user_id, room_id + FROM room_memberships + WHERE user_id = ? + AND room_id = ? + ORDER BY event_stream_ordering DESC + LIMIT 1 + ) + """ + txn.execute(sql, (user_id, room_id)) + + await self.db_pool.runInteraction( + "_set_room_participation_txn", _set_room_participation_txn, user_id, room_id + ) + + async def get_room_participation(self, user_id: str, room_id: str) -> bool: + """ + Check whether a user is listed as a participant in a room + + Args: + user_id: user ID of the user + room_id: ID of the room to check in + """ + + def _get_room_participation_txn( + txn: LoggingTransaction, user_id: str, room_id: str + ) -> bool: + sql = """ + SELECT participant + FROM room_memberships + WHERE user_id = ? + AND room_id = ? + ORDER BY event_stream_ordering DESC + LIMIT 1 + """ + txn.execute(sql, (user_id, room_id)) + res = txn.fetchone() + if res: + return res[0] + return False + + return await self.db_pool.runInteraction( + "_get_room_participation_txn", _get_room_participation_txn, user_id, room_id + ) + class RoomMemberBackgroundUpdateStore(SQLBaseStore): def __init__( @@ -1636,6 +1697,93 @@ class RoomMemberBackgroundUpdateStore(SQLBaseStore): columns=["user_id", "room_id"], ) + self.db_pool.updates.register_background_update_handler( + "populate_participant_bg_update", self._populate_participant + ) + + async def _populate_participant(self, progress: JsonDict, batch_size: int) -> int: + """ + Background update to populate column `participant` on `room_memberships` table + + A 'participant' is someone who is currently joined to a room and has sent at least + one `m.room.message` or `m.room.encrypted` event. + + This background update will set the `participant` column across all rows in + `room_memberships` based on the user's *current* join status, and if + they've *ever* sent a message or encrypted event. Therefore one should + never assume the `participant` column's value is based solely on whether + the user participated in a previous "session" (where a "session" is defined + as a period between the user joining and leaving). See + https://github.com/element-hq/synapse/pull/18068#discussion_r1931070291 + for further detail. + """ + stream_token = progress.get("last_stream_token", None) + + def _get_max_stream_token_txn(txn: LoggingTransaction) -> int: + sql = """ + SELECT event_stream_ordering from room_memberships + ORDER BY event_stream_ordering DESC + LIMIT 1; + """ + txn.execute(sql) + res = txn.fetchone() + if not res or not res[0]: + return 0 + return res[0] + + def _background_populate_participant_txn( + txn: LoggingTransaction, stream_token: str + ) -> None: + sql = """ + UPDATE room_memberships + SET participant = True + FROM ( + SELECT DISTINCT c.state_key, e.room_id + FROM current_state_events AS c + INNER JOIN events AS e ON c.room_id = e.room_id + WHERE c.membership = 'join' + AND c.state_key = e.sender + AND ( + e.type = 'm.room.message' + OR e.type = 'm.room.encrypted' + ) + ) AS subquery + WHERE room_memberships.user_id = subquery.state_key + AND room_memberships.room_id = subquery.room_id + AND room_memberships.event_stream_ordering <= ? + AND room_memberships.event_stream_ordering > ?; + """ + batch = int(stream_token) - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE + txn.execute(sql, (stream_token, batch)) + + if stream_token is None: + stream_token = await self.db_pool.runInteraction( + "_get_max_stream_token", _get_max_stream_token_txn + ) + + if stream_token < 0: + await self.db_pool.updates._end_background_update( + "populate_participant_bg_update" + ) + return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE + + await self.db_pool.runInteraction( + "_background_populate_participant_txn", + _background_populate_participant_txn, + stream_token, + ) + + progress["last_stream_token"] = ( + stream_token - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE + ) + await self.db_pool.runInteraction( + "populate_participant_bg_update", + self.db_pool.updates._background_update_progress_txn, + "populate_participant_bg_update", + progress, + ) + return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE + async def _background_add_membership_profile( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 49e648a92f..f87b1a4a0a 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 89 # remember to update the list below when updating +SCHEMA_VERSION = 90 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -158,6 +158,9 @@ Changes in SCHEMA_VERSION = 88 Changes in SCHEMA_VERSION = 89 - Add `state_groups_pending_deletion` and `state_groups_persisting` tables. + +Changes in SCHEMA_VERSION = 90 + - Add a column `participant` to `room_memberships` table """ diff --git a/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql new file mode 100644 index 0000000000..672be1031e --- /dev/null +++ b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql @@ -0,0 +1,20 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a column `participant` to `room_memberships` table to track whether a room member has sent +-- a `m.room.message` or `m.room.encrypted` event into a room they are a member of +ALTER TABLE room_memberships ADD COLUMN participant BOOLEAN DEFAULT FALSE; + +-- Add a background update to populate `participant` column +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9001, 'populate_participant_bg_update', '{}'); \ No newline at end of file diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index dd8350ddd1..6c93ead3b8 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -4208,3 +4208,196 @@ class UserSuspensionTests(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(channel.code, 200) + + +class RoomParticipantTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + room.register_servlets, + profile.register_servlets, + admin.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user1 = self.register_user("thomas", "hackme") + self.tok1 = self.login("thomas", "hackme") + + self.user2 = self.register_user("teresa", "hackme") + self.tok2 = self.login("teresa", "hackme") + + self.room1 = self.helper.create_room_as( + room_creator=self.user1, + tok=self.tok1, + # Allow user2 to send state events into the room. + extra_content={ + "power_level_content_override": { + "state_default": 0, + }, + }, + ) + self.store = hs.get_datastores().main + + @parameterized.expand( + [ + # Should record participation. + param( + is_state=False, + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + record_participation=True, + ), + param( + is_state=False, + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + record_participation=True, + ), + # Should not record participation. + param( + is_state=False, + event_type="m.sticker", + event_content={ + "body": "My great sticker", + "info": {}, + "url": "mxc://unused/mxcurl", + }, + record_participation=False, + ), + # An invalid **state event** with type `m.room.message` + param( + is_state=True, + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + record_participation=False, + ), + # An invalid **state event** with type `m.room.encrypted` + # Note: this may become valid in the future with encrypted state, though we + # still may not want to consider it grounds for marking a user as participating. + param( + is_state=True, + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + record_participation=False, + ), + ] + ) + def test_sending_message_records_participation( + self, + is_state: bool, + event_type: str, + event_content: JsonDict, + record_participation: bool, + ) -> None: + """ + Test that sending an various events into a room causes the user to + appropriately marked or not marked as a participant in that room. + """ + self.helper.join(self.room1, self.user2, tok=self.tok2) + + # user has not sent any messages, so should not be a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) + + # send an event into the room + if is_state: + # send a state event + self.helper.send_state( + self.room1, + event_type, + body=event_content, + tok=self.tok2, + ) + else: + # send a non-state event + self.helper.send_event( + self.room1, + event_type, + content=event_content, + tok=self.tok2, + ) + + # check whether the user has been marked as a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertEqual(participant, record_participation) + + @parameterized.expand( + [ + param( + event_type="m.room.message", + event_content={ + "msgtype": "m.text", + "body": "I am engaging in this room", + }, + ), + param( + event_type="m.room.encrypted", + event_content={ + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + }, + ), + ] + ) + def test_sending_event_and_leaving_does_not_record_participation( + self, + event_type: str, + event_content: JsonDict, + ) -> None: + """ + Test that sending an event into a room that should mark a user as a + participant, but then leaving the room, results in the user no longer + be marked as a participant in that room. + """ + self.helper.join(self.room1, self.user2, tok=self.tok2) + + # user has not sent any messages, so should not be a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) + + # sending a message should now mark user as participant + self.helper.send_event( + self.room1, + event_type, + content=event_content, + tok=self.tok2, + ) + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertTrue(participant) + + # leave the room + self.helper.leave(self.room1, self.user2, tok=self.tok2) + + # user should no longer be considered a participant + participant = self.get_success( + self.store.get_room_participation(self.user2, self.room1) + ) + self.assertFalse(participant) From 47e295bf3a0ec62b6a9ae675414a34dd25a6b589 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 18 Mar 2025 18:38:18 +0000 Subject: [PATCH 206/691] Add index to sliding sync membership snapshot table, to fix a performance issue. (#18074) To address a performance problem due to the foreign key on the same column. cc @erikjohnston --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18074.bugfix | 1 + synapse/storage/databases/main/sliding_sync.py | 9 ++++++++- .../01_sliding_sync_membership_snapshot_index.sql | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18074.bugfix create mode 100644 synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql diff --git a/changelog.d/18074.bugfix b/changelog.d/18074.bugfix new file mode 100644 index 0000000000..03e0b95988 --- /dev/null +++ b/changelog.d/18074.bugfix @@ -0,0 +1 @@ +Add index to sliding sync membership snapshot table, to fix a performance issue. \ No newline at end of file diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 874dfdcb77..a287fd2a3f 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -1,7 +1,7 @@ # # This file is licensed under the Affero General Public License (AGPL) version 3. # -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -61,6 +61,13 @@ class SlidingSyncStore(SQLBaseStore): columns=("required_state_id",), ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_membership_snapshots_membership_event_id_idx", + index_name="sliding_sync_membership_snapshots_membership_event_id_idx", + table="sliding_sync_membership_snapshots", + columns=("membership_event_id",), + ) + async def get_latest_bump_stamp_for_room( self, room_id: str, diff --git a/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql b/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql new file mode 100644 index 0000000000..7799cffdce --- /dev/null +++ b/synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8901, 'sliding_sync_membership_snapshots_membership_event_id_idx', '{}'); From 51deadec41979cd94e16b5d1e42a47fedc3da98c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:16:04 +0000 Subject: [PATCH 207/691] Pin our GitHub Actions dependencies (#18255) After the [recent supply chain attack](https://www.wiz.io/blog/new-github-action-supply-chain-attack-reviewdog-action-setup) in `tj-actions/changed-files` and actions based on it, it's become clear that relying on git tags to pin our dependencies is not enough (as tags can simply be replaced). Therefore we need to switch to hashes. Dependabot should continue to update these dependencies for us. Best reviewed commit-by-commit. Though if CI passes, we're *probably* fine. --- .github/workflows/docker.yml | 16 +-- .github/workflows/docs-pr-netlify.yaml | 2 +- .github/workflows/docs-pr.yaml | 8 +- .github/workflows/docs.yaml | 4 +- .github/workflows/fix_lint.yaml | 10 +- .github/workflows/latest_deps.yml | 32 ++--- .github/workflows/poetry_lockfile.yaml | 4 +- .github/workflows/push_complement_image.yml | 10 +- .github/workflows/release-artifacts.yml | 32 ++--- .github/workflows/tests.yml | 136 ++++++++++---------- .github/workflows/triage-incoming.yml | 2 +- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 30 ++--- changelog.d/18255.misc | 1 + 14 files changed, 145 insertions(+), 144 deletions(-) create mode 100644 changelog.d/18255.misc diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e8319364dd..052dcf800b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,22 +18,22 @@ jobs: steps: - name: Set up QEMU id: qemu - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 with: platforms: arm64 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - name: Inspect builder run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@v3.8.1 + uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1 - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see @@ -43,13 +43,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -57,7 +57,7 @@ jobs: - name: Calculate docker image tag id: set-tag - uses: docker/metadata-action@master + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 with: images: | docker.io/matrixdotorg/synapse @@ -72,7 +72,7 @@ jobs: - name: Build and push all platforms id: build-and-push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0 with: push: true labels: | diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 0fbf6e02b7..d1b789c8d0 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -22,7 +22,7 @@ jobs: path: book - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@v3 + uses: matrix-org/netlify-pr-preview@9805cd123fc9a7e421e35340a05e1ebc5dee46b5 # v3 with: path: book owner: ${{ github.event.workflow_run.head_repository.owner.login }} diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 07dc301b1a..1b416407d8 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.4.17' - name: Setup python - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" @@ -39,7 +39,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: book path: book @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 434dcbb6c7..121c29571a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index d6aed83774..3dcda1bab9 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -13,19 +13,19 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: # We use nightly so that `fmt` correctly groups together imports, and # clippy correctly fixes up the benchmarks. toolchain: nightly-2022-12-01 components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: install-project: "false" @@ -43,6 +43,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@v5 + - uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # v5.1.0 with: commit_message: "Attempt to fix linting" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 3884b6d402..740c83758f 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -39,14 +39,14 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: "3.x" poetry-version: "1.3.2" @@ -72,11 +72,11 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -86,7 +86,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v5 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - run: pip install .[all,test] @@ -145,11 +145,11 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` @@ -164,7 +164,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -192,15 +192,15 @@ jobs: database: Postgres steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + - name: Check out synapse codebase + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: synapse - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -225,7 +225,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 496e536b93..b3251d379e 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,8 +16,8 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 6fbd2ed015..7c8eb446cd 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,29 +33,29 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: github.event_name == 'push' with: ref: master - name: Login to registry - uses: docker/login-action@v3 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 3311f09b2d..9985084a73 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,8 +27,8 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.x' - id: set-distros @@ -55,18 +55,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: src - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 with: install: true - name: Set up docker layer caching - uses: actions/cache@v4 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -74,7 +74,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.x' @@ -101,7 +101,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -130,9 +130,9 @@ jobs: arch: aarch64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -143,7 +143,7 @@ jobs: - name: Set up QEMU to emulate aarch64 if: matrix.arch == 'aarch64' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 with: platforms: arm64 @@ -165,7 +165,7 @@ jobs: CARGO_NET_GIT_FETCH_WITH_CLI: true CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: Wheel-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl @@ -176,8 +176,8 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.10' @@ -186,7 +186,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: Sdist path: dist/*.tar.gz @@ -203,7 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@b14cf4c92620c250e1c074ab0a5800e37df86765 # v4.2.0 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. @@ -213,7 +213,7 @@ jobs: tar -cvJf debs.tar.xz debs - name: Attach to release # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 - uses: softprops/action-gh-release@v0.1.15 + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 084b08b249..a9fb982b11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} steps: - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') @@ -83,11 +83,11 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: "3.x" poetry-version: "1.3.2" @@ -101,8 +101,8 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" @@ -111,8 +111,8 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -124,10 +124,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: install-project: "false" @@ -145,14 +145,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 + uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: # We want to make use of type hints in optional dependencies too. extras: all @@ -165,7 +165,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@v4 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: | .mypy_cache @@ -178,7 +178,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -186,11 +186,11 @@ jobs: if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -204,13 +204,13 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: poetry-version: "1.3.2" extras: "all" @@ -222,13 +222,13 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 with: components: clippy - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo clippy -- -D warnings @@ -240,14 +240,14 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: toolchain: nightly-2022-12-01 components: clippy - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo clippy --all-features -- -D warnings @@ -257,15 +257,15 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: # We use nightly so that it correctly groups together imports toolchain: nightly-2022-12-01 components: rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo fmt --check @@ -276,8 +276,8 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -301,7 +301,7 @@ jobs: - lint-readme runs-on: ubuntu-latest steps: - - uses: matrix-org/done-action@v3 + - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3 with: needs: ${{ toJSON(needs) }} @@ -324,8 +324,8 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" - id: get-matrix @@ -345,7 +345,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -360,10 +360,10 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: ${{ matrix.job.python-version }} poetry-version: "1.3.2" @@ -399,11 +399,11 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -412,7 +412,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@v5 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.9' @@ -462,10 +462,10 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: ${{ matrix.python-version }} poetry-version: "1.3.2" @@ -512,13 +512,13 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Run SyTest run: /bootstrap.sh synapse @@ -527,7 +527,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -557,9 +557,9 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: poetry-version: "1.3.2" extras: "postgres" @@ -601,7 +601,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -612,7 +612,7 @@ jobs: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: ${{ matrix.python-version }} poetry-version: "1.3.2" @@ -625,7 +625,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps @@ -655,19 +655,19 @@ jobs: database: Postgres steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + - name: Checkout synapse codebase + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: synapse - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -690,11 +690,11 @@ jobs: - changes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@1.66.0 - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo test @@ -708,13 +708,13 @@ jobs: - changes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo bench --no-run @@ -733,7 +733,7 @@ jobs: - linting-done runs-on: ubuntu-latest steps: - - uses: matrix-org/done-action@v3 + - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3 with: needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/triage-incoming.yml b/.github/workflows/triage-incoming.yml index 7a369b77fe..1d291a319b 100644 --- a/.github/workflows/triage-incoming.yml +++ b/.github/workflows/triage-incoming.yml @@ -6,7 +6,7 @@ on: jobs: triage: - uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v2 + uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@18beaf3c8e536108bd04d18e6c3dc40ba3931e28 # v2.0.3 with: project_id: 'PVT_kwDOAIB0Bs4AFDdZ' content_id: ${{ github.event.issue.node_id }} diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index d1ac4357b1..c08c674c88 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -11,7 +11,7 @@ jobs: if: > contains(github.event.issue.labels.*.name, 'X-Needs-Info') steps: - - uses: actions/add-to-project@main + - uses: actions/add-to-project@f5473ace9aeee8b97717b281e26980aa5097023f # main (v1.0.2 + 10 commits) id: add_project with: project-url: "https://github.com/orgs/matrix-org/projects/67" diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index cdaa00ef90..45f70e5420 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -40,13 +40,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: "3.x" extras: "all" @@ -64,14 +64,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@v1 + - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 with: python-version: "3.x" extras: "all test" @@ -108,11 +108,11 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, @@ -136,7 +136,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -164,14 +164,14 @@ jobs: steps: - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: synapse - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -206,7 +206,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/changelog.d/18255.misc b/changelog.d/18255.misc new file mode 100644 index 0000000000..49a5b0e7b6 --- /dev/null +++ b/changelog.d/18255.misc @@ -0,0 +1 @@ +Pin GitHub Actions dependencies by commit hash. \ No newline at end of file From 33bcef9dc7eee6684dc1b8df69a30c61cae879e8 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Fri, 21 Mar 2025 15:32:52 +0000 Subject: [PATCH 208/691] Update Poetry to 2.1.1, including updating the lock file version. (#18251) --- .ci/scripts/check_lockfile.py | 6 +- .github/workflows/fix_lint.yaml | 3 +- .github/workflows/latest_deps.yml | 4 +- .github/workflows/tests.yml | 30 ++-- .github/workflows/twisted_trunk.yml | 10 +- changelog.d/18251.misc | 1 + debian/build_virtualenv | 2 +- debian/changelog | 6 + docker/Dockerfile | 4 +- docs/development/dependencies.md | 8 +- poetry.lock | 220 ++++++++++++++++++++++++---- 11 files changed, 235 insertions(+), 59 deletions(-) create mode 100644 changelog.d/18251.misc diff --git a/.ci/scripts/check_lockfile.py b/.ci/scripts/check_lockfile.py index 19cec7ddd6..46d3952b4c 100755 --- a/.ci/scripts/check_lockfile.py +++ b/.ci/scripts/check_lockfile.py @@ -11,12 +11,12 @@ with open("poetry.lock", "rb") as f: try: lock_version = lockfile["metadata"]["lock-version"] - assert lock_version == "2.0" + assert lock_version == "2.1" except Exception: print( """\ - Lockfile is not version 2.0. You probably need to upgrade poetry on your local box - and re-run `poetry lock --no-update`. See the Poetry cheat sheet at + Lockfile is not version 2.1. You probably need to upgrade poetry on your local box + and re-run `poetry lock`. See the Poetry cheat sheet at https://element-hq.github.io/synapse/develop/development/dependencies.html """ ) diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 3dcda1bab9..fe699c1b2f 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -25,9 +25,10 @@ jobs: - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: install-project: "false" + poetry-version: "2.1.1" - name: Run ruff check continue-on-error: true diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 740c83758f..b7c1b727c9 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -46,10 +46,10 @@ jobs: # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9fb982b11..ff83d6e365 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,10 +87,10 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "all" - run: poetry run scripts-dev/generate_sample_config.sh --check - run: poetry run scripts-dev/config-lint.sh @@ -127,8 +127,9 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: + poetry-version: "2.1.1" install-project: "false" - name: Run ruff check @@ -152,7 +153,7 @@ jobs: - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Setup Poetry - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: # We want to make use of type hints in optional dependencies too. extras: all @@ -161,6 +162,7 @@ jobs: # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 # To make CI green, err towards caution and install the project. install-project: "true" + poetry-version: "2.1.1" # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 @@ -210,9 +212,9 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "all" - run: poetry run scripts-dev/check_pydantic_models.py @@ -363,10 +365,10 @@ jobs: uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.job.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: ${{ matrix.job.extras }} - name: Await PostgreSQL if: ${{ matrix.job.postgres-version }} @@ -465,10 +467,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: ${{ matrix.extras }} - run: poetry run trial --jobs=2 tests - name: Dump logs @@ -559,9 +561,9 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "postgres" - run: .ci/scripts/test_export_data_command.sh env: @@ -612,10 +614,10 @@ jobs: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" + poetry-version: "2.1.1" extras: "postgres" - run: .ci/scripts/test_synapse_port_db.sh id: run_tester_script diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 45f70e5420..aac1e350a2 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -46,10 +46,11 @@ jobs: uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" extras: "all" + poetry-version: "2.1.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }} @@ -71,10 +72,11 @@ jobs: uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - - uses: matrix-org/setup-python-poetry@4421c92b6223f03ae55560e29aa8ebd39cf6314a # v1.2.4 + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" extras: "all test" + poetry-version: "2.1.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk @@ -181,11 +183,11 @@ jobs: run: | set -x DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==1.3.2 + pipx install poetry==2.1.1 poetry remove -n twisted poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry lock --no-update + poetry lock working-directory: synapse - run: | diff --git a/changelog.d/18251.misc b/changelog.d/18251.misc new file mode 100644 index 0000000000..3b557a06c4 --- /dev/null +++ b/changelog.d/18251.misc @@ -0,0 +1 @@ +Update Poetry to 2.1.1, including updating the lock file version. \ No newline at end of file diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 5fc817b607..9e7fb95c8e 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -35,7 +35,7 @@ TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip -pip install poetry==1.3.2 +pip install poetry==2.1.1 poetry-plugin-export==1.9.0 poetry export \ --extras all \ --extras test \ diff --git a/debian/changelog b/debian/changelog index c50249f081..3d77fe8e92 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.127.0~rc1+nmu1) UNRELEASED; urgency=medium + + * Update Poetry to 2.1.1. + + -- Synapse Packaging team Wed, 19 Mar 2025 17:38:49 +0000 + matrix-synapse-py3 (1.127.0~rc1) stable; urgency=medium * New Synapse release 1.127.0rc1. diff --git a/docker/Dockerfile b/docker/Dockerfile index 1dd65f2413..27125c4c0b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ ARG DEBIAN_VERSION=bookworm ARG PYTHON_VERSION=3.12 -ARG POETRY_VERSION=1.8.3 +ARG POETRY_VERSION=2.1.1 ### ### Stage 0: generate requirements.txt @@ -56,7 +56,7 @@ ENV UV_LINK_MODE=copy ARG POETRY_VERSION RUN --mount=type=cache,target=/root/.cache/uv \ if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - uvx --with poetry-plugin-export==1.8.0 \ + uvx --with poetry-plugin-export==1.9.0 \ poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ else \ touch /synapse/requirements.txt; \ diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index e4378231aa..8e29ff3a57 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -187,7 +187,7 @@ useful. ## ...add a new dependency? Either: -- manually update `pyproject.toml`; then `poetry lock --no-update`; or else +- manually update `pyproject.toml`; then `poetry lock`; or else - `poetry add packagename`. See `poetry add --help`; note the `--dev`, `--extras` and `--optional` flags in particular. @@ -202,12 +202,12 @@ poetry remove packagename ``` ought to do the trick. Alternatively, manually update `pyproject.toml` and -`poetry lock --no-update`. Include the updated `pyproject.toml` and `poetry.lock` +`poetry lock`. Include the updated `pyproject.toml` and `poetry.lock` files in your commit. ## ...update the version range for an existing dependency? -Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`. +Best done by manually editing `pyproject.toml`, then `poetry lock`. Include the updated `pyproject.toml` and `poetry.lock` in your commit. ## ...update a dependency in the locked environment? @@ -233,7 +233,7 @@ poetry add packagename==1.2.3 # Get poetry to recompute the content-hash of pyproject.toml without changing # the locked package versions. -poetry lock --no-update +poetry lock ``` Either way, include the updated `poetry.lock` file in your commit. diff --git a/poetry.lock b/poetry.lock index cbf22d9806..ed98dd9f4e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -6,6 +6,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -17,18 +18,19 @@ version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] [[package]] name = "authlib" @@ -36,6 +38,8 @@ version = "1.4.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "Authlib-1.4.1-py2.py3-none-any.whl", hash = "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"}, {file = "authlib-1.4.1.tar.gz", hash = "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d"}, @@ -50,6 +54,7 @@ version = "22.10.0" description = "Self-service finite-state machines for the programmer on the go." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "Automat-22.10.0-py2.py3-none-any.whl", hash = "sha256:c3164f8742b9dc440f3682482d32aaff7bb53f71740dd018533f9de286b64180"}, {file = "Automat-22.10.0.tar.gz", hash = "sha256:e56beb84edad19dcc11d30e8d9b895f75deeb5ef5e96b84a467066b3b84bb04e"}, @@ -68,6 +73,7 @@ version = "4.3.0" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, @@ -132,6 +138,7 @@ version = "6.2.0" description = "An easy safelist-based HTML-sanitizing tool." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "bleach-6.2.0-py3-none-any.whl", hash = "sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e"}, {file = "bleach-6.2.0.tar.gz", hash = "sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f"}, @@ -149,6 +156,7 @@ version = "2.0.0" description = "Canonical JSON" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "canonicaljson-2.0.0-py3-none-any.whl", hash = "sha256:c38a315de3b5a0532f1ec1f9153cd3d716abfc565a558d00a4835428a34fca5b"}, {file = "canonicaljson-2.0.0.tar.gz", hash = "sha256:e2fdaef1d7fadc5d9cb59bd3d0d41b064ddda697809ac4325dced721d12f113f"}, @@ -160,6 +168,7 @@ version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, @@ -171,6 +180,7 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -250,6 +260,7 @@ version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main", "dev"] files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, @@ -334,6 +345,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -348,6 +360,8 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -359,6 +373,7 @@ version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, @@ -373,6 +388,7 @@ version = "15.1.0" description = "Symbolic constants in Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "constantly-15.1.0-py2.py3-none-any.whl", hash = "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"}, {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, @@ -384,6 +400,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -433,6 +450,8 @@ version = "0.7.1" description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -444,6 +463,7 @@ version = "1.2.13" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["dev"] files = [ {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, @@ -453,7 +473,7 @@ files = [ wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] +dev = ["PyTest (<5) ; python_version < \"3.6\"", "PyTest ; python_version >= \"3.6\"", "PyTest-Cov (<2.6) ; python_version < \"3.6\"", "PyTest-Cov ; python_version >= \"3.6\"", "bump2version (<1)", "configparser (<5) ; python_version < \"3\"", "importlib-metadata (<3) ; python_version < \"3\"", "importlib-resources (<4) ; python_version < \"3\"", "sphinx (<2)", "sphinxcontrib-websupport (<2) ; python_version < \"3\"", "tox", "zipp (<2) ; python_version < \"3\""] [[package]] name = "docutils" @@ -461,6 +481,7 @@ version = "0.19" description = "Docutils -- Python Documentation Utilities" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, @@ -472,6 +493,8 @@ version = "4.1.5" description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and lxml" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -486,6 +509,7 @@ version = "4.0.10" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, @@ -500,6 +524,7 @@ version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, @@ -510,7 +535,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "hiredis" @@ -518,6 +543,8 @@ version = "3.1.0" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -636,6 +663,7 @@ version = "21.0.0" description = "A featureful, immutable, and correct URL for Python." optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"}, {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"}, @@ -650,6 +678,7 @@ version = "1.5.0" description = "A tool for generating OIDC identities" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "id-1.5.0-py3-none-any.whl", hash = "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658"}, {file = "id-1.5.0.tar.gz", hash = "sha256:292cb8a49eacbbdbce97244f47a97b4c62540169c976552e497fd57df0734c1d"}, @@ -669,6 +698,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -683,6 +713,7 @@ version = "3.3.0" description = "Iterative JSON parser with standard Python iterator interfaces" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7f7a5250599c366369fbf3bc4e176f5daa28eb6bc7d6130d02462ed335361675"}, {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f87a7e52f79059f9c58f6886c262061065eb6f7554a587be7ed3aa63e6b71b34"}, @@ -786,6 +817,7 @@ version = "4.2.1" description = "Immutable wrapper around dictionaries (a fork of frozendict)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "immutabledict-4.2.1-py3-none-any.whl", hash = "sha256:c56a26ced38c236f79e74af3ccce53772827cef5c3bce7cab33ff2060f756373"}, {file = "immutabledict-4.2.1.tar.gz", hash = "sha256:d91017248981c72eb66c8ff9834e99c2f53562346f23e7f51e7a5ebcf66a3bcc"}, @@ -797,6 +829,8 @@ version = "6.7.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\" or python_version < \"3.10\"" files = [ {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, @@ -808,7 +842,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "importlib-resources" @@ -816,6 +850,8 @@ version = "5.12.0" description = "Read resources from Python packages" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.10\"" files = [ {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, @@ -826,7 +862,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "incremental" @@ -834,6 +870,7 @@ version = "24.7.2" description = "A small library that versions your Python projects." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "incremental-24.7.2-py3-none-any.whl", hash = "sha256:8cb2c3431530bec48ad70513931a760f446ad6c25e8333ca5d95e24b0ed7b8fe"}, {file = "incremental-24.7.2.tar.gz", hash = "sha256:fb4f1d47ee60efe87d4f6f0ebb5f70b9760db2b2574c59c8e8912be4ebd464c9"}, @@ -852,6 +889,8 @@ version = "4.8.0" description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -871,6 +910,8 @@ version = "3.2.3" description = "Utility functions for Python class constructs" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "jaraco.classes-3.2.3-py3-none-any.whl", hash = "sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158"}, {file = "jaraco.classes-3.2.3.tar.gz", hash = "sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a"}, @@ -881,7 +922,7 @@ more-itertools = "*" [package.extras] docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "jeepney" @@ -889,6 +930,8 @@ version = "0.8.0" description = "Low-level, pure Python DBus protocol wrapper." optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"linux\"" files = [ {file = "jeepney-0.8.0-py3-none-any.whl", hash = "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"}, {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, @@ -896,7 +939,7 @@ files = [ [package.extras] test = ["async-timeout", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"] -trio = ["async_generator", "trio"] +trio = ["async_generator ; python_version == \"3.6\"", "trio"] [[package]] name = "jinja2" @@ -904,6 +947,7 @@ version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, @@ -921,6 +965,7 @@ version = "4.23.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, @@ -942,6 +987,7 @@ version = "2023.6.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema_specifications-2023.6.1-py3-none-any.whl", hash = "sha256:3d2b82663aff01815f744bb5c7887e2121a63399b49b104a3c96145474d091d7"}, {file = "jsonschema_specifications-2023.6.1.tar.gz", hash = "sha256:ca1c4dd059a9e7b34101cf5b3ab7ff1d18b139f35950d598d629837ef66e8f28"}, @@ -956,6 +1002,8 @@ version = "23.13.1" description = "Store and access your passwords safely." optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "keyring-23.13.1-py3-none-any.whl", hash = "sha256:771ed2a91909389ed6148631de678f82ddc73737d85a927f382a8a1b157898cd"}, {file = "keyring-23.13.1.tar.gz", hash = "sha256:ba2e15a9b35e21908d0aaf4e0a47acc52d6ae33444df0da2b49d41a46ef6d678"}, @@ -971,7 +1019,7 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} [package.extras] completion = ["shtab"] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "ldap3" @@ -979,6 +1027,8 @@ version = "2.9.1" description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -993,6 +1043,8 @@ version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -1147,6 +1199,7 @@ version = "0.5.1" description = "Type annotations for the lxml package" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "lxml-stubs-0.5.1.tar.gz", hash = "sha256:e0ec2aa1ce92d91278b719091ce4515c12adc1d564359dfaf81efa7d4feab79d"}, {file = "lxml_stubs-0.5.1-py3-none-any.whl", hash = "sha256:1f689e5dbc4b9247cb09ae820c7d34daeb1fdbd1db06123814b856dae7787272"}, @@ -1161,6 +1214,7 @@ version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, @@ -1185,6 +1239,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -1255,6 +1310,7 @@ version = "1.3.0" description = "Common utilities for Synapse, Sydent and Sygnal" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "matrix_common-1.3.0-py3-none-any.whl", hash = "sha256:524e2785b9b03be4d15f3a8a6b857c5b6af68791ffb1b9918f0ad299abc4db20"}, {file = "matrix_common-1.3.0.tar.gz", hash = "sha256:62e121cccd9f243417b57ec37a76dc44aeb198a7a5c67afd6b8275992ff2abd1"}, @@ -1273,6 +1329,8 @@ version = "0.3.0" description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1292,6 +1350,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1303,6 +1362,8 @@ version = "9.1.0" description = "More routines for operating on iterables, beyond itertools" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "more-itertools-9.1.0.tar.gz", hash = "sha256:cabaa341ad0389ea83c17a94566a53ae4c9d07349861ecb14dc6d0345cf9ac5d"}, {file = "more_itertools-9.1.0-py3-none-any.whl", hash = "sha256:d2bc7f02446e86a68911e58ded76d6561eea00cddfb2a91e7019bbb586c799f3"}, @@ -1314,6 +1375,7 @@ version = "1.1.0" description = "MessagePack serializer" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd"}, {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d"}, @@ -1387,6 +1449,7 @@ version = "1.13.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, @@ -1440,6 +1503,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -1451,6 +1515,7 @@ version = "1.0.9" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "mypy_zope-1.0.9-py3-none-any.whl", hash = "sha256:6666c1556891a3cb186137519dbd7a58cb30fb72b2504798cad47b35391921ba"}, {file = "mypy_zope-1.0.9.tar.gz", hash = "sha256:37d6985dfb05a4c27b35cff47577fd5bad878db4893ddedf54d165f7389a1cdb"}, @@ -1470,6 +1535,7 @@ version = "1.3.0" description = "A network address manipulation library for Python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "netaddr-1.3.0-py3-none-any.whl", hash = "sha256:c2c6a8ebe5554ce33b7d5b3a306b71bbb373e000bbbf2350dd5213cc56e3dbbe"}, {file = "netaddr-1.3.0.tar.gz", hash = "sha256:5c3c3d9895b551b763779ba7db7a03487dc1f8e3b385af819af341ae9ef6e48a"}, @@ -1484,6 +1550,8 @@ version = "2.4.0" description = "OpenTracing API for Python. See documentation at http://opentracing.io" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1497,6 +1565,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -1508,6 +1577,7 @@ version = "0.9.0" description = "Parameterized testing with any Python test framework" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, @@ -1522,6 +1592,7 @@ version = "8.13.50" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "phonenumbers-8.13.50-py2.py3-none-any.whl", hash = "sha256:bb95dbc0d9979c51f7ad94bcd780784938958861fbb4b75a2fe39ccd3d58954a"}, {file = "phonenumbers-8.13.50.tar.gz", hash = "sha256:e05ac6fb7b98c6d719a87ea895b9fc153673b4a51f455ec9afaf557ef4629da6"}, @@ -1533,6 +1604,7 @@ version = "11.1.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, @@ -1612,7 +1684,7 @@ docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline fpx = ["olefile"] mic = ["olefile"] tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] -typing = ["typing-extensions"] +typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] [[package]] @@ -1621,6 +1693,7 @@ version = "0.21.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"}, {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"}, @@ -1635,6 +1708,8 @@ version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1654,6 +1729,8 @@ version = "2.9.0" description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=master" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1668,6 +1745,8 @@ version = "1.1" description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1681,6 +1760,7 @@ version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, @@ -1692,6 +1772,7 @@ version = "0.4.1" description = "A collection of ASN.1-based protocols modules" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, @@ -1706,6 +1787,7 @@ version = "2.21" description = "C parser in Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main", "dev"] files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -1717,6 +1799,7 @@ version = "2.10.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, @@ -1729,7 +1812,7 @@ typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" @@ -1737,6 +1820,7 @@ version = "2.27.1" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, @@ -1849,6 +1933,7 @@ version = "2.5.0" description = "Use the full Github API v3" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2"}, {file = "pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf"}, @@ -1868,13 +1953,14 @@ version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, ] [package.extras] -plugins = ["importlib-metadata"] +plugins = ["importlib-metadata ; python_version < \"3.8\""] [[package]] name = "pyicu" @@ -1882,6 +1968,8 @@ version = "2.14" description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"user-search\" or extra == \"all\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -1892,6 +1980,7 @@ version = "2.6.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, @@ -1912,6 +2001,7 @@ version = "0.13.0" description = "Macaroon library for Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pymacaroons-0.13.0-py2.py3-none-any.whl", hash = "sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907"}, {file = "pymacaroons-0.13.0.tar.gz", hash = "sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8"}, @@ -1927,6 +2017,8 @@ version = "1.0.1" description = "A development tool to measure, monitor and analyze the memory behavior of Python objects." optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -1938,6 +2030,7 @@ version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" optional = false python-versions = ">=3.6" +groups = ["main", "dev"] files = [ {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, @@ -1964,6 +2057,7 @@ version = "24.3.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, @@ -1982,6 +2076,8 @@ version = "7.5.0" description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2005,6 +2101,8 @@ version = "2.8.2" description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2019,6 +2117,7 @@ version = "0.0.20" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104"}, {file = "python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13"}, @@ -2030,6 +2129,8 @@ version = "2022.7.1" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2041,6 +2142,8 @@ version = "0.2.0" description = "" optional = false python-versions = "*" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, @@ -2052,6 +2155,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -2114,6 +2218,7 @@ version = "37.3" description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "readme_renderer-37.3-py3-none-any.whl", hash = "sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343"}, {file = "readme_renderer-37.3.tar.gz", hash = "sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273"}, @@ -2133,6 +2238,7 @@ version = "0.29.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "referencing-0.29.1-py3-none-any.whl", hash = "sha256:d3c8f323ee1480095da44d55917cfb8278d73d6b4d5f677e3e40eb21314ac67f"}, {file = "referencing-0.29.1.tar.gz", hash = "sha256:90cb53782d550ba28d2166ef3f55731f38397def8832baac5d45235f1995e35e"}, @@ -2148,6 +2254,7 @@ version = "2.32.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, @@ -2169,6 +2276,7 @@ version = "1.0.0" description = "A utility belt for advanced users of python-requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["dev"] files = [ {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, @@ -2183,6 +2291,7 @@ version = "2.0.0" description = "Validating URI References per RFC 3986" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, @@ -2197,6 +2306,7 @@ version = "13.3.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" +groups = ["dev"] files = [ {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, @@ -2215,6 +2325,7 @@ version = "0.8.10" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "rpds_py-0.8.10-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:93d06cccae15b3836247319eee7b6f1fdcd6c10dabb4e6d350d27bd0bdca2711"}, {file = "rpds_py-0.8.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3816a890a6a9e9f1de250afa12ca71c9a7a62f2b715a29af6aaee3aea112c181"}, @@ -2321,6 +2432,7 @@ version = "0.7.3" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.7.3-py3-none-linux_armv6l.whl", hash = "sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344"}, {file = "ruff-0.7.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0"}, @@ -2348,6 +2460,8 @@ version = "3.3.3" description = "Python bindings to FreeDesktop.org Secret Service API" optional = false python-versions = ">=3.6" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"linux\"" files = [ {file = "SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"}, {file = "SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77"}, @@ -2363,13 +2477,14 @@ version = "2.10.0" description = "A library implementing the 'SemVer' scheme." optional = false python-versions = ">=2.7" +groups = ["main"] files = [ {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, ] [package.extras] -dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1)", "coverage", "flake8", "nose2", "readme-renderer (<25.0)", "tox", "wheel", "zest.releaser[recommended]"] +dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1) ; python_version == \"3.4\"", "coverage", "flake8", "nose2", "readme-renderer (<25.0) ; python_version == \"3.4\"", "tox", "wheel", "zest.releaser[recommended]"] doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] @@ -2378,6 +2493,8 @@ version = "2.22.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2434,6 +2551,7 @@ version = "24.2.0" description = "Service identity verification for pyOpenSSL & cryptography." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "service_identity-24.2.0-py3-none-any.whl", hash = "sha256:6b047fbd8a84fd0bb0d55ebce4031e400562b9196e1e0d3e0fe2b8a59f6d4a85"}, {file = "service_identity-24.2.0.tar.gz", hash = "sha256:b8683ba13f0d39c6cd5d625d2c5f65421d6d707b013b375c355751557cbe8e09"}, @@ -2458,15 +2576,16 @@ version = "72.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (<0.4) ; platform_system == \"Windows\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.3.2) ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "setuptools-rust" @@ -2474,6 +2593,7 @@ version = "1.10.2" description = "Setuptools Rust extension plugin" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "setuptools_rust-1.10.2-py3-none-any.whl", hash = "sha256:4b39c435ae9670315d522ed08fa0e8cb29f2a6048033966b6be2571a90ce4f1c"}, {file = "setuptools_rust-1.10.2.tar.gz", hash = "sha256:5d73e7eee5f87a6417285b617c97088a7c20d1a70fcea60e3bdc94ff567c29dc"}, @@ -2489,6 +2609,7 @@ version = "1.1.4" description = "Sign JSON with Ed25519 signatures" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, {file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, @@ -2508,6 +2629,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2519,6 +2641,7 @@ version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, @@ -2530,6 +2653,7 @@ version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -2541,6 +2665,8 @@ version = "235" description = "Python interface for libsystemd" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"systemd\"" files = [ {file = "systemd-python-235.tar.gz", hash = "sha256:4e57f39797fd5d9e2d22b8806a252d7c0106c936039d1e71c8c6b8008e695c0a"}, ] @@ -2551,6 +2677,8 @@ version = "1.0.2" description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2565,6 +2693,8 @@ version = "0.16.0" description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2583,6 +2713,7 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -2624,6 +2755,8 @@ version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, @@ -2644,6 +2777,7 @@ version = "24.8.0" description = "Building newsfiles for your project." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "towncrier-24.8.0-py3-none-any.whl", hash = "sha256:9343209592b839209cdf28c339ba45792fbfe9775b5f9c177462fd693e127d8d"}, {file = "towncrier-24.8.0.tar.gz", hash = "sha256:013423ee7eed102b2f393c287d22d95f66f1a3ea10a4baa82d298001a7f18af3"}, @@ -2665,6 +2799,7 @@ version = "24.9.1" description = "High-level Twisted HTTP Client API" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "treq-24.9.1-py3-none-any.whl", hash = "sha256:eee4756fd9a857c77f180fd5202b52c518f2d3e2826dce28b89066c03bfc45d0"}, {file = "treq-24.9.1.tar.gz", hash = "sha256:15da7fc404f3e4ed59d0abe5f8eef4966fabbe618039a2a23bc7c15305cefea8"}, @@ -2688,6 +2823,7 @@ version = "6.1.0" description = "Collection of utilities for publishing packages on PyPI" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384"}, {file = "twine-6.1.0.tar.gz", hash = "sha256:be324f6272eff91d07ee93f251edf232fc647935dd585ac003539b42404a8dbd"}, @@ -2714,6 +2850,7 @@ version = "24.7.0" description = "An asynchronous networking framework written in Python" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "twisted-24.7.0-py3-none-any.whl", hash = "sha256:734832ef98108136e222b5230075b1079dad8a3fc5637319615619a7725b0c81"}, {file = "twisted-24.7.0.tar.gz", hash = "sha256:5a60147f044187a127ec7da96d170d49bcce50c6fd36f594e60f4587eff4d394"}, @@ -2732,19 +2869,19 @@ typing-extensions = ">=4.2.0" zope-interface = ">=5" [package.extras] -all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] -gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] -osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] +macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] +osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +serial = ["pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\""] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] -windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] +windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] [[package]] name = "txredisapi" @@ -2752,6 +2889,8 @@ version = "1.4.10" description = "non-blocking redis client for python" optional = true python-versions = "*" +groups = ["main"] +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.10-py3-none-any.whl", hash = "sha256:0a6ea77f27f8cf092f907654f08302a97b48fa35f24e0ad99dfb74115f018161"}, {file = "txredisapi-1.4.10.tar.gz", hash = "sha256:7609a6af6ff4619a3189c0adfb86aeda789afba69eb59fc1e19ac0199e725395"}, @@ -2767,6 +2906,7 @@ version = "6.2.0.20241123" description = "Typing stubs for bleach" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types_bleach-6.2.0.20241123-py3-none-any.whl", hash = "sha256:c6e58b3646665ca7c6b29890375390f4569e84f0cf5c171e0fe1ddb71a7be86a"}, {file = "types_bleach-6.2.0.20241123.tar.gz", hash = "sha256:dac5fe9015173514da3ac810c1a935619a3ccbcc5d66c4cbf4707eac00539057"}, @@ -2781,6 +2921,7 @@ version = "1.16.0.20240331" description = "Typing stubs for cffi" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-cffi-1.16.0.20240331.tar.gz", hash = "sha256:b8b20d23a2b89cfed5f8c5bc53b0cb8677c3aac6d970dbc771e28b9c698f5dee"}, {file = "types_cffi-1.16.0.20240331-py3-none-any.whl", hash = "sha256:a363e5ea54a4eb6a4a105d800685fde596bc318089b025b27dee09849fe41ff0"}, @@ -2795,6 +2936,7 @@ version = "0.9.2.20240106" description = "Typing stubs for commonmark" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-commonmark-0.9.2.20240106.tar.gz", hash = "sha256:52a062b71766d6ab258fca2d8e19fb0853796e25ca9afa9d0f67a1e42c93479f"}, {file = "types_commonmark-0.9.2.20240106-py3-none-any.whl", hash = "sha256:606d9de1e3a96cab0b1c0b6cccf4df099116148d1d864d115fde2e27ad6877c3"}, @@ -2806,6 +2948,7 @@ version = "1.1.11.20240228" description = "Typing stubs for html5lib" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-html5lib-1.1.11.20240228.tar.gz", hash = "sha256:22736b7299e605ec4ba539d48691e905fd0c61c3ea610acc59922232dc84cede"}, {file = "types_html5lib-1.1.11.20240228-py3-none-any.whl", hash = "sha256:af5de0125cb0fe5667543b158db83849b22e25c0e36c9149836b095548bf1020"}, @@ -2817,6 +2960,7 @@ version = "4.23.0.20240813" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-jsonschema-4.23.0.20240813.tar.gz", hash = "sha256:c93f48206f209a5bc4608d295ac39f172fb98b9e24159ce577dbd25ddb79a1c0"}, {file = "types_jsonschema-4.23.0.20240813-py3-none-any.whl", hash = "sha256:be283e23f0b87547316c2ee6b0fd36d95ea30e921db06478029e10b5b6aa6ac3"}, @@ -2831,6 +2975,7 @@ version = "1.3.0.20240530" description = "Typing stubs for netaddr" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-netaddr-1.3.0.20240530.tar.gz", hash = "sha256:742c2ec1f202b666f544223e2616b34f1f13df80c91e5aeaaa93a72e4d0774ea"}, {file = "types_netaddr-1.3.0.20240530-py3-none-any.whl", hash = "sha256:354998d018e326da4f1d9b005fc91137b7c2c473aaf03c4ef64bf83c6861b440"}, @@ -2842,6 +2987,7 @@ version = "2.4.10.6" description = "Typing stubs for opentracing" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "types-opentracing-2.4.10.6.tar.gz", hash = "sha256:87a1bdfce9de5e555e30497663583b9b9c3bb494d029ef9806aa1f137c19e744"}, {file = "types_opentracing-2.4.10.6-py3-none-any.whl", hash = "sha256:25914c834db033a4a38fc322df0b5e5e14503b0ac97f78304ae180d721555e97"}, @@ -2853,6 +2999,7 @@ version = "10.2.0.20240822" description = "Typing stubs for Pillow" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-Pillow-10.2.0.20240822.tar.gz", hash = "sha256:559fb52a2ef991c326e4a0d20accb3bb63a7ba8d40eb493e0ecb0310ba52f0d3"}, {file = "types_Pillow-10.2.0.20240822-py3-none-any.whl", hash = "sha256:d9dab025aba07aeb12fd50a6799d4eac52a9603488eca09d7662543983f16c5d"}, @@ -2864,6 +3011,7 @@ version = "2.9.21.20250121" description = "Typing stubs for psycopg2" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "types_psycopg2-2.9.21.20250121-py3-none-any.whl", hash = "sha256:b890dc6f5a08b6433f0ff73a4ec9a834deedad3e914f2a4a6fd43df021f745f1"}, {file = "types_psycopg2-2.9.21.20250121.tar.gz", hash = "sha256:2b0e2cd0f3747af1ae25a7027898716d80209604770ef3cbf350fe055b9c349b"}, @@ -2875,6 +3023,7 @@ version = "24.1.0.20240722" description = "Typing stubs for pyOpenSSL" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-pyOpenSSL-24.1.0.20240722.tar.gz", hash = "sha256:47913b4678a01d879f503a12044468221ed8576263c1540dcb0484ca21b08c39"}, {file = "types_pyOpenSSL-24.1.0.20240722-py3-none-any.whl", hash = "sha256:6a7a5d2ec042537934cfb4c9d4deb0e16c4c6250b09358df1f083682fe6fda54"}, @@ -2890,6 +3039,7 @@ version = "6.0.12.20241230" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"}, {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"}, @@ -2901,6 +3051,7 @@ version = "2.32.0.20241016" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, @@ -2915,6 +3066,7 @@ version = "75.2.0.20241019" description = "Typing stubs for setuptools" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-setuptools-75.2.0.20241019.tar.gz", hash = "sha256:86ea31b5f6df2c6b8f2dc8ae3f72b213607f62549b6fa2ed5866e5299f968694"}, {file = "types_setuptools-75.2.0.20241019-py3-none-any.whl", hash = "sha256:2e48ff3acd4919471e80d5e3f049cce5c177e108d5d36d2d4cee3fa4d4104258"}, @@ -2926,6 +3078,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -2937,6 +3090,7 @@ version = "2.1.0" description = "Encode and decode Base64 without \"=\" padding" optional = false python-versions = ">=3.6,<4.0" +groups = ["main"] files = [ {file = "unpaddedbase64-2.1.0-py3-none-any.whl", hash = "sha256:485eff129c30175d2cd6f0cd8d2310dff51e666f7f36175f738d75dfdbd0b1c6"}, {file = "unpaddedbase64-2.1.0.tar.gz", hash = "sha256:7273c60c089de39d90f5d6d4a7883a79e319dc9d9b1c8924a7fab96178a5f005"}, @@ -2948,13 +3102,14 @@ version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2965,6 +3120,7 @@ version = "0.5.1" description = "Character encoding aliases for legacy web content" optional = false python-versions = "*" +groups = ["main", "dev"] files = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, @@ -2976,6 +3132,7 @@ version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +groups = ["dev"] files = [ {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, @@ -3060,6 +3217,8 @@ version = "2.4.0" description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3079,6 +3238,8 @@ version = "3.19.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\" or python_version < \"3.10\"" files = [ {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, @@ -3094,6 +3255,7 @@ version = "4.6" description = "Very basic event publishing system" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "zope.event-4.6-py2.py3-none-any.whl", hash = "sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c"}, {file = "zope.event-4.6.tar.gz", hash = "sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80"}, @@ -3112,6 +3274,7 @@ version = "7.1.0" description = "Interfaces for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "zope.interface-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2bd9e9f366a5df08ebbdc159f8224904c1c5ce63893984abb76954e6fbe4381a"}, {file = "zope.interface-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:661d5df403cd3c5b8699ac480fa7f58047a3253b029db690efa0c3cf209993ef"}, @@ -3166,6 +3329,7 @@ version = "7.0.1" description = "zope.interface extension for defining data schemas" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "zope.schema-7.0.1-py3-none-any.whl", hash = "sha256:cf006c678793b00e0075ad54d55281c8785ea21e5bc1f5ec0584787719c2aab2"}, {file = "zope.schema-7.0.1.tar.gz", hash = "sha256:ead4dbcb03354d4e410c9a3b904451eb44d90254751b1cbdedf4a61aede9fbb9"}, @@ -3197,6 +3361,6 @@ url-preview = ["lxml"] user-search = ["pyicu"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.9.0" content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d" From 1efb826b54f4a8fdfa56c27a32da9c332fa87cc3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 21 Mar 2025 17:09:49 +0000 Subject: [PATCH 209/691] Delete unreferenced state groups in background (#18254) This PR fixes #18154 to avoid de-deltaing state groups which resulted in DB size temporarily increasing until the DB was `VACUUM`'ed. As a result, less state groups will get deleted now. It also attempts to improve performance by not duplicating work when processing state groups it has already processed in previous iterations. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Erik Johnston --- changelog.d/18254.feature | 1 + docs/development/database_schema.md | 2 +- synapse/_scripts/synapse_port_db.py | 30 +++ synapse/storage/controllers/purge_events.py | 247 +++++++++++++++++- synapse/storage/databases/state/bg_updates.py | 10 +- synapse/storage/databases/state/deletion.py | 44 +++- synapse/storage/schema/__init__.py | 1 + .../02_delete_unreferenced_state_groups.sql | 16 ++ synapse/types/storage/__init__.py | 4 + tests/storage/test_purge.py | 154 +++++++++++ 10 files changed, 495 insertions(+), 14 deletions(-) create mode 100644 changelog.d/18254.feature create mode 100644 synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql diff --git a/changelog.d/18254.feature b/changelog.d/18254.feature new file mode 100644 index 0000000000..62e1b79a15 --- /dev/null +++ b/changelog.d/18254.feature @@ -0,0 +1 @@ +Add background job to clear unreferenced state groups. diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 37a06acc12..620d1c16b0 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -162,7 +162,7 @@ by a unique name, the current status (stored in JSON), and some dependency infor * Whether the update requires a previous update to be complete. * A rough ordering for which to complete updates. -A new background updates needs to be added to the `background_updates` table: +A new background update needs to be added to the `background_updates` table: ```sql INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 1bb9940180..438b2ff8a0 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -192,6 +192,11 @@ APPEND_ONLY_TABLES = [ IGNORED_TABLES = { + # Porting the auto generated sequence in this table is non-trivial. + # None of the entries in this list are mandatory for Synapse to keep working. + # If state group disk space is an issue after the port, the + # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again. + "state_groups_pending_deletion", # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", @@ -217,6 +222,15 @@ IGNORED_TABLES = { } +# These background updates will not be applied upon creation of the postgres database. +IGNORED_BACKGROUND_UPDATES = { + # Reapplying this background update to the postgres database is unnecessary after + # already having waited for the SQLite database to complete all running background + # updates. + "mark_unreferenced_state_groups_for_deletion_bg_update", +} + + # Error returned by the run function. Used at the top-level part of the script to # handle errors and return codes. end_error: Optional[str] = None @@ -688,6 +702,20 @@ class Porter: # 0 means off. 1 means full. 2 means incremental. return autovacuum_setting != 0 + async def remove_ignored_background_updates_from_database(self) -> None: + def _remove_delete_unreferenced_state_groups_bg_updates( + txn: LoggingTransaction, + ) -> None: + txn.execute( + "DELETE FROM background_updates WHERE update_name = ANY(?)", + (list(IGNORED_BACKGROUND_UPDATES),), + ) + + await self.postgres_store.db_pool.runInteraction( + "remove_delete_unreferenced_state_groups_bg_updates", + _remove_delete_unreferenced_state_groups_bg_updates, + ) + async def run(self) -> None: """Ports the SQLite database to a PostgreSQL database. @@ -733,6 +761,8 @@ class Porter: self.hs_config.database.get_single_database() ) + await self.remove_ignored_background_updates_from_database() + await self.run_background_updates_on_postgres() self.progress.set_state("Creating port tables") diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 47cec8c469..c2d4bf8290 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -21,11 +21,19 @@ import itertools import logging -from typing import TYPE_CHECKING, Collection, Mapping, Set +from typing import ( + TYPE_CHECKING, + Collection, + Mapping, + Optional, + Set, +) from synapse.logging.context import nested_logging_context from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases +from synapse.types.storage import _BackgroundUpdates if TYPE_CHECKING: from synapse.server import HomeServer @@ -44,6 +52,11 @@ class PurgeEventsStorageController: self._delete_state_groups_loop, 60 * 1000 ) + self.stores.state.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, + self._background_delete_unrefereneced_state_groups, + ) + async def purge_room(self, room_id: str) -> None: """Deletes all record of a room""" @@ -81,7 +94,8 @@ class PurgeEventsStorageController: ) async def _find_unreferenced_groups( - self, state_groups: Collection[int] + self, + state_groups: Collection[int], ) -> Set[int]: """Used when purging history to figure out which state groups can be deleted. @@ -203,3 +217,232 @@ class PurgeEventsStorageController: room_id, groups_to_sequences, ) + + async def _background_delete_unrefereneced_state_groups( + self, progress: dict, batch_size: int + ) -> int: + """This background update will slowly delete any unreferenced state groups""" + + last_checked_state_group = progress.get("last_checked_state_group") + + if last_checked_state_group is None: + # This is the first run. + last_checked_state_group = ( + await self.stores.state.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={}, + retcol="MAX(id)", + allow_none=True, + desc="get_max_state_group", + ) + ) + if last_checked_state_group is None: + # There are no state groups so the background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE + ) + return batch_size + last_checked_state_group += 1 + + ( + last_checked_state_group, + final_batch, + ) = await self._delete_unreferenced_state_groups_batch( + last_checked_state_group, + batch_size, + ) + + if not final_batch: + # There are more state groups to check. + progress = { + "last_checked_state_group": last_checked_state_group, + } + await self.stores.state.db_pool.updates._background_update_progress( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, + progress, + ) + else: + # This background process is finished. + await self.stores.state.db_pool.updates._end_background_update( + _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE + ) + + return batch_size + + async def _delete_unreferenced_state_groups_batch( + self, + last_checked_state_group: int, + batch_size: int, + ) -> tuple[int, bool]: + """Looks for unreferenced state groups starting from the last state group + checked and marks them for deletion. + + Args: + last_checked_state_group: The last state group that was checked. + batch_size: How many state groups to process in this iteration. + + Returns: + (last_checked_state_group, final_batch) + """ + + # Find all state groups that can be deleted if any of the original set are deleted. + ( + to_delete, + last_checked_state_group, + final_batch, + ) = await self._find_unreferenced_groups_for_background_deletion( + last_checked_state_group, batch_size + ) + + if len(to_delete) == 0: + return last_checked_state_group, final_batch + + await self.stores.state_deletion.mark_state_groups_as_pending_deletion( + to_delete + ) + + return last_checked_state_group, final_batch + + async def _find_unreferenced_groups_for_background_deletion( + self, + last_checked_state_group: int, + batch_size: int, + ) -> tuple[Set[int], int, bool]: + """Used when deleting unreferenced state groups in the background to figure out + which state groups can be deleted. + To avoid increased DB usage due to de-deltaing state groups, this returns only + state groups which are free standing (ie. no shared edges with referenced groups) or + state groups which do not share edges which result in a future referenced group. + + The following scenarios outline the possibilities based on state group data in + the DB. + + ie. Free standing -> state groups 1-N would be returned: + SG_1 + | + ... + | + SG_N + + ie. Previous reference -> state groups 2-N would be returned: + SG_1 <- referenced by event + | + SG_2 + | + ... + | + SG_N + + ie. Future reference -> none of the following state groups would be returned: + SG_1 + | + SG_2 + | + ... + | + SG_N <- referenced by event + + Args: + last_checked_state_group: The last state group that was checked. + batch_size: How many state groups to process in this iteration. + + Returns: + (to_delete, last_checked_state_group, final_batch) + """ + + # If a state group's next edge is not pending deletion then we don't delete the state group. + # If there is no next edge or the next edges are all marked for deletion, then delete + # the state group. + # This holds since we walk backwards from the latest state groups, ensuring that + # we've already checked newer state groups for event references along the way. + def get_next_state_groups_marked_for_deletion_txn( + txn: LoggingTransaction, + ) -> tuple[dict[int, bool], dict[int, int]]: + state_group_sql = """ + SELECT s.id, e.state_group, d.state_group + FROM ( + SELECT id FROM state_groups + WHERE id < ? ORDER BY id DESC LIMIT ? + ) as s + LEFT JOIN state_group_edges AS e ON (s.id = e.prev_state_group) + LEFT JOIN state_groups_pending_deletion AS d ON (e.state_group = d.state_group) + """ + txn.execute(state_group_sql, (last_checked_state_group, batch_size)) + + # Mapping from state group to whether we should delete it. + state_groups_to_deletion: dict[int, bool] = {} + + # Mapping from state group to prev state group. + state_groups_to_prev: dict[int, int] = {} + + for row in txn: + state_group = row[0] + next_edge = row[1] + pending_deletion = row[2] + + if next_edge is not None: + state_groups_to_prev[next_edge] = state_group + + if next_edge is not None and not pending_deletion: + # We have found an edge not marked for deletion. + # Check previous results to see if this group is part of a chain + # within this batch that qualifies for deletion. + # ie. batch contains: + # SG_1 -> SG_2 -> SG_3 + # If SG_3 is a candidate for deletion, then SG_2 & SG_1 should also + # be, even though they have edges which may not be marked for + # deletion. + # This relies on SQL results being sorted in DESC order to work. + next_is_deletion_candidate = state_groups_to_deletion.get(next_edge) + if ( + next_is_deletion_candidate is None + or not next_is_deletion_candidate + ): + state_groups_to_deletion[state_group] = False + else: + state_groups_to_deletion.setdefault(state_group, True) + else: + # This state group may be a candidate for deletion + state_groups_to_deletion.setdefault(state_group, True) + + return state_groups_to_deletion, state_groups_to_prev + + ( + state_groups_to_deletion, + state_group_edges, + ) = await self.stores.state.db_pool.runInteraction( + "get_next_state_groups_marked_for_deletion", + get_next_state_groups_marked_for_deletion_txn, + ) + deletion_candidates = { + state_group + for state_group, deletion in state_groups_to_deletion.items() + if deletion + } + + final_batch = False + state_groups = state_groups_to_deletion.keys() + if len(state_groups) < batch_size: + final_batch = True + else: + last_checked_state_group = min(state_groups) + + if len(state_groups) == 0: + return set(), last_checked_state_group, final_batch + + # Determine if any of the remaining state groups are directly referenced. + referenced = await self.stores.main.get_referenced_state_groups( + deletion_candidates + ) + + # Remove state groups from deletion_candidates which are directly referenced or share a + # future edge with a referenced state group within this batch. + def filter_reference_chains(group: Optional[int]) -> None: + while group is not None: + deletion_candidates.discard(group) + group = state_group_edges.get(group) + + for referenced_group in referenced: + filter_reference_chains(referenced_group) + + return deletion_candidates, last_checked_state_group, final_batch diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index f7824cba0f..95fd0ae73a 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -20,7 +20,15 @@ # import logging -from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Dict, + List, + Mapping, + Optional, + Tuple, + Union, +) from synapse.logging.opentracing import tag_args, trace from synapse.storage._base import SQLBaseStore diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index d4b1c20a45..f77c46f6ae 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -321,18 +321,42 @@ class StateDeletionDataStore: async def mark_state_groups_as_pending_deletion( self, state_groups: Collection[int] ) -> None: - """Mark the given state groups as pending deletion""" + """Mark the given state groups as pending deletion. + + If any of the state groups are already pending deletion, then those records are + left as is. + """ + + await self.db_pool.runInteraction( + "mark_state_groups_as_pending_deletion", + self._mark_state_groups_as_pending_deletion_txn, + state_groups, + ) + + def _mark_state_groups_as_pending_deletion_txn( + self, + txn: LoggingTransaction, + state_groups: Collection[int], + ) -> None: + sql = """ + INSERT INTO state_groups_pending_deletion (state_group, insertion_ts) + VALUES %s + ON CONFLICT (state_group) + DO NOTHING + """ now = self._clock.time_msec() - - await self.db_pool.simple_upsert_many( - table="state_groups_pending_deletion", - key_names=("state_group",), - key_values=[(state_group,) for state_group in state_groups], - value_names=("insertion_ts",), - value_values=[(now,) for _ in state_groups], - desc="mark_state_groups_as_pending_deletion", - ) + rows = [ + ( + state_group, + now, + ) + for state_group in state_groups + ] + if isinstance(txn.database_engine, PostgresEngine): + txn.execute_values(sql % ("?",), rows, fetch=False) + else: + txn.execute_batch(sql % ("(?, ?)",), rows) async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None: """Mark the given state groups as now being referenced""" diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index f87b1a4a0a..2160edb014 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -161,6 +161,7 @@ Changes in SCHEMA_VERSION = 89 Changes in SCHEMA_VERSION = 90 - Add a column `participant` to `room_memberships` table + - Add background update to delete unreferenced state groups. """ diff --git a/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql b/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql new file mode 100644 index 0000000000..55a038e2b8 --- /dev/null +++ b/synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a background update to delete any unreferenced state groups +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9002, 'mark_unreferenced_state_groups_for_deletion_bg_update', '{}'); diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index b5fa20a41a..e03ff7ffc8 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -48,3 +48,7 @@ class _BackgroundUpdates: SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = ( "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update" ) + + MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = ( + "mark_unreferenced_state_groups_for_deletion_bg_update" + ) diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 916e42e731..0aa14fd1f4 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -24,6 +24,7 @@ from synapse.api.errors import NotFoundError, SynapseError from synapse.rest.client import room from synapse.server import HomeServer from synapse.types.state import StateFilter +from synapse.types.storage import _BackgroundUpdates from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -303,3 +304,156 @@ class PurgeTests(HomeserverTestCase): ) ) self.assertEqual(len(state_groups), 1) + + def test_clear_unreferenced_state_groups(self) -> None: + """Test that any unreferenced state groups are automatically cleaned up.""" + + self.helper.send(self.room_id, body="test1") + state1 = self.helper.send_state( + self.room_id, "org.matrix.test", body={"number": 2} + ) + # Create enough state events to require multiple batches of + # mark_unreferenced_state_groups_for_deletion_bg_update to be run. + for i in range(200): + self.helper.send_state(self.room_id, "org.matrix.test", body={"number": i}) + self.helper.send(self.room_id, body="test4") + last = self.helper.send(self.room_id, body="test5") + + # Create an unreferenced state group that has no prev group. + unreferenced_free_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=None, + delta_ids={("org.matrix.test", ""): state1["event_id"]}, + current_state_ids={("org.matrix.test", ""): ""}, + ) + ) + + # Create some unreferenced state groups that have a prev group of one of the + # existing state groups. + prev_group = self.get_success( + self.store._get_state_group_for_event(state1["event_id"]) + ) + unreferenced_end_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=prev_group, + delta_ids={("org.matrix.test", ""): state1["event_id"]}, + current_state_ids=None, + ) + ) + another_unreferenced_end_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=unreferenced_end_state_group, + delta_ids={("org.matrix.test", ""): state1["event_id"]}, + current_state_ids=None, + ) + ) + + # Add some other unreferenced state groups which lead to a referenced state + # group. + # These state groups should not get deleted. + chain_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=None, + delta_ids={("org.matrix.test", ""): ""}, + current_state_ids={("org.matrix.test", ""): ""}, + ) + ) + chain_state_group_2 = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=chain_state_group, + delta_ids={("org.matrix.test", ""): ""}, + current_state_ids=None, + ) + ) + referenced_chain_state_group = self.get_success( + self.state_store.store_state_group( + event_id=last["event_id"], + room_id=self.room_id, + prev_group=chain_state_group_2, + delta_ids={("org.matrix.test", ""): ""}, + current_state_ids=None, + ) + ) + self.get_success( + self.store.db_pool.simple_insert( + "event_to_state_groups", + { + "event_id": "$new_event", + "state_group": referenced_chain_state_group, + }, + ) + ) + + # Insert and run the background update. + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Advance so that the background job to delete the state groups runs + self.reactor.advance( + 1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000 + ) + + # We expect that the unreferenced free state group has been deleted. + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": unreferenced_free_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + # We expect that both unreferenced end state groups have been deleted. + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": unreferenced_end_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + row = self.get_success( + self.state_store.db_pool.simple_select_one_onecol( + table="state_groups", + keyvalues={"id": another_unreferenced_end_state_group}, + retcol="id", + allow_none=True, + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertIsNone(row) + + # We expect there to now only be one state group for the room, which is + # the state group of the last event (as the only outlier). + state_groups = self.get_success( + self.state_store.db_pool.simple_select_onecol( + table="state_groups", + keyvalues={"room_id": self.room_id}, + retcol="id", + desc="test_purge_unreferenced_state_group", + ) + ) + self.assertEqual(len(state_groups), 210) From d8fef721a07870dc2dd6d86cd2063ea1b3e2a491 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:35:01 +0000 Subject: [PATCH 210/691] Correct typo "SAML" -> SSO in mapping providers docs (#18276) --- changelog.d/18276.doc | 1 + docs/sso_mapping_providers.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18276.doc diff --git a/changelog.d/18276.doc b/changelog.d/18276.doc new file mode 100644 index 0000000000..cacf6daf76 --- /dev/null +++ b/changelog.d/18276.doc @@ -0,0 +1 @@ +Correct a small typo in the SSO mapping providers documentation. \ No newline at end of file diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index d6c4e860ae..4d33c8da75 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -10,7 +10,7 @@ As an example, a SSO service may return the email address to turn that into a displayname when creating a Matrix user for this individual. It may choose `John Smith`, or `Smith, John [Example.com]` or any number of variations. As each Synapse configuration may want something different, this is -where SAML mapping providers come into play. +where SSO mapping providers come into play. SSO mapping providers are currently supported for OpenID and SAML SSO configurations. Please see the details below for how to implement your own. From 7af299b3652927a47a592ece0627d203fa7c5c73 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 25 Mar 2025 12:04:21 +0000 Subject: [PATCH 211/691] 1.127.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3053724c9d..f63eabb58a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.127.0 (2025-03-25) + +No significant changes since 1.127.0rc1. + + + + # Synapse 1.127.0rc1 (2025-03-18) ### Features diff --git a/debian/changelog b/debian/changelog index c50249f081..2e36b368d0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.127.0) stable; urgency=medium + + * New Synapse release 1.127.0. + + -- Synapse Packaging team Tue, 25 Mar 2025 12:04:15 +0000 + matrix-synapse-py3 (1.127.0~rc1) stable; urgency=medium * New Synapse release 1.127.0rc1. diff --git a/pyproject.toml b/pyproject.toml index bc0f526390..6a29362919 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.127.0rc1" +version = "1.127.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 92a29dcffc4fca0029c2f3ee63a66152c5e54e60 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 25 Mar 2025 09:57:55 -0400 Subject: [PATCH 212/691] Docker: Use an ARG for debian version more often (#18272) --- changelog.d/18272.docker | 1 + docker/Dockerfile-workers | 5 +++-- docker/complement/Dockerfile | 7 +++++-- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18272.docker diff --git a/changelog.d/18272.docker b/changelog.d/18272.docker new file mode 100644 index 0000000000..ceec619b6c --- /dev/null +++ b/changelog.d/18272.docker @@ -0,0 +1 @@ +Always specify base image debian versions with a build argument. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 2ceb6ab67c..3dec4bba05 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,12 +2,13 @@ ARG SYNAPSE_VERSION=latest ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION +ARG DEBIAN_VERSION=bookworm # first of all, we create a base image with an nginx which we can copy into the # target image. For repeated rebuilds, this is much faster than apt installing # each time. -FROM docker.io/library/debian:bookworm-slim AS deps_base +FROM docker.io/library/debian:${DEBIAN_VERSION}-slim AS deps_base RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -21,7 +22,7 @@ FROM docker.io/library/debian:bookworm-slim AS deps_base # which makes it much easier to copy (but we need to make sure we use an image # based on the same debian version as the synapse image, to make sure we get # the expected version of libc. -FROM docker.io/library/redis:7-bookworm AS redis_base +FROM docker.io/library/redis:7-${DEBIAN_VERSION} AS redis_base # now build the final image, based on the the regular Synapse docker image FROM $FROM diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index ce82c400eb..3e7f808cc5 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -9,6 +9,9 @@ ARG SYNAPSE_VERSION=latest # This is an intermediate image, to be built locally (not pulled from a registry). ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION +ARG DEBIAN_VERSION=bookworm + +FROM docker.io/library/postgres:13-${DEBIAN_VERSION} AS postgres_base FROM $FROM # First of all, we copy postgres server from the official postgres image, @@ -20,8 +23,8 @@ FROM $FROM # the same debian version as Synapse's docker image (so the versions of the # shared libraries match). RUN adduser --system --uid 999 postgres --home /var/lib/postgresql -COPY --from=docker.io/library/postgres:13-bookworm /usr/lib/postgresql /usr/lib/postgresql -COPY --from=docker.io/library/postgres:13-bookworm /usr/share/postgresql /usr/share/postgresql +COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql +COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" ENV PGDATA=/var/lib/postgresql/data From bd08a01fc808d31e8c3e4d20b19dd22bd225df4b Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 25 Mar 2025 09:58:40 -0400 Subject: [PATCH 213/691] Dockerfile: set package arch via APT config option (#18271) --- changelog.d/18271.docker | 1 + docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18271.docker diff --git a/changelog.d/18271.docker b/changelog.d/18271.docker new file mode 100644 index 0000000000..fcb2ac1e00 --- /dev/null +++ b/changelog.d/18271.docker @@ -0,0 +1 @@ +Specify the architecture of installed packages via an APT config option, which is more reliable than appending package names with ":{arch}". diff --git a/docker/Dockerfile b/docker/Dockerfile index 27125c4c0b..54aa355370 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -148,7 +148,7 @@ RUN \ for arch in arm64 amd64; do \ mkdir -p /tmp/debs-${arch} && \ cd /tmp/debs-${arch} && \ - apt-get download $(sed "s/$/:${arch}/" /tmp/pkg-list); \ + apt-get -o APT::Architecture="${arch}" download $(cat /tmp/pkg-list); \ done # Extract the debs for each architecture From a227d20c25ff8fce45b9e3c3874cf3004633a9f4 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 25 Mar 2025 10:09:38 -0400 Subject: [PATCH 214/691] Pass args to start_for_complement.sh (#18273) --- changelog.d/18273.docker | 1 + docker/complement/conf/start_for_complement.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18273.docker diff --git a/changelog.d/18273.docker b/changelog.d/18273.docker new file mode 100644 index 0000000000..1da0c3efb1 --- /dev/null +++ b/changelog.d/18273.docker @@ -0,0 +1 @@ +Allow passing arguments to start_for_complement.sh (to be sent to configure_workers_and_start.py). diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index cc798a3210..59b30e2051 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -5,12 +5,12 @@ set -e echo "Complement Synapse launcher" -echo " Args: $@" +echo " Args: $*" echo " Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR" function log { d=$(date +"%Y-%m-%d %H:%M:%S,%3N") - echo "$d $@" + echo "$d $*" } # Set the server name of the homeserver @@ -131,4 +131,4 @@ export SYNAPSE_TLS_KEY=/conf/server.tls.key # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else -exec /configure_workers_and_start.py +exec /configure_workers_and_start.py "$@" From 5e83434f3a9715895c10de4b8174fb02d94d025d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 14:11:51 +0000 Subject: [PATCH 215/691] Bump log from 0.4.26 to 0.4.27 (#18267) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f53d029cbd..20110694a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,9 +223,9 @@ checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "memchr" From 2277df2a1eb685f85040ef98fa21d41aa4cdd389 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Mar 2025 16:38:15 +0000 Subject: [PATCH 216/691] =?UTF-8?q?Fix=20GHSA-v56r-hwv5-mxg6=20=E2=80=94?= =?UTF-8?q?=20Federation=20denial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/element-hq/synapse/security/advisories/GHSA-v56r-hwv5-mxg6 Federation denial of service via malformed events. --- synapse/api/constants.py | 9 +++++++-- synapse/events/utils.py | 5 ++--- synapse/events/validator.py | 4 +--- synapse/federation/federation_base.py | 12 +++++++++++- synapse/federation/federation_client.py | 13 +++++-------- synapse/federation/federation_server.py | 21 ++++++++++++-------- synapse/federation/units.py | 26 +++++++++++++++++++++++-- 7 files changed, 63 insertions(+), 27 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 9806e2b0fe..c564a8635a 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -29,8 +29,13 @@ from typing import Final # the max size of a (canonical-json-encoded) event MAX_PDU_SIZE = 65536 -# the "depth" field on events is limited to 2**63 - 1 -MAX_DEPTH = 2**63 - 1 +# Max/min size of ints in canonical JSON +CANONICALJSON_MAX_INT = (2**53) - 1 +CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT + +# the "depth" field on events is limited to the same as what +# canonicaljson accepts +MAX_DEPTH = CANONICALJSON_MAX_INT # the maximum length for a room alias is 255 characters MAX_ALIAS_LENGTH = 255 diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 54f94add4d..eb18ba2db7 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -40,6 +40,8 @@ import attr from canonicaljson import encode_canonical_json from synapse.api.constants import ( + CANONICALJSON_MAX_INT, + CANONICALJSON_MIN_INT, MAX_PDU_SIZE, EventContentFields, EventTypes, @@ -61,9 +63,6 @@ SPLIT_FIELD_REGEX = re.compile(r"\\*\.") # Find escaped characters, e.g. those with a \ in front of them. ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)") -CANONICALJSON_MAX_INT = (2**53) - 1 -CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT - # Module API callback that allows adding fields to the unsigned section of # events that are sent to clients. diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 8aa8d7e017..d1fb026cd6 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -86,9 +86,7 @@ class EventValidator: # Depending on the room version, ensure the data is spec compliant JSON. if event.room_version.strict_canonicaljson: - # Note that only the client controlled portion of the event is - # checked, since we trust the portions of the event we created. - validate_canonicaljson(event.content) + validate_canonicaljson(event.get_pdu_json()) if event.type == EventTypes.Aliases: if "aliases" in event.content: diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index b101a389ef..3796bff5e7 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -20,7 +20,7 @@ # # import logging -from typing import TYPE_CHECKING, Awaitable, Callable, Optional +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Sequence from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership from synapse.api.errors import Codes, SynapseError @@ -29,6 +29,7 @@ from synapse.crypto.event_signing import check_event_content_hash from synapse.crypto.keyring import Keyring from synapse.events import EventBase, make_event_from_dict from synapse.events.utils import prune_event, validate_canonicaljson +from synapse.federation.units import filter_pdus_for_valid_depth from synapse.http.servlet import assert_params_in_dict from synapse.logging.opentracing import log_kv, trace from synapse.types import JsonDict, get_domain_from_id @@ -267,6 +268,15 @@ def _is_invite_via_3pid(event: EventBase) -> bool: ) +def parse_events_from_pdu_json( + pdus_json: Sequence[JsonDict], room_version: RoomVersion +) -> List[EventBase]: + return [ + event_from_pdu_json(pdu_json, room_version) + for pdu_json in filter_pdus_for_valid_depth(pdus_json) + ] + + def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: """Construct an EventBase from an event json received over federation diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7d80ff6998..9fc5b70e9a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -68,6 +68,7 @@ from synapse.federation.federation_base import ( FederationBase, InvalidEventSignatureError, event_from_pdu_json, + parse_events_from_pdu_json, ) from synapse.federation.transport.client import SendJoinResponse from synapse.http.client import is_unknown_endpoint @@ -349,7 +350,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - pdus = [event_from_pdu_json(p, room_version) for p in transaction_data_pdus] + pdus = parse_events_from_pdu_json(transaction_data_pdus, room_version) # Check signatures and hash of pdus, removing any from the list that fail checks pdus[:] = await self._check_sigs_and_hash_for_pulled_events_and_fetch( @@ -393,9 +394,7 @@ class FederationClient(FederationBase): transaction_data, ) - pdu_list: List[EventBase] = [ - event_from_pdu_json(p, room_version) for p in transaction_data["pdus"] - ] + pdu_list = parse_events_from_pdu_json(transaction_data["pdus"], room_version) if pdu_list and pdu_list[0]: pdu = pdu_list[0] @@ -809,7 +808,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - auth_chain = [event_from_pdu_json(p, room_version) for p in res["auth_chain"]] + auth_chain = parse_events_from_pdu_json(res["auth_chain"], room_version) signed_auth = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, auth_chain, room_version=room_version @@ -1529,9 +1528,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - events = [ - event_from_pdu_json(e, room_version) for e in content.get("events", []) - ] + events = parse_events_from_pdu_json(content.get("events", []), room_version) signed_events = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, events, room_version=room_version diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1932fa82a4..f9e97ea13e 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -66,7 +66,7 @@ from synapse.federation.federation_base import ( event_from_pdu_json, ) from synapse.federation.persistence import TransactionActions -from synapse.federation.units import Edu, Transaction +from synapse.federation.units import Edu, Transaction, serialize_and_filter_pdus from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import ( @@ -469,7 +469,12 @@ class FederationServer(FederationBase): logger.info("Ignoring PDU: %s", e) continue - event = event_from_pdu_json(p, room_version) + try: + event = event_from_pdu_json(p, room_version) + except SynapseError as e: + logger.info("Ignoring PDU for failing to deserialize: %s", e) + continue + pdus_by_room.setdefault(room_id, []).append(event) if event.origin_server_ts > newest_pdu_ts: @@ -636,8 +641,8 @@ class FederationServer(FederationBase): ) return { - "pdus": [pdu.get_pdu_json() for pdu in pdus], - "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + "pdus": serialize_and_filter_pdus(pdus), + "auth_chain": serialize_and_filter_pdus(auth_chain), } async def on_pdu_request( @@ -761,8 +766,8 @@ class FederationServer(FederationBase): event_json = event.get_pdu_json(time_now) resp = { "event": event_json, - "state": [p.get_pdu_json(time_now) for p in state_events], - "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], + "state": serialize_and_filter_pdus(state_events, time_now), + "auth_chain": serialize_and_filter_pdus(auth_chain_events, time_now), "members_omitted": caller_supports_partial_state, } @@ -1005,7 +1010,7 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() auth_pdus = await self.handler.on_event_auth(event_id) - res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]} + res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} return 200, res async def on_query_client_keys( @@ -1090,7 +1095,7 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() - return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]} + return {"events": serialize_and_filter_pdus(missing_events, time_now)} async def on_openid_userinfo(self, token: str) -> Optional[str]: ts_now_ms = self._clock.time_msec() diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d8b67a6a5b..3bb5f824b7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -24,10 +24,12 @@ server protocol. """ import logging -from typing import List, Optional +from typing import List, Optional, Sequence import attr +from synapse.api.constants import CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT +from synapse.events import EventBase from synapse.types import JsonDict logger = logging.getLogger(__name__) @@ -104,8 +106,28 @@ class Transaction: result = { "origin": self.origin, "origin_server_ts": self.origin_server_ts, - "pdus": self.pdus, + "pdus": filter_pdus_for_valid_depth(self.pdus), } if self.edus: result["edus"] = self.edus return result + + +def filter_pdus_for_valid_depth(pdus: Sequence[JsonDict]) -> List[JsonDict]: + filtered_pdus = [] + for pdu in pdus: + # Drop PDUs that have a depth that is outside of the range allowed + # by canonical json. + if ( + "depth" in pdu + and CANONICALJSON_MIN_INT <= pdu["depth"] <= CANONICALJSON_MAX_INT + ): + filtered_pdus.append(pdu) + + return filtered_pdus + + +def serialize_and_filter_pdus( + pdus: Sequence[EventBase], time_now: Optional[int] = None +) -> List[JsonDict]: + return filter_pdus_for_valid_depth([pdu.get_pdu_json(time_now) for pdu in pdus]) From 31110f35d92ea8dfcae14d7a8897d6a89277640d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 26 Mar 2025 14:35:54 -0500 Subject: [PATCH 217/691] Add docs for how to clear out the Poetry wheel cache (#18283) As shared by @reivilibre, https://github.com/element-hq/synapse/pull/18261#issuecomment-2754607816 Relevant Poetry issue around how this should be handled by them: https://github.com/python-poetry/poetry/issues/10304 --- changelog.d/18283.doc | 1 + docs/development/dependencies.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog.d/18283.doc diff --git a/changelog.d/18283.doc b/changelog.d/18283.doc new file mode 100644 index 0000000000..77fa08fa6b --- /dev/null +++ b/changelog.d/18283.doc @@ -0,0 +1 @@ +Add docs for how to clear out the Poetry wheel cache. diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index 8e29ff3a57..fa5ff4dcf7 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -150,6 +150,28 @@ $ poetry shell $ poetry install --extras all ``` +If you want to go even further and remove the Poetry caches: + +```shell +# Find your Poetry cache directory +# Docs: https://github.com/python-poetry/poetry/blob/main/docs/configuration.md#cache-directory +$ poetry config cache-dir + +# Remove packages from all cached repositories +$ poetry cache clear --all . + +# Go completely nuclear and clear out everything Poetry cache related +# including the wheel artifacts which is not covered by the above command +# (see https://github.com/python-poetry/poetry/issues/10304) +# +# This is necessary in order to rebuild or fetch new wheels. For example, if you update +# the `icu` library in on your system, you will need to rebuild the PyICU Python package +# in order to incorporate the correct dynamically linked library locations otherwise you +# will run into errors like: `ImportError: libicui18n.so.75: cannot open shared object file: No such file or directory` +$ rm -rf $(poetry config cache-dir) +``` + + ## ...run a command in the `poetry` virtualenv? Use `poetry run cmd args` when you need the python virtualenv context. From ecc09b15f108a49348777c908af0187cf26d281e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 26 Mar 2025 21:08:00 +0000 Subject: [PATCH 218/691] 1.127.1 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f63eabb58a..0176c6e45d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.127.1 (2025-03-26) + +## Security +- Fix [CVE-2025-30355](https://www.cve.org/CVERecord?id=CVE-2025-30355) / [GHSA-v56r-hwv5-mxg6](https://github.com/element-hq/synapse/security/advisories/GHSA-v56r-hwv5-mxg6). **High severity vulnerability affecting federation. The vulnerability has been exploited in the wild.** + + + # Synapse 1.127.0 (2025-03-25) No significant changes since 1.127.0rc1. diff --git a/debian/changelog b/debian/changelog index 2e36b368d0..f25c28c9dc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.127.1) stable; urgency=medium + + * New Synapse release 1.127.1. + + -- Synapse Packaging team Wed, 26 Mar 2025 21:07:31 +0000 + matrix-synapse-py3 (1.127.0) stable; urgency=medium * New Synapse release 1.127.0. diff --git a/pyproject.toml b/pyproject.toml index 6a29362919..e91a75445c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.127.0" +version = "1.127.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From a39b856cf0e73def842ca9697f57dd9df646f080 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 27 Mar 2025 14:56:16 +0000 Subject: [PATCH 219/691] Add DB delta to remove the old state group deletion job (#18284) This background DB delta removes the old state group deletion background update from the `background_updates` table if it exists. The `delete_unreferenced_state_groups_bg_update` update should only exist in that table if a homeserver ran v1.126.0rc1/v1.126.0rc2, and rolled back or forward to any other version of Synapse before letting the update finish. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18284.misc | 1 + .../delta/90/03_remove_old_deletion_bg_update.sql | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 changelog.d/18284.misc create mode 100644 synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql diff --git a/changelog.d/18284.misc b/changelog.d/18284.misc new file mode 100644 index 0000000000..69610adc65 --- /dev/null +++ b/changelog.d/18284.misc @@ -0,0 +1 @@ +Add DB delta to remove the old state group deletion job. diff --git a/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql b/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql new file mode 100644 index 0000000000..1cc6d612b6 --- /dev/null +++ b/synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove the old unreferenced state group deletion background update if it exists +DELETE FROM background_updates WHERE update_name = 'delete_unreferenced_state_groups_bg_update'; From d17295e5c3de642ba2c4e47f1bb2be7b2e4c9c06 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 27 Mar 2025 17:26:34 +0000 Subject: [PATCH 220/691] Store hashes of media files, and allow quarantining by hash. (#18277) This PR makes a few radical changes to media. This now stores the SHA256 hash of each file stored in the database (excluding thumbnails, more on that later). If a set of media is quarantined, any additional uploads of the same file contents or any other files with the same hash will be quarantined at the same time. Currently this does NOT: - De-duplicate media, although a future extension could be to do that. - Run any background jobs to identify the hashes of older files. This could also be a future extension, though the value of doing so is limited to combat the abuse of recent media. - Hash thumbnails. It's assumed that thumbnails are parented to some form of media, so you'd likely be wanting to quarantine the media and the thumbnail at the same time. --- changelog.d/18277.feature | 1 + synapse/media/media_repository.py | 46 ++++- synapse/media/media_storage.py | 84 ++++++++- .../databases/main/media_repository.py | 95 +++++++++- synapse/storage/databases/main/room.py | 172 +++++++++++++++--- synapse/storage/schema/__init__.py | 2 +- .../schema/main/delta/91/01_media_hash.sql | 21 +++ tests/handlers/test_profile.py | 1 + tests/media/test_media_retention.py | 11 +- tests/media/test_media_storage.py | 107 ++++++++++- tests/rest/admin/test_admin.py | 21 ++- tests/rest/admin/test_media.py | 74 ++++++-- tests/rest/client/test_media.py | 3 + tests/rest/media/test_domain_blocking.py | 1 + tests/test_utils/__init__.py | 2 + 15 files changed, 579 insertions(+), 62 deletions(-) create mode 100644 changelog.d/18277.feature create mode 100644 synapse/storage/schema/main/delta/91/01_media_hash.sql diff --git a/changelog.d/18277.feature b/changelog.d/18277.feature new file mode 100644 index 0000000000..3604e732d5 --- /dev/null +++ b/changelog.d/18277.feature @@ -0,0 +1 @@ +Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index cf4cba722a..859b30e029 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -59,7 +59,11 @@ from synapse.media._base import ( respond_with_responder, ) from synapse.media.filepath import MediaFilePaths -from synapse.media.media_storage import MediaStorage +from synapse.media.media_storage import ( + MediaStorage, + SHA256TransparentIOReader, + SHA256TransparentIOWriter, +) from synapse.media.storage_provider import StorageProviderWrapper from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer @@ -301,15 +305,26 @@ class MediaRepository: auth_user: The user_id of the uploader """ file_info = FileInfo(server_name=None, file_id=media_id) - fname = await self.media_storage.store_file(content, file_info) + sha256reader = SHA256TransparentIOReader(content) + # This implements all of IO as it has a passthrough + fname = await self.media_storage.store_file(sha256reader.wrap(), file_info) + sha256 = sha256reader.hexdigest() + should_quarantine = await self.store.get_is_hash_quarantined(sha256) logger.info("Stored local media in file %r", fname) + if should_quarantine: + logger.warn( + "Media has been automatically quarantined as it matched existing quarantined media" + ) + await self.store.update_local_media( media_id=media_id, media_type=media_type, upload_name=upload_name, media_length=content_length, user_id=auth_user, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, ) try: @@ -342,11 +357,19 @@ class MediaRepository: media_id = random_string(24) file_info = FileInfo(server_name=None, file_id=media_id) - - fname = await self.media_storage.store_file(content, file_info) + # This implements all of IO as it has a passthrough + sha256reader = SHA256TransparentIOReader(content) + fname = await self.media_storage.store_file(sha256reader.wrap(), file_info) + sha256 = sha256reader.hexdigest() + should_quarantine = await self.store.get_is_hash_quarantined(sha256) logger.info("Stored local media in file %r", fname) + if should_quarantine: + logger.warn( + "Media has been automatically quarantined as it matched existing quarantined media" + ) + await self.store.store_local_media( media_id=media_id, media_type=media_type, @@ -354,6 +377,9 @@ class MediaRepository: upload_name=upload_name, media_length=content_length, user_id=auth_user, + sha256=sha256, + # TODO: Better name? + quarantined_by="system" if should_quarantine else None, ) try: @@ -756,11 +782,13 @@ class MediaRepository: file_info = FileInfo(server_name=server_name, file_id=file_id) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) try: length, headers = await self.client.download_media( server_name, media_id, - output_stream=f, + # This implements all of BinaryIO as it has a passthrough + output_stream=sha256writer.wrap(), max_size=self.max_upload_size, max_timeout_ms=max_timeout_ms, download_ratelimiter=download_ratelimiter, @@ -825,6 +853,7 @@ class MediaRepository: upload_name=upload_name, media_length=length, filesystem_id=file_id, + sha256=sha256writer.hexdigest(), ) logger.info("Stored remote media in file %r", fname) @@ -845,6 +874,7 @@ class MediaRepository: last_access_ts=time_now_ms, quarantined_by=None, authenticated=authenticated, + sha256=sha256writer.hexdigest(), ) async def _federation_download_remote_file( @@ -879,11 +909,13 @@ class MediaRepository: file_info = FileInfo(server_name=server_name, file_id=file_id) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) try: res = await self.client.federation_download_media( server_name, media_id, - output_stream=f, + # This implements all of BinaryIO as it has a passthrough + output_stream=sha256writer.wrap(), max_size=self.max_upload_size, max_timeout_ms=max_timeout_ms, download_ratelimiter=download_ratelimiter, @@ -954,6 +986,7 @@ class MediaRepository: upload_name=upload_name, media_length=length, filesystem_id=file_id, + sha256=sha256writer.hexdigest(), ) logger.debug("Stored remote media in file %r", fname) @@ -974,6 +1007,7 @@ class MediaRepository: last_access_ts=time_now_ms, quarantined_by=None, authenticated=authenticated, + sha256=sha256writer.hexdigest(), ) def _get_thumbnail_requirements( diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index c25d1a9ba3..afd33c02a1 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -19,6 +19,7 @@ # # import contextlib +import hashlib import json import logging import os @@ -70,6 +71,88 @@ logger = logging.getLogger(__name__) CRLF = b"\r\n" +class SHA256TransparentIOWriter: + """Will generate a SHA256 hash from a source stream transparently. + + Args: + source: Source stream. + """ + + def __init__(self, source: BinaryIO): + self._hash = hashlib.sha256() + self._source = source + + def write(self, buffer: Union[bytes, bytearray]) -> int: + """Wrapper for source.write() + + Args: + buffer + + Returns: + the value of source.write() + """ + res = self._source.write(buffer) + self._hash.update(buffer) + return res + + def hexdigest(self) -> str: + """The digest of the written or read value. + + Returns: + The digest in hex formaat. + """ + return self._hash.hexdigest() + + def wrap(self) -> BinaryIO: + # This class implements a subset the IO interface and passes through everything else via __getattr__ + return cast(BinaryIO, self) + + # Passthrough any other calls + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._source, attr_name) + + +class SHA256TransparentIOReader: + """Will generate a SHA256 hash from a source stream transparently. + + Args: + source: Source IO stream. + """ + + def __init__(self, source: IO): + self._hash = hashlib.sha256() + self._source = source + + def read(self, n: int = -1) -> bytes: + """Wrapper for source.read() + + Args: + n + + Returns: + the value of source.read() + """ + bytes = self._source.read(n) + self._hash.update(bytes) + return bytes + + def hexdigest(self) -> str: + """The digest of the written or read value. + + Returns: + The digest in hex formaat. + """ + return self._hash.hexdigest() + + def wrap(self) -> IO: + # This class implements a subset the IO interface and passes through everything else via __getattr__ + return cast(IO, self) + + # Passthrough any other calls + def __getattr__(self, attr_name: str) -> Any: + return getattr(self._source, attr_name) + + class MediaStorage: """Responsible for storing/fetching files from local sources. @@ -107,7 +190,6 @@ class MediaStorage: Returns: the file path written to in the primary media store """ - async with self.store_into_file(file_info) as (f, fname): # Write to the main media repository await self.write_to_file(source, f) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 7a96e25432..8a5ba44922 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -19,6 +19,7 @@ # [This file includes modifications made by New Vector Limited] # # +import logging from enum import Enum from typing import ( TYPE_CHECKING, @@ -51,6 +52,8 @@ BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2 = ( "media_repository_drop_index_wo_method_2" ) +logger = logging.getLogger(__name__) + @attr.s(slots=True, frozen=True, auto_attribs=True) class LocalMedia: @@ -65,6 +68,7 @@ class LocalMedia: safe_from_quarantine: bool user_id: Optional[str] authenticated: Optional[bool] + sha256: Optional[str] @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -79,6 +83,7 @@ class RemoteMedia: last_access_ts: int quarantined_by: Optional[str] authenticated: Optional[bool] + sha256: Optional[str] @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -154,6 +159,26 @@ class MediaRepositoryBackgroundUpdateStore(SQLBaseStore): unique=True, ) + self.db_pool.updates.register_background_index_update( + update_name="local_media_repository_sha256_idx", + index_name="local_media_repository_sha256", + table="local_media_repository", + where_clause="sha256 IS NOT NULL", + columns=[ + "sha256", + ], + ) + + self.db_pool.updates.register_background_index_update( + update_name="remote_media_cache_sha256_idx", + index_name="remote_media_cache_sha256", + table="remote_media_cache", + where_clause="sha256 IS NOT NULL", + columns=[ + "sha256", + ], + ) + self.db_pool.updates.register_background_update_handler( BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2, self._drop_media_index_without_method, @@ -221,6 +246,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "safe_from_quarantine", "user_id", "authenticated", + "sha256", ), allow_none=True, desc="get_local_media", @@ -239,6 +265,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): safe_from_quarantine=row[7], user_id=row[8], authenticated=row[9], + sha256=row[10], ) async def get_local_media_by_user_paginate( @@ -295,7 +322,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): quarantined_by, safe_from_quarantine, user_id, - authenticated + authenticated, + sha256 FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -320,6 +348,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): safe_from_quarantine=bool(row[8]), user_id=row[9], authenticated=row[10], + sha256=row[11], ) for row in txn ] @@ -449,6 +478,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): media_length: int, user_id: UserID, url_cache: Optional[str] = None, + sha256: Optional[str] = None, + quarantined_by: Optional[str] = None, ) -> None: if self.hs.config.media.enable_authenticated_media: authenticated = True @@ -466,6 +497,8 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "user_id": user_id.to_string(), "url_cache": url_cache, "authenticated": authenticated, + "sha256": sha256, + "quarantined_by": quarantined_by, }, desc="store_local_media", ) @@ -477,20 +510,28 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): upload_name: Optional[str], media_length: int, user_id: UserID, + sha256: str, url_cache: Optional[str] = None, + quarantined_by: Optional[str] = None, ) -> None: + updatevalues = { + "media_type": media_type, + "upload_name": upload_name, + "media_length": media_length, + "url_cache": url_cache, + "sha256": sha256, + } + + # This should never be un-set by this function. + if quarantined_by is not None: + updatevalues["quarantined_by"] = quarantined_by + await self.db_pool.simple_update_one( "local_media_repository", keyvalues={ - "user_id": user_id.to_string(), "media_id": media_id, }, - updatevalues={ - "media_type": media_type, - "upload_name": upload_name, - "media_length": media_length, - "url_cache": url_cache, - }, + updatevalues=updatevalues, desc="update_local_media", ) @@ -657,6 +698,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "last_access_ts", "quarantined_by", "authenticated", + "sha256", ), allow_none=True, desc="get_cached_remote_media", @@ -674,6 +716,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): last_access_ts=row[5], quarantined_by=row[6], authenticated=row[7], + sha256=row[8], ) async def store_cached_remote_media( @@ -685,6 +728,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): time_now_ms: int, upload_name: Optional[str], filesystem_id: str, + sha256: Optional[str], ) -> None: if self.hs.config.media.enable_authenticated_media: authenticated = True @@ -703,6 +747,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "filesystem_id": filesystem_id, "last_access_ts": time_now_ms, "authenticated": authenticated, + "sha256": sha256, }, desc="store_cached_remote_media", ) @@ -946,3 +991,37 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): await self.db_pool.runInteraction( "delete_url_cache_media", _delete_url_cache_media_txn ) + + async def get_is_hash_quarantined(self, sha256: str) -> bool: + """Get whether a specific sha256 hash digest matches any quarantined media. + + Returns: + None if the media_id doesn't exist. + """ + + def get_matching_media_txn( + txn: LoggingTransaction, table: str, sha256: str + ) -> bool: + # Return on first match + sql = """ + SELECT 1 + FROM local_media_repository + WHERE sha256 = ? AND quarantined_by IS NOT NULL + + UNION ALL + + SELECT 1 + FROM remote_media_cache + WHERE sha256 = ? AND quarantined_by IS NOT NULL + LIMIT 1 + """ + txn.execute(sql, (sha256, sha256)) + row = txn.fetchone() + return row is not None + + return await self.db_pool.runInteraction( + "get_matching_media_txn", + get_matching_media_txn, + "local_media_repository", + sha256, + ) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index d673adba16..56217fccdf 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -51,11 +51,15 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.events import EventBase from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream -from synapse.storage._base import db_to_json, make_in_list_sql_clause +from synapse.storage._base import ( + db_to_json, + make_in_list_sql_clause, +) from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_tuple_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.types import Cursor @@ -1127,6 +1131,109 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): return local_media_ids + def _quarantine_local_media_txn( + self, + txn: LoggingTransaction, + hashes: Set[str], + media_ids: Set[str], + quarantined_by: Optional[str], + ) -> int: + """Quarantine and unquarantine local media items. + + Args: + txn (cursor) + hashes: A set of sha256 hashes for any media that should be quarantined + media_ids: A set of media IDs for any media that should be quarantined + quarantined_by: The ID of the user who initiated the quarantine request + If it is `None` media will be removed from quarantine + Returns: + The total number of media items quarantined + """ + total_media_quarantined = 0 + + # Effectively a legacy path, update any media that was explicitly named. + if media_ids: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "media_id", media_ids + ) + sql = f""" + UPDATE local_media_repository + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + + if quarantined_by is not None: + sql += " AND safe_from_quarantine = FALSE" + + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + # Note that a rowcount of -1 can be used to indicate no rows were affected. + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + # Update any media that was identified via hash. + if hashes: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "sha256", hashes + ) + sql = f""" + UPDATE local_media_repository + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + + if quarantined_by is not None: + sql += " AND safe_from_quarantine = FALSE" + + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + return total_media_quarantined + + def _quarantine_remote_media_txn( + self, + txn: LoggingTransaction, + hashes: Set[str], + media: Set[Tuple[str, str]], + quarantined_by: Optional[str], + ) -> int: + """Quarantine and unquarantine remote items + + Args: + txn (cursor) + hashes: A set of sha256 hashes for any media that should be quarantined + media_ids: A set of tuples (media_origin, media_id) for any media that should be quarantined + quarantined_by: The ID of the user who initiated the quarantine request + If it is `None` media will be removed from quarantine + Returns: + The total number of media items quarantined + """ + total_media_quarantined = 0 + + if media: + sql_in_list_clause, sql_args = make_tuple_in_list_sql_clause( + txn.database_engine, + ("media_origin", "media_id"), + media, + ) + sql = f""" + UPDATE remote_media_cache + SET quarantined_by = ? + WHERE {sql_in_list_clause}""" + + txn.execute(sql, [quarantined_by] + sql_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + total_media_quarantined = 0 + if hashes: + sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( + txn.database_engine, "sha256", hashes + ) + sql = f""" + UPDATE remote_media_cache + SET quarantined_by = ? + WHERE {sql_many_clause_sql}""" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) + total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + + return total_media_quarantined + def _quarantine_media_txn( self, txn: LoggingTransaction, @@ -1146,40 +1253,49 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): Returns: The total number of media items quarantined """ + hashes = set() + media_ids = set() + remote_media = set() - # Update all the tables to set the quarantined_by flag - sql = """ - UPDATE local_media_repository - SET quarantined_by = ? - WHERE media_id = ? - """ - - # set quarantine - if quarantined_by is not None: - sql += "AND safe_from_quarantine = FALSE" - txn.executemany( - sql, [(quarantined_by, media_id) for media_id in local_mxcs] + # First, determine the hashes of the media we want to delete. + # We also want the media_ids for any media that lacks a hash. + if local_mxcs: + hash_sql_many_clause_sql, hash_sql_many_clause_args = ( + make_in_list_sql_clause(txn.database_engine, "media_id", local_mxcs) ) - # remove from quarantine - else: - txn.executemany( - sql, [(quarantined_by, media_id) for media_id in local_mxcs] + hash_sql = f"SELECT sha256, media_id FROM local_media_repository WHERE {hash_sql_many_clause_sql}" + if quarantined_by is not None: + hash_sql += " AND safe_from_quarantine = FALSE" + + txn.execute(hash_sql, hash_sql_many_clause_args) + for sha256, media_id in txn: + if sha256: + hashes.add(sha256) + else: + media_ids.add(media_id) + + # Do the same for remote media + if remote_mxcs: + hash_sql_in_list_clause, hash_sql_args = make_tuple_in_list_sql_clause( + txn.database_engine, + ("media_origin", "media_id"), + remote_mxcs, ) - # Note that a rowcount of -1 can be used to indicate no rows were affected. - total_media_quarantined = txn.rowcount if txn.rowcount > 0 else 0 + hash_sql = f"SELECT sha256, media_origin, media_id FROM remote_media_cache WHERE {hash_sql_in_list_clause}" + txn.execute(hash_sql, hash_sql_args) + for sha256, media_origin, media_id in txn: + if sha256: + hashes.add(sha256) + else: + remote_media.add((media_origin, media_id)) - txn.executemany( - """ - UPDATE remote_media_cache - SET quarantined_by = ? - WHERE media_origin = ? AND media_id = ? - """, - [(quarantined_by, origin, media_id) for origin, media_id in remote_mxcs], + count = self._quarantine_local_media_txn(txn, hashes, media_ids, quarantined_by) + count += self._quarantine_remote_media_txn( + txn, hashes, remote_media, quarantined_by ) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 - return total_media_quarantined + return count async def block_room(self, room_id: str, user_id: str) -> None: """Marks the room as blocked. diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 2160edb014..ad683a3a07 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 90 # remember to update the list below when updating +SCHEMA_VERSION = 91 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the diff --git a/synapse/storage/schema/main/delta/91/01_media_hash.sql b/synapse/storage/schema/main/delta/91/01_media_hash.sql new file mode 100644 index 0000000000..2dbd2c7df7 --- /dev/null +++ b/synapse/storage/schema/main/delta/91/01_media_hash.sql @@ -0,0 +1,21 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Store the SHA256 content hash of media files. +ALTER TABLE local_media_repository ADD COLUMN sha256 TEXT; +ALTER TABLE remote_media_cache ADD COLUMN sha256 TEXT; + +-- Add a background updates to handle creating the new index. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9101, 'local_media_repository_sha256_idx', '{}'), + (9101, 'remote_media_cache_sha256_idx', '{}'); \ No newline at end of file diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index cb1c6fbb80..2b9b56da95 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -369,6 +369,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): time_now_ms=self.clock.time_msec(), upload_name=None, filesystem_id="xyz", + sha256="abcdefg12345", ) ) diff --git a/tests/media/test_media_retention.py b/tests/media/test_media_retention.py index 417d17ebd2..d8f4f57c8c 100644 --- a/tests/media/test_media_retention.py +++ b/tests/media/test_media_retention.py @@ -31,6 +31,9 @@ from synapse.rest.client import login, register, room from synapse.server import HomeServer from synapse.types import UserID from synapse.util import Clock +from synapse.util.stringutils import ( + random_string, +) from tests import unittest from tests.unittest import override_config @@ -65,7 +68,6 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): # quarantined media) into both the local store and the remote cache, plus # one additional local media that is marked as protected from quarantine. media_repository = hs.get_media_repository() - test_media_content = b"example string" def _create_media_and_set_attributes( last_accessed_ms: Optional[int], @@ -73,12 +75,14 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): is_protected: Optional[bool] = False, ) -> MXCUri: # "Upload" some media to the local media store + # If the meda + random_content = bytes(random_string(24), "utf-8") mxc_uri: MXCUri = self.get_success( media_repository.create_content( media_type="text/plain", upload_name=None, - content=io.BytesIO(test_media_content), - content_length=len(test_media_content), + content=io.BytesIO(random_content), + content_length=len(random_content), auth_user=UserID.from_string(test_user_id), ) ) @@ -129,6 +133,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="testfile.txt", filesystem_id="abcdefg12345", + sha256=random_string(24), ) ) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index c2e0e592d7..35e16a99ba 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -42,6 +42,7 @@ from twisted.web.resource import Resource from synapse.api.errors import Codes, HttpResponseException from synapse.api.ratelimiting import Ratelimiter from synapse.events import EventBase +from synapse.http.client import ByteWriteable from synapse.http.types import QueryParams from synapse.logging.context import make_deferred_yieldable from synapse.media._base import FileInfo, ThumbnailInfo @@ -59,7 +60,7 @@ from synapse.util import Clock from tests import unittest from tests.server import FakeChannel -from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG +from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG, SMALL_PNG_SHA256 from tests.unittest import override_config from tests.utils import default_config @@ -1257,3 +1258,107 @@ class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): ) assert channel.code == 502 assert channel.json_body["errcode"] == "M_TOO_LARGE" + + +def read_body( + response: IResponse, stream: ByteWriteable, max_size: Optional[int] +) -> Deferred: + d: Deferred = defer.Deferred() + stream.write(SMALL_PNG) + d.callback(len(SMALL_PNG)) + return d + + +class MediaHashesTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + self.store = hs.get_datastores().main + self.client = hs.get_federation_http_client() + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def test_ensure_correct_sha256(self) -> None: + """Check that the hash does not change""" + media = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc = media.get("content_uri") + assert mxc + store_media = self.get_success(self.store.get_local_media(mxc[11:])) + assert store_media + self.assertEqual( + store_media.sha256, + SMALL_PNG_SHA256, + ) + + def test_ensure_multiple_correct_sha256(self) -> None: + """Check that two media items have the same hash.""" + media_a = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc_a = media_a.get("content_uri") + assert mxc_a + store_media_a = self.get_success(self.store.get_local_media(mxc_a[11:])) + assert store_media_a + + media_b = self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + mxc_b = media_b.get("content_uri") + assert mxc_b + store_media_b = self.get_success(self.store.get_local_media(mxc_b[11:])) + assert store_media_b + + self.assertNotEqual( + store_media_a.media_id, + store_media_b.media_id, + ) + self.assertEqual( + store_media_a.sha256, + store_media_b.sha256, + ) + + @override_config( + { + "enable_authenticated_media": False, + } + ) + # mock actually reading file body + @patch( + "synapse.http.matrixfederationclient.read_body_with_max_size", + read_body, + ) + def test_ensure_correct_sha256_federated(self) -> None: + """Check that federated media have the same hash.""" + + # Mock getting a file over federation + async def _send_request(*args: Any, **kwargs: Any) -> IResponse: + resp = MagicMock(spec=IResponse) + resp.code = 200 + resp.length = 500 + resp.headers = Headers({"Content-Type": ["application/octet-stream"]}) + resp.phrase = b"OK" + return resp + + self.client._send_request = _send_request # type: ignore + + # first request should go through + channel = self.make_request( + "GET", + "/_matrix/media/v3/download/remote.org/abc", + shorthand=False, + access_token=self.tok, + ) + assert channel.code == 200 + store_media = self.get_success( + self.store.get_cached_remote_media("remote.org", "abc") + ) + assert store_media + self.assertEqual( + store_media.sha256, + SMALL_PNG_SHA256, + ) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 5483f8f37f..fc2a6c569b 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -20,7 +20,7 @@ # import urllib.parse -from typing import Dict +from typing import Dict, cast from parameterized import parameterized @@ -32,6 +32,7 @@ from synapse.http.server import JsonResource from synapse.rest.admin import VersionServlet from synapse.rest.client import login, media, room from synapse.server import HomeServer +from synapse.types import UserID from synapse.util import Clock from tests import unittest @@ -227,10 +228,25 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): # Upload some media response_1 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) response_2 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) + response_3 = self.helper.upload_media(SMALL_PNG, tok=non_admin_user_tok) # Extract media IDs server_and_media_id_1 = response_1["content_uri"][6:] server_and_media_id_2 = response_2["content_uri"][6:] + server_and_media_id_3 = response_3["content_uri"][6:] + + # Remove the hash from the media to simulate historic media. + self.get_success( + self.hs.get_datastores().main.update_local_media( + media_id=server_and_media_id_3.split("/")[1], + media_type="image/png", + upload_name=None, + media_length=123, + user_id=UserID.from_string(non_admin_user), + # Hack to force some media to have no hash. + sha256=cast(str, None), + ) + ) # Quarantine all media by this user url = "/_synapse/admin/v1/user/%s/media/quarantine" % urllib.parse.quote( @@ -244,12 +260,13 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): self.pump(1.0) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual( - channel.json_body, {"num_quarantined": 2}, "Expected 2 quarantined items" + channel.json_body, {"num_quarantined": 3}, "Expected 3 quarantined items" ) # Attempt to access each piece of media self._ensure_quarantined(admin_user_tok, server_and_media_id_1) self._ensure_quarantined(admin_user_tok, server_and_media_id_2) + self._ensure_quarantined(admin_user_tok, server_and_media_id_3) def test_cannot_quarantine_safe_media(self) -> None: self.register_user("user_admin", "pass", admin=True) diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 19c244cfcf..da0e9749aa 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -35,7 +35,7 @@ from synapse.server import HomeServer from synapse.util import Clock from tests import unittest -from tests.test_utils import SMALL_PNG +from tests.test_utils import SMALL_CMYK_JPEG, SMALL_PNG from tests.unittest import override_config VALID_TIMESTAMP = 1609459200000 # 2021-01-01 in milliseconds @@ -598,23 +598,27 @@ class DeleteMediaByDateSizeTestCase(_AdminMediaTests): class QuarantineMediaByIDTestCase(_AdminMediaTests): + def upload_media_and_return_media_id(self, data: bytes) -> str: + # Upload some media into the room + response = self.helper.upload_media( + data, + tok=self.admin_user_tok, + expect_code=200, + ) + # Extract media ID from the response + server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' + return server_and_media_id.split("/")[1] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") - - # Upload some media into the room - response = self.helper.upload_media( - SMALL_PNG, - tok=self.admin_user_tok, - expect_code=200, - ) - # Extract media ID from the response - server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' - self.media_id = server_and_media_id.split("/")[1] - + self.media_id = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_2 = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_3 = self.upload_media_and_return_media_id(SMALL_PNG) + self.media_id_other = self.upload_media_and_return_media_id(SMALL_CMYK_JPEG) self.url = "/_synapse/admin/v1/media/%s/%s/%s" @parameterized.expand(["quarantine", "unquarantine"]) @@ -686,6 +690,52 @@ class QuarantineMediaByIDTestCase(_AdminMediaTests): assert media_info is not None self.assertFalse(media_info.quarantined_by) + def test_quarantine_media_match_hash(self) -> None: + """ + Tests that quarantining removes all media with the same hash + """ + + media_info = self.get_success(self.store.get_local_media(self.media_id)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + + # quarantining + channel = self.make_request( + "POST", + self.url % ("quarantine", self.server_name, self.media_id), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertFalse(channel.json_body) + + # Test that ALL similar media was quarantined. + for media in [self.media_id, self.media_id_2, self.media_id_3]: + media_info = self.get_success(self.store.get_local_media(media)) + assert media_info is not None + self.assertTrue(media_info.quarantined_by) + + # Test that other media was not. + media_info = self.get_success(self.store.get_local_media(self.media_id_other)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + + # remove from quarantine + channel = self.make_request( + "POST", + self.url % ("unquarantine", self.server_name, self.media_id), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertFalse(channel.json_body) + + # Test that ALL similar media is now reset. + for media in [self.media_id, self.media_id_2, self.media_id_3]: + media_info = self.get_success(self.store.get_local_media(media)) + assert media_info is not None + self.assertFalse(media_info.quarantined_by) + def test_quarantine_protected_media(self) -> None: """ Tests that quarantining from protected media fails diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 0e3e370ee8..1ea2a5c884 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -137,6 +137,7 @@ class MediaDomainBlockingTests(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="test.png", filesystem_id=file_id, + sha256=file_id, ) ) self.register_user("user", "password") @@ -2593,6 +2594,7 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): time_now_ms=self.clock.time_msec(), upload_name="remote_test.png", filesystem_id=file_id, + sha256=file_id, ) ) @@ -2725,6 +2727,7 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): time_now_ms=self.clock.time_msec(), upload_name="remote_test.png", filesystem_id=file_id, + sha256=file_id, ) ) diff --git a/tests/rest/media/test_domain_blocking.py b/tests/rest/media/test_domain_blocking.py index 49d81f4b28..26453f70dd 100644 --- a/tests/rest/media/test_domain_blocking.py +++ b/tests/rest/media/test_domain_blocking.py @@ -61,6 +61,7 @@ class MediaDomainBlockingTests(unittest.HomeserverTestCase): time_now_ms=clock.time_msec(), upload_name="test.png", filesystem_id=file_id, + sha256=file_id, ) ) diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index e3611852b7..3e6fd03600 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -139,6 +139,8 @@ SMALL_PNG = unhexlify( b"0000001f15c4890000000a49444154789c63000100000500010d" b"0a2db40000000049454e44ae426082" ) +# The SHA256 hexdigest for the above bytes. +SMALL_PNG_SHA256 = "ebf4f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a" # A small CMYK-encoded JPEG image used in some tests. # From 3c188231c76ee8c05a6a40d12ccfdebada86b406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Pennewi=C3=9F?= Date: Thu, 27 Mar 2025 18:31:37 +0100 Subject: [PATCH 221/691] Update admin_faq - Fix how to obtain access token (#18225) Riot is now known as element and Access token moved to Help & About --- changelog.d/18225.doc | 1 + docs/usage/administration/admin_faq.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18225.doc diff --git a/changelog.d/18225.doc b/changelog.d/18225.doc new file mode 100644 index 0000000000..854e275120 --- /dev/null +++ b/changelog.d/18225.doc @@ -0,0 +1 @@ +Fix how to obtain access token and change naming from riot to element diff --git a/docs/usage/administration/admin_faq.md b/docs/usage/administration/admin_faq.md index 0dce3d3e37..1be432000c 100644 --- a/docs/usage/administration/admin_faq.md +++ b/docs/usage/administration/admin_faq.md @@ -160,7 +160,7 @@ Using the following curl command: ```console curl -H 'Authorization: Bearer ' -X DELETE https://matrix.org/_matrix/client/r0/directory/room/ ``` -`` - can be obtained in riot by looking in the riot settings, down the bottom is: +`` - can be obtained in element by looking in All settings, clicking Help & About and down the bottom is: Access Token:\ `` - the room alias, eg. #my_room:matrix.org this possibly needs to be URL encoded also, for example %23my_room%3Amatrix.org From 76ffd3ba018f822d2309c1a69ab37251ce587f83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:55:30 +0200 Subject: [PATCH 222/691] Bump actions/cache from 4.2.2 to 4.2.3 (#18266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 4.2.2 to 4.2.3.
Release notes

Sourced from actions/cache's releases.

v4.2.3

What's Changed

  • Update to use @​actions/cache 4.0.3 package & prepare for new release by @​salmanmkc in actions/cache#1577 (SAS tokens for cache entries are now masked in debug logs)

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v4.2.2...v4.2.3

Changelog

Sourced from actions/cache's changelog.

Releases

4.2.3

  • Bump @actions/cache to v4.0.3 (obfuscates SAS token in debug logs for cache entries)

4.2.2

  • Bump @actions/cache to v4.0.2

4.2.1

  • Bump @actions/cache to v4.0.1

4.2.0

TLDR; The cache backend service has been rewritten from the ground up for improved performance and reliability. actions/cache now integrates with the new cache service (v2) APIs.

The new service will gradually roll out as of February 1st, 2025. The legacy service will also be sunset on the same date. Changes in these release are fully backward compatible.

We are deprecating some versions of this action. We recommend upgrading to version v4 or v3 as soon as possible before February 1st, 2025. (Upgrade instructions below).

If you are using pinned SHAs, please use the SHAs of versions v4.2.0 or v3.4.0

If you do not upgrade, all workflow runs using any of the deprecated actions/cache will fail.

Upgrading to the recommended versions will not break your workflows.

4.1.2

  • Add GitHub Enterprise Cloud instances hostname filters to inform API endpoint choices - #1474
  • Security fix: Bump braces from 3.0.2 to 3.0.3 - #1475

4.1.1

  • Restore original behavior of cache-hit output - #1467

4.1.0

  • Ensure cache-hit output is set when a cache is missed - #1404
  • Deprecate save-always input - #1452

4.0.2

  • Fixed restore fail-on-cache-miss not working.

4.0.1

  • Updated isGhes check

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=4.2.2&new-version=4.2.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-artifacts.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 9985084a73..dda2b834fc 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -66,7 +66,7 @@ jobs: install: true - name: Set up docker layer caching - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff83d6e365..4193d3e2f9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -167,7 +167,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: | .mypy_cache From 4dba011c316139eb58971d4fa706f9aca36f02ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:56:53 +0200 Subject: [PATCH 223/691] Bump dawidd6/action-download-artifact from 8 to 9 (#18204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [dawidd6/action-download-artifact](https://github.com/dawidd6/action-download-artifact) from 8 to 9.
Release notes

Sourced from dawidd6/action-download-artifact's releases.

v9

What's Changed

New Contributors

Full Changelog: https://github.com/dawidd6/action-download-artifact/compare/v8...v9

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=dawidd6/action-download-artifact&package-manager=github_actions&previous-version=8&new-version=9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr-netlify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index d1b789c8d0..8a06ad6362 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 + uses: dawidd6/action-download-artifact@07ab29fd4a977ae4d2b275087cf67563dfdf0295 # v9 with: workflow: docs-pr.yaml run_id: ${{ github.event.workflow_run.id }} From 3bc04d05a4454e59c55688a26cea3f542f2697ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:58:27 +0200 Subject: [PATCH 224/691] Bump pygithub from 2.5.0 to 2.6.1 (#18243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pygithub](https://github.com/pygithub/pygithub) from 2.5.0 to 2.6.1.
Release notes

Sourced from pygithub's releases.

v2.6.1

Bug Fixes

Maintenance

Full Changelog: https://github.com/PyGithub/PyGithub/compare/v2.6.0...v2.6.1

v2.6.0

Breaking Changes

  • Rework Views and Clones by @​EnricoMi in PyGithub/PyGithub#3168: View and clones traffic information returned by Repository.get_views_traffic and Repository.get_clones_traffic now return proper PyGithub objects, instead of a dict, with all information that used to be provided by the dict:

Code like

repo.get_views_traffic().["views"].timestamp
repo.get_clones_traffic().["clones"].timestamp

should be replaced with

repo.get_views_traffic().views.timestamp
repo.get_clones_traffic().clones.timestamp

New Features

Improvements

... (truncated)

Changelog

Sourced from pygithub's changelog.

Version 2.6.1 (February 21, 2025)

Bug Fixes ^^^^^^^^^

  • Fix broken pickle support for Auth classes ([#3211](https://github.com/pygithub/pygithub/issues/3211) <https://github.com/PyGithub/PyGithub/pull/3211>) (f975552a <https://github.com/PyGithub/PyGithub/commit/f975552a>)
  • Remove schema from Deployment, remove message attribute ([#3223](https://github.com/pygithub/pygithub/issues/3223) <https://github.com/PyGithub/PyGithub/pull/3223>) (d12e7d4c <https://github.com/PyGithub/PyGithub/commit/d12e7d4c>)
  • Fix incorrect deprecated import ([#3225](https://github.com/pygithub/pygithub/issues/3225) <https://github.com/PyGithub/PyGithub/pull/3225>) (93297440 <https://github.com/PyGithub/PyGithub/commit/93297440>)
  • Add CodeSecurityConfigRepository returned by get_repos_for_code_security_config ([#3219](https://github.com/pygithub/pygithub/issues/3219) <https://github.com/PyGithub/PyGithub/pull/3219>) (f997a2f6 <https://github.com/PyGithub/PyGithub/commit/f997a2f6>)
  • Make GitTag.verification return GitCommitVerification ([#3226](https://github.com/pygithub/pygithub/issues/3226) <https://github.com/PyGithub/PyGithub/pull/3226>) (048a1a38 <https://github.com/PyGithub/PyGithub/commit/048a1a38>)

Maintenance ^^^^^^^^^^^

  • Mention removal of AppAuth.private_key in changelog ([#3212](https://github.com/pygithub/pygithub/issues/3212) <https://github.com/PyGithub/PyGithub/pull/3212>) (f5dc1c76 <https://github.com/PyGithub/PyGithub/commit/f5dc1c76>)

Version 2.6.0 (February 15, 2025)

Breaking Changes ^^^^^^^^^^^^^^^^

  • Rework Views and Clones ([#3168](https://github.com/pygithub/pygithub/issues/3168) <https://github.com/PyGithub/PyGithub/pull/3168>) (f7d52249 <https://github.com/PyGithub/PyGithub/commit/f7d52249>):

    View and clones traffic information returned by Repository.get_views_traffic and Repository.get_clones_traffic now return proper PyGithub objects, instead of a dict, with all information that used to be provided by the dict:

Code like

.. code-block:: python

repo.get_views_traffic().["views"].timestamp repo.get_clones_traffic().["clones"].timestamp

should be replaced with

.. code-block:: python

repo.get_views_traffic().views.timestamp repo.get_clones_traffic().clones.timestamp

  • Add GitCommitVerification class ([#3028](https://github.com/pygithub/pygithub/issues/3028) <https://github.com/PyGithub/PyGithub/pull/3028>) (822e6d71 <https://github.com/PyGithub/PyGithub/commit/822e6d71>):

    Changes the return value of GitTag.verification and GitCommit.verification from dict to GitCommitVerification.

    Code like

    .. code-block:: python

    tag.verification["reason"] commit.verification["reason"]

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pygithub&package-manager=pip&previous-version=2.5.0&new-version=2.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index ed98dd9f4e..cfbfdc427a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "Authlib-1.4.1-py2.py3-none-any.whl", hash = "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"}, {file = "authlib-1.4.1.tar.gz", hash = "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d"}, @@ -451,7 +451,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -494,7 +494,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -544,7 +544,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -890,7 +890,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1028,7 +1028,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1044,7 +1044,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -1330,7 +1330,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1551,7 +1551,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1709,7 +1709,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1730,7 +1730,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1746,7 +1746,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1929,14 +1929,14 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pygithub" -version = "2.5.0" +version = "2.6.1" description = "Use the full Github API v3" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2"}, - {file = "pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf"}, + {file = "PyGithub-2.6.1-py3-none-any.whl", hash = "sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3"}, + {file = "pygithub-2.6.1.tar.gz", hash = "sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf"}, ] [package.dependencies] @@ -1969,7 +1969,7 @@ description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"user-search\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"user-search\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -2018,7 +2018,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2077,7 +2077,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2102,7 +2102,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2130,7 +2130,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2494,7 +2494,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2678,7 +2678,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2694,7 +2694,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2756,7 +2756,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, @@ -2890,7 +2890,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.10-py3-none-any.whl", hash = "sha256:0a6ea77f27f8cf092f907654f08302a97b48fa35f24e0ad99dfb74115f018161"}, {file = "txredisapi-1.4.10.tar.gz", hash = "sha256:7609a6af6ff4619a3189c0adfb86aeda789afba69eb59fc1e19ac0199e725395"}, @@ -3218,7 +3218,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From 9f8ed145356026c83d4af8d25ed60fbec8d99b32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 08:08:57 +0000 Subject: [PATCH 225/691] Bump actions/download-artifact from 4.2.0 to 4.2.1 (#18268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.2.0 to 4.2.1.
Release notes

Sourced from actions/download-artifact's releases.

v4.2.1

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v4.2.0...v4.2.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=4.2.0&new-version=4.2.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index dda2b834fc..e9e2ac2606 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -203,7 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@b14cf4c92620c250e1c074ab0a5800e37df86765 # v4.2.0 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. From 02eed668b844b887e519be3a8995e1a7ebe7ff3f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 1 Apr 2025 11:43:05 +0100 Subject: [PATCH 226/691] Document media hashing changes (#18296) Essentially document the change in behaviour in #18277 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18296.doc | 1 + docs/admin_api/media_admin_api.md | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/18296.doc diff --git a/changelog.d/18296.doc b/changelog.d/18296.doc new file mode 100644 index 0000000000..3604e732d5 --- /dev/null +++ b/changelog.d/18296.doc @@ -0,0 +1 @@ +Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 30833f3109..1177711c1e 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -46,6 +46,14 @@ to any local media, and any locally-cached copies of remote media. The media file itself (and any thumbnails) is not deleted from the server. +Since Synapse 1.128.0, hashes of uploaded media are tracked. If this media +is quarantined, Synapse will: + + - Quarantine any media with a matching hash that has already been uploaded. + - Quarantine any future media. + - Quarantine any existing cached remote media. + - Quarantine any future remote media. + ## Quarantining media by ID This API quarantines a single piece of local or remote media. From 1deb6e03e0388c2acc76d74257968253107c40a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:12:58 +0200 Subject: [PATCH 227/691] Bump pyo3-log from 0.12.1 to 0.12.2 (#18269) Bumps [pyo3-log](https://github.com/vorner/pyo3-log) from 0.12.1 to 0.12.2.
Changelog

Sourced from pyo3-log's changelog.

0.12.2

  • Allow pyo3 0.24.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pyo3-log&package-manager=cargo&previous-version=0.12.1&new-version=0.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20110694a5..1b17e9910a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" +checksum = "4b78e4983ba15bc62833a0e0941d965bc03690163f1127864f1408db25063466" dependencies = [ "arc-swap", "log", From cf02b8fea5659154274a50d98831d1e8e71cc758 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 12:31:59 +0000 Subject: [PATCH 228/691] Bump actions/setup-python from 5.4.0 to 5.5.0 (#18298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.4.0 to 5.5.0.
Release notes

Sourced from actions/setup-python's releases.

v5.5.0

What's Changed

Enhancements:

Bug fixes:

Dependency updates:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v5...v5.5.0

Commits
  • 8d9ed9a Add e2e Testing for free threaded and Bump @​action/cache from 4.0.0 to 4.0.3 ...
  • 19e4675 Add support for .tool-versions file in setup-python (#1043)
  • 6fd11e1 Bump @​actions/glob from 0.4.0 to 0.5.0 (#1015)
  • 9e62be8 Support free threaded Python versions like '3.13t' (#973)
  • 6ca8e85 Bump @​vercel/ncc from 0.38.1 to 0.38.3 (#1016)
  • 8039c45 fix: install PyPy on Linux ARM64 (#1011)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=5.4.0&new-version=5.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/tests.yml | 12 ++++++------ 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 1b416407d8..4a865930ac 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.4.17' - name: Setup python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 121c29571a..05ae608d06 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index b7c1b727c9..e37e0c8598 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -86,7 +86,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - run: pip install .[all,test] diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index b3251d379e..31b9147e98 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index e9e2ac2606..9fd38e49a6 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: '3.x' - id: set-distros @@ -74,7 +74,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: '3.x' @@ -132,7 +132,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -177,7 +177,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: '3.10' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4193d3e2f9..9df1ec506b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" @@ -112,7 +112,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -192,7 +192,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -279,7 +279,7 @@ jobs: if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -327,7 +327,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: "3.x" - id: get-matrix @@ -414,7 +414,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 with: python-version: '3.9' From f638a76ba4ef242794493146b944529d3a2bd5ad Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 1 Apr 2025 08:32:34 -0400 Subject: [PATCH 229/691] Avoid relying on rsync during Docker build (#18287) Use targeted COPY commands instead of rsync to avoid having a symlinked /lib as the destination of a COPY (which buildkit does not support). ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18287.docker | 1 + docker/Dockerfile | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18287.docker diff --git a/changelog.d/18287.docker b/changelog.d/18287.docker new file mode 100644 index 0000000000..ef45ad72ba --- /dev/null +++ b/changelog.d/18287.docker @@ -0,0 +1 @@ +Avoid needing to download & use rsync in a build layer. diff --git a/docker/Dockerfile b/docker/Dockerfile index 54aa355370..15c458fa28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -134,7 +134,6 @@ RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ - apt-get install -y --no-install-recommends rsync && \ apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends \ curl \ gosu \ @@ -152,10 +151,6 @@ RUN \ done # Extract the debs for each architecture -# On the runtime image, /lib is a symlink to /usr/lib, so we need to copy the -# libraries to the right place, else the `COPY` won't work. -# On amd64, we'll also have a /lib64 folder with ld-linux-x86-64.so.2, which is -# already present in the runtime image. RUN \ for arch in arm64 amd64; do \ mkdir -p /install-${arch}/var/lib/dpkg/status.d/ && \ @@ -165,8 +160,6 @@ RUN \ dpkg --ctrl-tarfile $deb | tar -Ox ./control > /install-${arch}/var/lib/dpkg/status.d/${package_name}; \ dpkg --extract $deb /install-${arch}; \ done; \ - rsync -avr /install-${arch}/lib/ /install-${arch}/usr/lib; \ - rm -rf /install-${arch}/lib /install-${arch}/lib64; \ done @@ -183,7 +176,14 @@ LABEL org.opencontainers.image.documentation='https://github.com/element-hq/syna LABEL org.opencontainers.image.source='https://github.com/element-hq/synapse.git' LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later' -COPY --from=runtime-deps /install-${TARGETARCH} / +# On the runtime image, /lib is a symlink to /usr/lib, so we need to copy the +# libraries to the right place, else the `COPY` won't work. +# On amd64, we'll also have a /lib64 folder with ld-linux-x86-64.so.2, which is +# already present in the runtime image. +COPY --from=runtime-deps /install-${TARGETARCH}/lib /usr/lib +COPY --from=runtime-deps /install-${TARGETARCH}/etc /etc +COPY --from=runtime-deps /install-${TARGETARCH}/usr /usr +COPY --from=runtime-deps /install-${TARGETARCH}/var /var COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py COPY ./docker/conf /conf From 90f346183a267723c750f3f9bfe0d492cefc1cea Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 1 Apr 2025 08:32:56 -0400 Subject: [PATCH 230/691] Use uv pip to install supervisor in workers image (#18275) --- changelog.d/18275.docker | 1 + docker/Dockerfile-workers | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18275.docker diff --git a/changelog.d/18275.docker b/changelog.d/18275.docker new file mode 100644 index 0000000000..154226e5f6 --- /dev/null +++ b/changelog.d/18275.docker @@ -0,0 +1 @@ +Use uv pip to install supervisor in the worker image. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 3dec4bba05..dd0bf59994 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -27,10 +27,15 @@ FROM docker.io/library/redis:7-${DEBIAN_VERSION} AS redis_base # now build the final image, based on the the regular Synapse docker image FROM $FROM - # Install supervisord with pip instead of apt, to avoid installing a second + # Install supervisord with uv pip instead of apt, to avoid installing a second # copy of python. - RUN --mount=type=cache,target=/root/.cache/pip \ - pip install supervisor~=4.2 + # --link-mode=copy silences a warning as uv isn't able to do hardlinks between its cache + # (mounted as --mount=type=cache) and the target directory. + RUN \ + --mount=type=bind,from=ghcr.io/astral-sh/uv:0.6.8,source=/uv,target=/uv \ + --mount=type=cache,target=/root/.cache/uv \ + /uv pip install --link-mode=copy --prefix="/usr/local" supervisor~=4.2 + RUN mkdir -p /etc/supervisor/conf.d # Copy over redis and nginx From 2f812c2eb61c147b3ca8db47ed56725a5333a74e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 12:42:01 +0000 Subject: [PATCH 231/691] Bump jinja2 from 3.1.5 to 3.1.6 (#18223) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.5 to 3.1.6.
Release notes

Sourced from jinja2's releases.

3.1.6

This is the Jinja 3.1.6 security release, which fixes security issues but does not otherwise change behavior and should not result in breaking changes compared to the latest feature release.

PyPI: https://pypi.org/project/Jinja2/3.1.6/ Changes: https://jinja.palletsprojects.com/en/stable/changes/#version-3-1-6

Changelog

Sourced from jinja2's changelog.

Version 3.1.6

Released 2025-03-05

  • The |attr filter does not bypass the environment's attribute lookup, allowing the sandbox to apply its checks. :ghsa:cpwx-vrp4-4pq7
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=jinja2&package-manager=pip&previous-version=3.1.5&new-version=3.1.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index cfbfdc427a..236ff892ad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -943,14 +943,14 @@ trio = ["async_generator ; python_version == \"3.6\"", "trio"] [[package]] name = "jinja2" -version = "3.1.5" +version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" groups = ["main", "dev"] files = [ - {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, - {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, ] [package.dependencies] From 7ace290f07c7e5a7ec6c6c7de4447ead62d55b7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:13:58 +0000 Subject: [PATCH 232/691] Bump actions/add-to-project from f5473ace9aeee8b97717b281e26980aa5097023f to 280af8ae1f83a494cfad2cb10f02f6d13529caa9 (#18303) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/add-to-project](https://github.com/actions/add-to-project) from f5473ace9aeee8b97717b281e26980aa5097023f to 280af8ae1f83a494cfad2cb10f02f6d13529caa9.
Commits
  • 280af8a Merge pull request #688 from actions/dependabot/npm_and_yarn/vercel/ncc-0.38.3
  • a5abfeb Update licensed cache and dist/ directory
  • f30c2e6 Bump @​vercel/ncc from 0.38.1 to 0.38.3
  • 81dd5ce Merge pull request #687 from actions/dependabot/npm_and_yarn/types/jest-29.5.14
  • 122a803 Bump @​types/jest from 29.5.12 to 29.5.14
  • 29c72ac Merge pull request #686 from actions/dependabot/npm_and_yarn/types/node-22.13.14
  • 46316d9 Bump @​types/node from 16.18.101 to 22.13.14
  • 95df5ae Merge pull request #685 from actions/dependabot/npm_and_yarn/eslint-plugin-je...
  • f14f229 Bump eslint-plugin-jest from 28.6.0 to 28.11.0
  • cc69618 Exit without failure if nothing to commit
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/triage_labelled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index c08c674c88..feab5906e0 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -11,7 +11,7 @@ jobs: if: > contains(github.event.issue.labels.*.name, 'X-Needs-Info') steps: - - uses: actions/add-to-project@f5473ace9aeee8b97717b281e26980aa5097023f # main (v1.0.2 + 10 commits) + - uses: actions/add-to-project@280af8ae1f83a494cfad2cb10f02f6d13529caa9 # main (v1.0.2 + 10 commits) id: add_project with: project-url: "https://github.com/orgs/matrix-org/projects/67" From 80b62d7903eddf5abbb92afb849ed12d49269334 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:30:44 +0000 Subject: [PATCH 233/691] Bump actions/upload-artifact from 4.6.1 to 4.6.2 (#18304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.6.1 to 4.6.2.
Release notes

Sourced from actions/upload-artifact's releases.

v4.6.2

What's Changed

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v4.6.2

Commits
  • ea165f8 Merge pull request #685 from salmanmkc/salmanmkc/3-new-upload-artifacts-release
  • 0839620 Prepare for new release of actions/upload-artifact with new toolkit cache ver...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=4.6.1&new-version=4.6.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/release-artifacts.yml | 6 +++--- .github/workflows/tests.yml | 4 ++-- .github/workflows/twisted_trunk.yml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 4a865930ac..616ef0f9cf 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -39,7 +39,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: book path: book diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index e37e0c8598..e7378ec0d3 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -164,7 +164,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 9fd38e49a6..573264229f 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -101,7 +101,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -165,7 +165,7 @@ jobs: CARGO_NET_GIT_FETCH_WITH_CLI: true CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: Wheel-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl @@ -186,7 +186,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: Sdist path: dist/*.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9df1ec506b..bb2e80a908 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -529,7 +529,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -627,7 +627,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index aac1e350a2..0176f17401 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -138,7 +138,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) From 1709234311d2395d4dd432b997db387bade1c677 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 1 Apr 2025 13:31:19 +0000 Subject: [PATCH 234/691] Add an access token introspection cache to make Matrix Authentication Service integration (MSC3861) more efficient. (#18231) Evolution of https://github.com/element-hq/synapse/commit/cd78f3d2ee15ccf3e8229a1f529e0e2c16e15c45 This cache does not have any explicit invalidation, but this is deemed acceptable (see code comment). We may still prefer to add it eventually, letting us bump up the Time-To-Live (TTL) on the cache as we currently set a 2 minute expiry to balance the fact that we have no explicit invalidation. This cache makes several things more efficient: - reduces number of outbound requests from Synapse, reducing CPU utilisation + network I/O - reduces request handling time in Synapse, which improves client-visible latency - reduces load on MAS and its database --- Other than that, this PR also introduces support for `expires_in` (seconds) on the introspection response. This lets the cached responses expire at the proper expiry time of the access token, whilst avoiding clock skew issues. Corresponds to: https://github.com/element-hq/matrix-authentication-service/pull/4241 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18231.feature | 1 + synapse/api/auth/msc3861_delegated.py | 113 ++++++++++++++++++++---- tests/handlers/test_oauth_delegation.py | 38 ++++++++ 3 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 changelog.d/18231.feature diff --git a/changelog.d/18231.feature b/changelog.d/18231.feature new file mode 100644 index 0000000000..7fa65e4fa6 --- /dev/null +++ b/changelog.d/18231.feature @@ -0,0 +1 @@ +Add an access token introspection cache to make Matrix Authentication Service integration (MSC3861) more efficient. \ No newline at end of file diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index e6bf271a1f..74e526123f 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -19,6 +19,7 @@ # # import logging +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from urllib.parse import urlencode @@ -47,6 +48,7 @@ from synapse.logging.context import make_deferred_yieldable from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall +from synapse.util.caches.response_cache import ResponseCache if TYPE_CHECKING: from synapse.rest.admin.experimental_features import ExperimentalFeature @@ -76,6 +78,61 @@ def scope_to_list(scope: str) -> List[str]: return scope.strip().split(" ") +@dataclass +class IntrospectionResult: + _inner: IntrospectionToken + + # when we retrieved this token, + # in milliseconds since the Unix epoch + retrieved_at_ms: int + + def is_active(self, now_ms: int) -> bool: + if not self._inner.get("active"): + return False + + expires_in = self._inner.get("expires_in") + if expires_in is None: + return True + if not isinstance(expires_in, int): + raise InvalidClientTokenError("token `expires_in` is not an int") + + absolute_expiry_ms = expires_in * 1000 + self.retrieved_at_ms + return now_ms < absolute_expiry_ms + + def get_scope_list(self) -> List[str]: + value = self._inner.get("scope") + if not isinstance(value, str): + return [] + return scope_to_list(value) + + def get_sub(self) -> Optional[str]: + value = self._inner.get("sub") + if not isinstance(value, str): + return None + return value + + def get_username(self) -> Optional[str]: + value = self._inner.get("username") + if not isinstance(value, str): + return None + return value + + def get_name(self) -> Optional[str]: + value = self._inner.get("name") + if not isinstance(value, str): + return None + return value + + def get_device_id(self) -> Optional[str]: + value = self._inner.get("device_id") + if value is not None and not isinstance(value, str): + raise AuthError( + 500, + "Invalid device ID in introspection result", + ) + return value + + class PrivateKeyJWTWithKid(PrivateKeyJWT): # type: ignore[misc] """An implementation of the private_key_jwt client auth method that includes a kid header. @@ -121,6 +178,31 @@ class MSC3861DelegatedAuth(BaseAuth): self._hostname = hs.hostname self._admin_token: Callable[[], Optional[str]] = self._config.admin_token + # # Token Introspection Cache + # This remembers what users/devices are represented by which access tokens, + # in order to reduce overall system load: + # - on Synapse (as requests are relatively expensive) + # - on the network + # - on MAS + # + # Since there is no invalidation mechanism currently, + # the entries expire after 2 minutes. + # This does mean tokens can be treated as valid by Synapse + # for longer than reality. + # + # Ideally, tokens should logically be invalidated in the following circumstances: + # - If a session logout happens. + # In this case, MAS will delete the device within Synapse + # anyway and this is good enough as an invalidation. + # - If the client refreshes their token in MAS. + # In this case, the device still exists and it's not the end of the world for + # the old access token to continue working for a short time. + self._introspection_cache: ResponseCache[str] = ResponseCache( + self._clock, + "token_introspection", + timeout_ms=120_000, + ) + self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( self._load_metadata ) @@ -193,7 +275,7 @@ class MSC3861DelegatedAuth(BaseAuth): metadata = await self._issuer_metadata.get() return metadata.get("introspection_endpoint") - async def _introspect_token(self, token: str) -> IntrospectionToken: + async def _introspect_token(self, token: str) -> IntrospectionResult: """ Send a token to the introspection endpoint and returns the introspection response @@ -266,7 +348,9 @@ class MSC3861DelegatedAuth(BaseAuth): "The introspection endpoint returned an invalid JSON response." ) - return IntrospectionToken(**resp) + return IntrospectionResult( + IntrospectionToken(**resp), retrieved_at_ms=self._clock.time_msec() + ) async def is_server_admin(self, requester: Requester) -> bool: return "urn:synapse:admin:*" in requester.scope @@ -344,7 +428,9 @@ class MSC3861DelegatedAuth(BaseAuth): ) try: - introspection_result = await self._introspect_token(token) + introspection_result = await self._introspection_cache.wrap( + token, self._introspect_token, token + ) except Exception: logger.exception("Failed to introspect token") raise SynapseError(503, "Unable to introspect the access token") @@ -353,11 +439,11 @@ class MSC3861DelegatedAuth(BaseAuth): # TODO: introspection verification should be more extensive, especially: # - verify the audience - if not introspection_result.get("active"): + if not introspection_result.is_active(self._clock.time_msec()): raise InvalidClientTokenError("Token is not active") # Let's look at the scope - scope: List[str] = scope_to_list(introspection_result.get("scope", "")) + scope: List[str] = introspection_result.get_scope_list() # Determine type of user based on presence of particular scopes has_user_scope = SCOPE_MATRIX_API in scope @@ -367,7 +453,7 @@ class MSC3861DelegatedAuth(BaseAuth): raise InvalidClientTokenError("No scope in token granting user rights") # Match via the sub claim - sub: Optional[str] = introspection_result.get("sub") + sub: Optional[str] = introspection_result.get_sub() if sub is None: raise InvalidClientTokenError( "Invalid sub claim in the introspection result" @@ -381,7 +467,7 @@ class MSC3861DelegatedAuth(BaseAuth): # or the external_id was never recorded # TODO: claim mapping should be configurable - username: Optional[str] = introspection_result.get("username") + username: Optional[str] = introspection_result.get_username() if username is None or not isinstance(username, str): raise AuthError( 500, @@ -399,7 +485,7 @@ class MSC3861DelegatedAuth(BaseAuth): # TODO: claim mapping should be configurable # If present, use the name claim as the displayname - name: Optional[str] = introspection_result.get("name") + name: Optional[str] = introspection_result.get_name() await self.store.register_user( user_id=user_id.to_string(), create_profile_with_displayname=name @@ -414,15 +500,8 @@ class MSC3861DelegatedAuth(BaseAuth): # MAS 0.15+ will give us the device ID as an explicit value for compatibility sessions # If present, we get it from here, if not we get it in thee scope - device_id = introspection_result.get("device_id") - if device_id is not None: - # We got the device ID explicitly, just sanity check that it's a string - if not isinstance(device_id, str): - raise AuthError( - 500, - "Invalid device ID in introspection result", - ) - else: + device_id = introspection_result.get_device_id() + if device_id is None: # Find device_ids in scope # We only allow a single device_id in the scope, so we find them all in the # scope list, and raise if there are more than one. The OIDC server should be diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 5f8c25557a..034a1594d9 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -539,6 +539,44 @@ class MSC3861OAuthDelegation(HomeserverTestCase): error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) self.assertEqual(error.value.code, 503) + def test_cached_expired_introspection(self) -> None: + """The handler should raise an error if the introspection response gives + an expiry time, the introspection response is cached and then the entry is + re-requested after it has expired.""" + + self.http_client.request = introspection_mock = AsyncMock( + return_value=FakeResponse.json( + code=200, + payload={ + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [ + MATRIX_USER_SCOPE, + f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", + ] + ), + "username": USERNAME, + "expires_in": 60, + }, + ) + ) + request = Mock(args={}) + request.args[b"access_token"] = [b"mockAccessToken"] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + + # The first CS-API request causes a successful introspection + self.get_success(self.auth.get_user_by_req(request)) + self.assertEqual(introspection_mock.call_count, 1) + + # Sleep for 60 seconds so the token expires. + self.reactor.advance(60.0) + + # Now the CS-API request fails because the token expired + self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) + # Ensure another introspection request was not sent + self.assertEqual(introspection_mock.call_count, 1) + def make_device_keys(self, user_id: str, device_id: str) -> JsonDict: # We only generate a master key to simplify the test. master_signing_key = generate_signing_key(device_id) From 87d374c639f82bd4bd398e0b0345f2c7b73031aa Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 1 Apr 2025 09:36:13 -0400 Subject: [PATCH 235/691] Tweaks to prefix-log (#18274) - Explicitly use `mawk` instead of `awk`, since an extension of the former is used - Use `fflush` to reduce interleaving the output of different processes & streams - Move the `mawk` command to a shell function, instead of writing it twice - Look up the `SUPERVISOR_PROCESS_NAME` environment variable in `mawk`, instead of reading it in the shell & using complex quoting to pass it to `mawk` ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18274.docker | 1 + docker/prefix-log | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18274.docker diff --git a/changelog.d/18274.docker b/changelog.d/18274.docker new file mode 100644 index 0000000000..57b81d8f4e --- /dev/null +++ b/changelog.d/18274.docker @@ -0,0 +1 @@ +Make some improvements to the prefix-log script in the workers image. diff --git a/docker/prefix-log b/docker/prefix-log index 32dddbbfd4..2a38de5686 100755 --- a/docker/prefix-log +++ b/docker/prefix-log @@ -10,6 +10,9 @@ # '-W interactive' is a `mawk` extension which disables buffering on stdout and sets line-buffered reads on # stdin. The effect is that the output is flushed after each line, rather than being batched, which helps reduce # confusion due to to interleaving of the different processes. -exec 1> >(awk -W interactive '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0 }' >&1) -exec 2> >(awk -W interactive '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0 }' >&2) +prefixer() { + mawk -W interactive '{printf("%s | %s\n", ENVIRON["SUPERVISOR_PROCESS_NAME"], $0); fflush() }' +} +exec 1> >(prefixer) +exec 2> >(prefixer >&2) exec "$@" From 8eb991b746731d72891cf28ad298d527d28c724e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:36:25 +0200 Subject: [PATCH 236/691] Bump authlib from 1.4.1 to 1.5.1 (#18306) Bumps [authlib](https://github.com/lepture/authlib) from 1.4.1 to 1.5.1.
Release notes

Sourced from authlib's releases.

Version 1.5.1

Released on Feb 28, 2025

  • Fix RFC9207 iss parameter. #715

Version 1.5.0

  • Fix token introspection auth method for clients. #662
  • Optional typ claim in JWT tokens. #696
  • JWT validation leeway. #689
  • Implement server-side RFC9207. #700 #701
  • generate_id_token can take a kid parameter. #702
  • More detailed InvalidClientError. #706
  • OpenID Connect Dynamic Client Registration implementation. #707
Changelog

Sourced from authlib's changelog.

Version 1.5.1

Released on Feb 28, 2025

  • Fix RFC9207 iss parameter. :pr:715

Version 1.5.0

Released on Feb 25, 2025

  • Fix token introspection auth method for clients. :pr:662
  • Optional typ claim in JWT tokens. :pr:696
  • JWT validation leeway. :pr:689
  • Implement server-side :rfc:RFC9207 <9207>. :issue:700 :pr:701
  • generate_id_token can take a kid parameter. :pr:702
  • More detailed InvalidClientError. :pr:706
  • OpenID Connect Dynamic Client Registration implementation. :pr:707
Commits
  • 4eafdc2 chore: release 1.5.1
  • 0e7e344 Merge pull request #715 from azmeuk/rfc9207
  • b57932b fix: RFC9207 iss parameter
  • 7833a88 Merge pull request #713 from geigerzaehler/full-entropy
  • 642dfa3 doc: fix an example import for rfc9207
  • 5c507a8 fix: Use full entropy from specified oct key size
  • 2d0396e chore: release 1.5.0
  • da87c8b doc: update changelog
  • b79d868 Merge pull request #662 from AdamWill/oauth2-fix-introspect-endpoint
  • 24c2bd8 chore: add a dependency group for the documentation
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=authlib&package-manager=pip&previous-version=1.4.1&new-version=1.5.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 236ff892ad..7bff50e153 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,15 +34,15 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a [[package]] name = "authlib" -version = "1.4.1" +version = "1.5.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "Authlib-1.4.1-py2.py3-none-any.whl", hash = "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"}, - {file = "authlib-1.4.1.tar.gz", hash = "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d"}, + {file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"}, + {file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"}, ] [package.dependencies] From fdbcb821ff7c11b091e60563d57dc5a4ad9a0372 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:56:32 +0000 Subject: [PATCH 237/691] Bump phonenumbers from 8.13.50 to 9.0.2 (#18299) Bumps [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) from 8.13.50 to 9.0.2.
Commits
  • 73ef5e6 Prep for 9.0.2 release
  • 528a98b Generated files for metadata
  • 28f5958 Merge metadata changes from upstream 9.0.2
  • 25ae49c Prep for 9.0.1 release
  • b8a1459 Generated files for metadata
  • f6cd233 Merge metadata changes from upstream 9.0.1
  • c46f104 Prep for 9.0.0 release
  • d542ec2 Generated files for metadata
  • a4da80e Merge metadata changes from upstream 9.0.0
  • 45c822e Prep for 8.13.55 release
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=phonenumbers&package-manager=pip&previous-version=8.13.50&new-version=9.0.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7bff50e153..2bf511e8a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1588,14 +1588,14 @@ dev = ["jinja2"] [[package]] name = "phonenumbers" -version = "8.13.50" +version = "9.0.2" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-8.13.50-py2.py3-none-any.whl", hash = "sha256:bb95dbc0d9979c51f7ad94bcd780784938958861fbb4b75a2fe39ccd3d58954a"}, - {file = "phonenumbers-8.13.50.tar.gz", hash = "sha256:e05ac6fb7b98c6d719a87ea895b9fc153673b4a51f455ec9afaf557ef4629da6"}, + {file = "phonenumbers-9.0.2-py2.py3-none-any.whl", hash = "sha256:dbcec6bdfdf3973f60b81dc0fcac3f7b1638f877ac42da4d7b46724ed413e2b9"}, + {file = "phonenumbers-9.0.2.tar.gz", hash = "sha256:f590ee2b729bdd9873ca2d52989466add14c9953b48805c0aeb408348d4d6224"}, ] [[package]] From fa53a8512ac58419b87955e155a1b80aa6d0419a Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Apr 2025 16:21:35 +0200 Subject: [PATCH 238/691] Make sure media hashes are not queried until the index is up (#18302) --- changelog.d/18302.feature | 1 + synapse/storage/databases/main/media_repository.py | 9 +++++++++ .../storage/schema/main/delta/91/01_media_hash.sql | 13 ++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18302.feature diff --git a/changelog.d/18302.feature b/changelog.d/18302.feature new file mode 100644 index 0000000000..3604e732d5 --- /dev/null +++ b/changelog.d/18302.feature @@ -0,0 +1 @@ +Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 8a5ba44922..04866524e3 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -999,6 +999,15 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): None if the media_id doesn't exist. """ + # If we don't have the index yet, performance tanks, so we return False. + # In the background updates, remote_media_cache_sha256_idx is created + # after local_media_repository_sha256_idx, which is why we only need to + # check for the completion of the former. + if not await self.db_pool.updates.has_completed_background_update( + "remote_media_cache_sha256_idx" + ): + return False + def get_matching_media_txn( txn: LoggingTransaction, table: str, sha256: str ) -> bool: diff --git a/synapse/storage/schema/main/delta/91/01_media_hash.sql b/synapse/storage/schema/main/delta/91/01_media_hash.sql index 2dbd2c7df7..34a372f1ed 100644 --- a/synapse/storage/schema/main/delta/91/01_media_hash.sql +++ b/synapse/storage/schema/main/delta/91/01_media_hash.sql @@ -16,6 +16,13 @@ ALTER TABLE local_media_repository ADD COLUMN sha256 TEXT; ALTER TABLE remote_media_cache ADD COLUMN sha256 TEXT; -- Add a background updates to handle creating the new index. -INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (9101, 'local_media_repository_sha256_idx', '{}'), - (9101, 'remote_media_cache_sha256_idx', '{}'); \ No newline at end of file +-- +-- Note that the ordering of the update is not following the usual scheme. This +-- is because when upgrading from Synapse 1.127, this index is fairly important +-- to have up quickly, so that it doesn't tank performance, which is why it is +-- scheduled before other background updates in the 1.127 -> 1.128 upgrade +INSERT INTO + background_updates (ordering, update_name, progress_json) +VALUES + (8890, 'local_media_repository_sha256_idx', '{}'), + (8891, 'remote_media_cache_sha256_idx', '{}'); From 01efc49554dec58aa27fc29e538bde56f4ef2617 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Apr 2025 16:41:42 +0200 Subject: [PATCH 239/691] 1.128.0rc1 --- CHANGES.md | 52 +++++++++++++++++++++++++++++++++++++++ changelog.d/18068.misc | 1 - changelog.d/18074.bugfix | 1 - changelog.d/18225.doc | 1 - changelog.d/18231.feature | 1 - changelog.d/18251.misc | 1 - changelog.d/18254.feature | 1 - changelog.d/18255.misc | 1 - changelog.d/18271.docker | 1 - changelog.d/18272.docker | 1 - changelog.d/18273.docker | 1 - changelog.d/18274.docker | 1 - changelog.d/18275.docker | 1 - changelog.d/18276.doc | 1 - changelog.d/18277.feature | 1 - changelog.d/18283.doc | 1 - changelog.d/18284.misc | 1 - changelog.d/18287.docker | 1 - changelog.d/18296.doc | 1 - changelog.d/18302.feature | 1 - debian/changelog | 5 ++-- pyproject.toml | 2 +- 22 files changed, 56 insertions(+), 22 deletions(-) delete mode 100644 changelog.d/18068.misc delete mode 100644 changelog.d/18074.bugfix delete mode 100644 changelog.d/18225.doc delete mode 100644 changelog.d/18231.feature delete mode 100644 changelog.d/18251.misc delete mode 100644 changelog.d/18254.feature delete mode 100644 changelog.d/18255.misc delete mode 100644 changelog.d/18271.docker delete mode 100644 changelog.d/18272.docker delete mode 100644 changelog.d/18273.docker delete mode 100644 changelog.d/18274.docker delete mode 100644 changelog.d/18275.docker delete mode 100644 changelog.d/18276.doc delete mode 100644 changelog.d/18277.feature delete mode 100644 changelog.d/18283.doc delete mode 100644 changelog.d/18284.misc delete mode 100644 changelog.d/18287.docker delete mode 100644 changelog.d/18296.doc delete mode 100644 changelog.d/18302.feature diff --git a/CHANGES.md b/CHANGES.md index 0176c6e45d..e591deff67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,55 @@ +# Synapse 1.128.0rc1 (2025-04-01) + +### Features + +- Add an access token introspection cache to make Matrix Authentication Service integration ([MSC3861](https://github.com/matrix-org/matrix-doc/pull/3861)) more efficient. ([\#18231](https://github.com/element-hq/synapse/issues/18231)) +- Add background job to clear unreferenced state groups. ([\#18254](https://github.com/element-hq/synapse/issues/18254)) +- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18277](https://github.com/element-hq/synapse/issues/18277), [\#18302](https://github.com/element-hq/synapse/issues/18302)) + +### Bugfixes + +- Add index to sliding sync ([MSC4186](https://github.com/matrix-org/matrix-doc/pull/4186)) membership snapshot table, to fix a performance issue. ([\#18074](https://github.com/element-hq/synapse/issues/18074)) + +### Updates to the Docker image + +- Specify the architecture of installed packages via an APT config option, which is more reliable than appending package names with `:{arch}`. ([\#18271](https://github.com/element-hq/synapse/issues/18271)) +- Always specify base image debian versions with a build argument. ([\#18272](https://github.com/element-hq/synapse/issues/18272)) +- Allow passing arguments to `start_for_complement.sh (to be sent to `configure_workers_and_start.py`). ([\#18273](https://github.com/element-hq/synapse/issues/18273)) +- Make some improvements to the `prefix-log` script in the workers image. ([\#18274](https://github.com/element-hq/synapse/issues/18274)) +- Use `uv pip` to install `supervisor` in the worker image. ([\#18275](https://github.com/element-hq/synapse/issues/18275)) +- Avoid needing to download & use `rsync` in a build layer. ([\#18287](https://github.com/element-hq/synapse/issues/18287)) + +### Improved Documentation + +- Fix how to obtain access token and change naming from riot to element ([\#18225](https://github.com/element-hq/synapse/issues/18225)) +- Correct a small typo in the SSO mapping providers documentation. ([\#18276](https://github.com/element-hq/synapse/issues/18276)) +- Add docs for how to clear out the Poetry wheel cache. ([\#18283](https://github.com/element-hq/synapse/issues/18283)) +- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18296](https://github.com/element-hq/synapse/issues/18296)) + +### Internal Changes + +- Add a column `participant` to `room_memberships` table. ([\#18068](https://github.com/element-hq/synapse/issues/18068)) +- Update Poetry to 2.1.1, including updating the lock file version. ([\#18251](https://github.com/element-hq/synapse/issues/18251)) +- Pin GitHub Actions dependencies by commit hash. ([\#18255](https://github.com/element-hq/synapse/issues/18255)) +- Add DB delta to remove the old state group deletion job. ([\#18284](https://github.com/element-hq/synapse/issues/18284)) + + + +### Updates to locked dependencies + +* Bump actions/add-to-project from f5473ace9aeee8b97717b281e26980aa5097023f to 280af8ae1f83a494cfad2cb10f02f6d13529caa9. ([\#18303](https://github.com/element-hq/synapse/issues/18303)) +* Bump actions/cache from 4.2.2 to 4.2.3. ([\#18266](https://github.com/element-hq/synapse/issues/18266)) +* Bump actions/download-artifact from 4.2.0 to 4.2.1. ([\#18268](https://github.com/element-hq/synapse/issues/18268)) +* Bump actions/setup-python from 5.4.0 to 5.5.0. ([\#18298](https://github.com/element-hq/synapse/issues/18298)) +* Bump actions/upload-artifact from 4.6.1 to 4.6.2. ([\#18304](https://github.com/element-hq/synapse/issues/18304)) +* Bump authlib from 1.4.1 to 1.5.1. ([\#18306](https://github.com/element-hq/synapse/issues/18306)) +* Bump dawidd6/action-download-artifact from 8 to 9. ([\#18204](https://github.com/element-hq/synapse/issues/18204)) +* Bump jinja2 from 3.1.5 to 3.1.6. ([\#18223](https://github.com/element-hq/synapse/issues/18223)) +* Bump log from 0.4.26 to 0.4.27. ([\#18267](https://github.com/element-hq/synapse/issues/18267)) +* Bump phonenumbers from 8.13.50 to 9.0.2. ([\#18299](https://github.com/element-hq/synapse/issues/18299)) +* Bump pygithub from 2.5.0 to 2.6.1. ([\#18243](https://github.com/element-hq/synapse/issues/18243)) +* Bump pyo3-log from 0.12.1 to 0.12.2. ([\#18269](https://github.com/element-hq/synapse/issues/18269)) + # Synapse 1.127.1 (2025-03-26) ## Security diff --git a/changelog.d/18068.misc b/changelog.d/18068.misc deleted file mode 100644 index af6f78f549..0000000000 --- a/changelog.d/18068.misc +++ /dev/null @@ -1 +0,0 @@ -Add a column `participant` to `room_memberships` table. \ No newline at end of file diff --git a/changelog.d/18074.bugfix b/changelog.d/18074.bugfix deleted file mode 100644 index 03e0b95988..0000000000 --- a/changelog.d/18074.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add index to sliding sync membership snapshot table, to fix a performance issue. \ No newline at end of file diff --git a/changelog.d/18225.doc b/changelog.d/18225.doc deleted file mode 100644 index 854e275120..0000000000 --- a/changelog.d/18225.doc +++ /dev/null @@ -1 +0,0 @@ -Fix how to obtain access token and change naming from riot to element diff --git a/changelog.d/18231.feature b/changelog.d/18231.feature deleted file mode 100644 index 7fa65e4fa6..0000000000 --- a/changelog.d/18231.feature +++ /dev/null @@ -1 +0,0 @@ -Add an access token introspection cache to make Matrix Authentication Service integration (MSC3861) more efficient. \ No newline at end of file diff --git a/changelog.d/18251.misc b/changelog.d/18251.misc deleted file mode 100644 index 3b557a06c4..0000000000 --- a/changelog.d/18251.misc +++ /dev/null @@ -1 +0,0 @@ -Update Poetry to 2.1.1, including updating the lock file version. \ No newline at end of file diff --git a/changelog.d/18254.feature b/changelog.d/18254.feature deleted file mode 100644 index 62e1b79a15..0000000000 --- a/changelog.d/18254.feature +++ /dev/null @@ -1 +0,0 @@ -Add background job to clear unreferenced state groups. diff --git a/changelog.d/18255.misc b/changelog.d/18255.misc deleted file mode 100644 index 49a5b0e7b6..0000000000 --- a/changelog.d/18255.misc +++ /dev/null @@ -1 +0,0 @@ -Pin GitHub Actions dependencies by commit hash. \ No newline at end of file diff --git a/changelog.d/18271.docker b/changelog.d/18271.docker deleted file mode 100644 index fcb2ac1e00..0000000000 --- a/changelog.d/18271.docker +++ /dev/null @@ -1 +0,0 @@ -Specify the architecture of installed packages via an APT config option, which is more reliable than appending package names with ":{arch}". diff --git a/changelog.d/18272.docker b/changelog.d/18272.docker deleted file mode 100644 index ceec619b6c..0000000000 --- a/changelog.d/18272.docker +++ /dev/null @@ -1 +0,0 @@ -Always specify base image debian versions with a build argument. diff --git a/changelog.d/18273.docker b/changelog.d/18273.docker deleted file mode 100644 index 1da0c3efb1..0000000000 --- a/changelog.d/18273.docker +++ /dev/null @@ -1 +0,0 @@ -Allow passing arguments to start_for_complement.sh (to be sent to configure_workers_and_start.py). diff --git a/changelog.d/18274.docker b/changelog.d/18274.docker deleted file mode 100644 index 57b81d8f4e..0000000000 --- a/changelog.d/18274.docker +++ /dev/null @@ -1 +0,0 @@ -Make some improvements to the prefix-log script in the workers image. diff --git a/changelog.d/18275.docker b/changelog.d/18275.docker deleted file mode 100644 index 154226e5f6..0000000000 --- a/changelog.d/18275.docker +++ /dev/null @@ -1 +0,0 @@ -Use uv pip to install supervisor in the worker image. diff --git a/changelog.d/18276.doc b/changelog.d/18276.doc deleted file mode 100644 index cacf6daf76..0000000000 --- a/changelog.d/18276.doc +++ /dev/null @@ -1 +0,0 @@ -Correct a small typo in the SSO mapping providers documentation. \ No newline at end of file diff --git a/changelog.d/18277.feature b/changelog.d/18277.feature deleted file mode 100644 index 3604e732d5..0000000000 --- a/changelog.d/18277.feature +++ /dev/null @@ -1 +0,0 @@ -Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/changelog.d/18283.doc b/changelog.d/18283.doc deleted file mode 100644 index 77fa08fa6b..0000000000 --- a/changelog.d/18283.doc +++ /dev/null @@ -1 +0,0 @@ -Add docs for how to clear out the Poetry wheel cache. diff --git a/changelog.d/18284.misc b/changelog.d/18284.misc deleted file mode 100644 index 69610adc65..0000000000 --- a/changelog.d/18284.misc +++ /dev/null @@ -1 +0,0 @@ -Add DB delta to remove the old state group deletion job. diff --git a/changelog.d/18287.docker b/changelog.d/18287.docker deleted file mode 100644 index ef45ad72ba..0000000000 --- a/changelog.d/18287.docker +++ /dev/null @@ -1 +0,0 @@ -Avoid needing to download & use rsync in a build layer. diff --git a/changelog.d/18296.doc b/changelog.d/18296.doc deleted file mode 100644 index 3604e732d5..0000000000 --- a/changelog.d/18296.doc +++ /dev/null @@ -1 +0,0 @@ -Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/changelog.d/18302.feature b/changelog.d/18302.feature deleted file mode 100644 index 3604e732d5..0000000000 --- a/changelog.d/18302.feature +++ /dev/null @@ -1 +0,0 @@ -Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 645001ca52..6e338ecd74 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -matrix-synapse-py3 (1.128.0~rc1+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.128.0~rc1) stable; urgency=medium * Update Poetry to 2.1.1. + * New synapse release 1.128.0rc1. - -- Synapse Packaging team Wed, 19 Mar 2025 17:38:49 +0000 + -- Synapse Packaging team Tue, 01 Apr 2025 14:35:33 +0000 matrix-synapse-py3 (1.127.1) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index e91a75445c..da469aab2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.127.1" +version = "1.128.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From b8b3896b1d6ce26525e5f4e8676914ff1f38ff88 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Apr 2025 16:45:11 +0200 Subject: [PATCH 240/691] Fix rendering of the changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e591deff67..c8d879dc3a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ - Specify the architecture of installed packages via an APT config option, which is more reliable than appending package names with `:{arch}`. ([\#18271](https://github.com/element-hq/synapse/issues/18271)) - Always specify base image debian versions with a build argument. ([\#18272](https://github.com/element-hq/synapse/issues/18272)) -- Allow passing arguments to `start_for_complement.sh (to be sent to `configure_workers_and_start.py`). ([\#18273](https://github.com/element-hq/synapse/issues/18273)) +- Allow passing arguments to `start_for_complement.sh` (to be sent to `configure_workers_and_start.py`). ([\#18273](https://github.com/element-hq/synapse/issues/18273)) - Make some improvements to the `prefix-log` script in the workers image. ([\#18274](https://github.com/element-hq/synapse/issues/18274)) - Use `uv pip` to install `supervisor` in the worker image. ([\#18275](https://github.com/element-hq/synapse/issues/18275)) - Avoid needing to download & use `rsync` in a build layer. ([\#18287](https://github.com/element-hq/synapse/issues/18287)) From 770768614b0743d03258000934a159c69026bcdc Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Apr 2025 16:49:19 +0200 Subject: [PATCH 241/691] Merge changelog entries --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c8d879dc3a..35c3f22aaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ - Add an access token introspection cache to make Matrix Authentication Service integration ([MSC3861](https://github.com/matrix-org/matrix-doc/pull/3861)) more efficient. ([\#18231](https://github.com/element-hq/synapse/issues/18231)) - Add background job to clear unreferenced state groups. ([\#18254](https://github.com/element-hq/synapse/issues/18254)) -- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18277](https://github.com/element-hq/synapse/issues/18277), [\#18302](https://github.com/element-hq/synapse/issues/18302)) +- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18277](https://github.com/element-hq/synapse/issues/18277), [\#18302](https://github.com/element-hq/synapse/issues/18302), [\#18296](https://github.com/element-hq/synapse/issues/18296)) ### Bugfixes @@ -24,7 +24,6 @@ - Fix how to obtain access token and change naming from riot to element ([\#18225](https://github.com/element-hq/synapse/issues/18225)) - Correct a small typo in the SSO mapping providers documentation. ([\#18276](https://github.com/element-hq/synapse/issues/18276)) - Add docs for how to clear out the Poetry wheel cache. ([\#18283](https://github.com/element-hq/synapse/issues/18283)) -- Hashes of media files are now tracked by Synapse. Media quarantines will now apply to all files with the same hash. ([\#18296](https://github.com/element-hq/synapse/issues/18296)) ### Internal Changes From 5c84f258095535aaa2a4a04c850f439fd00735cc Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 1 Apr 2025 11:51:00 -0400 Subject: [PATCH 242/691] complement-synapse: COPY existing dir from base (#18294) The base postgres image already has the /var/run/postgresql directory, and COPY can set file ownership with chown=, so COPY it instead of making it from scratch & manually setting its ownership. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18294.docker | 1 + docker/complement/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18294.docker diff --git a/changelog.d/18294.docker b/changelog.d/18294.docker new file mode 100644 index 0000000000..cc40ca90c0 --- /dev/null +++ b/changelog.d/18294.docker @@ -0,0 +1 @@ +Optimize the build of the complement-synapse image. diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index 3e7f808cc5..dd029c5fbc 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -25,7 +25,7 @@ FROM $FROM RUN adduser --system --uid 999 postgres --home /var/lib/postgresql COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql -RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql +COPY --from=postgres_base --chown=postgres /var/run/postgresql /var/run/postgresql ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" ENV PGDATA=/var/lib/postgresql/data From 0e3c0aeee833e52121b3167de486dff34018ab27 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 2 Apr 2025 09:37:50 -0500 Subject: [PATCH 243/691] Disable Postgres statement timeouts while purging room state (#18133) --- changelog.d/18133.misc | 1 + synapse/storage/databases/state/store.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/18133.misc diff --git a/changelog.d/18133.misc b/changelog.d/18133.misc new file mode 100644 index 0000000000..151ceb2cab --- /dev/null +++ b/changelog.d/18133.misc @@ -0,0 +1 @@ +Disable statement timeout during room purge. diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 90d7beb92f..c1a66dcba0 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -48,6 +48,7 @@ from synapse.storage.database import ( LoggingTransaction, ) from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore +from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor from synapse.storage.util.sequence import build_sequence_generator from synapse.types import MutableStateMap, StateKey, StateMap @@ -914,6 +915,12 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): ) -> None: # Delete all edges that reference a state group linked to room_id logger.info("[purge] removing %s from state_group_edges", room_id) + + if isinstance(self.database_engine, PostgresEngine): + # Disable statement timeouts for this transaction; purging rooms can + # take a while! + txn.execute("SET LOCAL statement_timeout = 0") + txn.execute( """ DELETE FROM state_group_edges AS sge WHERE sge.state_group IN ( From b30fcb03cc976c444c84b5c7137df81439b7f60e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Apr 2025 14:09:59 +0100 Subject: [PATCH 244/691] 1.128.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 35c3f22aaa..4ac25a3786 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.128.0 (2025-04-08) + +No significant changes since 1.128.0rc1. + + + + # Synapse 1.128.0rc1 (2025-04-01) ### Features diff --git a/debian/changelog b/debian/changelog index 6e338ecd74..56839ac5b4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.128.0) stable; urgency=medium + + * New Synapse release 1.128.0. + + -- Synapse Packaging team Tue, 08 Apr 2025 14:09:54 +0100 + matrix-synapse-py3 (1.128.0~rc1) stable; urgency=medium * Update Poetry to 2.1.1. diff --git a/pyproject.toml b/pyproject.toml index da469aab2c..8f8eb46e68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.128.0rc1" +version = "1.128.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From dd05cc55eedbf086ae224a13c9ae9f0332d96b1f Mon Sep 17 00:00:00 2001 From: Olivier D Date: Thu, 10 Apr 2025 15:39:27 +0200 Subject: [PATCH 245/691] Add passthrough_authorization_parameters support to OIDC configuration (#18232) # Add passthrough_authorization_parameters support to OIDC configuration This PR adds `the passthrough_authorization_parameters` option to OIDC configuration, allowing specific query parameters (like `login_hint`) to be passed from the redirect endpoint to the authorization grant URL. This enables clients to provide additional context to identity providers during authentication flows. # Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18232.feature | 1 + .../configuration/config_documentation.md | 4 +++ synapse/config/oidc.py | 6 +++++ synapse/handlers/oidc.py | 12 ++++++++- tests/handlers/test_oidc.py | 26 +++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18232.feature diff --git a/changelog.d/18232.feature b/changelog.d/18232.feature new file mode 100644 index 0000000000..ba5059ba80 --- /dev/null +++ b/changelog.d/18232.feature @@ -0,0 +1 @@ +Add `passthrough_authorization_parameters` in OIDC configuration to allow to pass parameters to the authorization grant URL. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index d2d282f203..73fd9622ce 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3672,6 +3672,9 @@ Options for each entry include: * `additional_authorization_parameters`: String to string dictionary that will be passed as additional parameters to the authorization grant URL. +* `passthrough_authorization_parameters`: List of parameters that will be passed through from the redirect endpoint + to the authorization grant URL. + * `allow_existing_users`: set to true to allow a user logging in via OIDC to match a pre-existing account instead of failing. This could be used if switching from password logins to OIDC. Defaults to false. @@ -3798,6 +3801,7 @@ oidc_providers: jwks_uri: "https://accounts.example.com/.well-known/jwks.json" additional_authorization_parameters: acr_values: 2fa + passthrough_authorization_parameters: ["login_hint"] skip_verification: true enable_registration: true user_mapping_provider: diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py index 8ba0ba2c36..3ddf65a3e9 100644 --- a/synapse/config/oidc.py +++ b/synapse/config/oidc.py @@ -356,6 +356,9 @@ def _parse_oidc_config_dict( additional_authorization_parameters=oidc_config.get( "additional_authorization_parameters", {} ), + passthrough_authorization_parameters=oidc_config.get( + "passthrough_authorization_parameters", [] + ), ) @@ -501,3 +504,6 @@ class OidcProviderConfig: # Additional parameters that will be passed to the authorization grant URL additional_authorization_parameters: Mapping[str, str] + + # Allow query parameters to the redirect endpoint that will be passed to the authorization grant URL + passthrough_authorization_parameters: Collection[str] diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 18efdd9f6e..c4cf0636a3 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -467,6 +467,10 @@ class OidcProvider: self._sso_handler.register_identity_provider(self) + self.passthrough_authorization_parameters = ( + provider.passthrough_authorization_parameters + ) + def _validate_metadata(self, m: OpenIDProviderMetadata) -> None: """Verifies the provider metadata. @@ -1005,7 +1009,6 @@ class OidcProvider: when everything is done (or None for UI Auth) ui_auth_session_id: The session ID of the ongoing UI Auth (or None if this is a login). - Returns: The redirect URL to the authorization endpoint. @@ -1078,6 +1081,13 @@ class OidcProvider: ) ) + # add passthrough additional authorization parameters + passthrough_authorization_parameters = self.passthrough_authorization_parameters + for parameter in passthrough_authorization_parameters: + parameter_value = parse_string(request, parameter) + if parameter_value: + additional_authorization_parameters.update({parameter: parameter_value}) + authorization_endpoint = metadata.get("authorization_endpoint") return prepare_grant_uri( authorization_endpoint, diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index cfd9969563..a7cead83d0 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -484,6 +484,32 @@ class OidcHandlerTestCase(HomeserverTestCase): self.assertEqual(code_verifier, "") self.assertEqual(redirect, "http://client/redirect") + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "passthrough_authorization_parameters": ["additional_parameter"], + } + } + ) + def test_passthrough_parameters(self) -> None: + """The redirect request has additional parameters, one is authorized, one is not""" + req = Mock(spec=["cookies", "args"]) + req.cookies = [] + req.args = {} + req.args[b"additional_parameter"] = ["a_value".encode("utf-8")] + req.args[b"not_authorized_parameter"] = ["any".encode("utf-8")] + + url = urlparse( + self.get_success( + self.provider.handle_redirect_request(req, b"http://client/redirect") + ) + ) + + params = parse_qs(url.query) + self.assertEqual(params["additional_parameter"], ["a_value"]) + self.assertNotIn("not_authorized_parameters", params) + @override_config({"oidc_config": DEFAULT_CONFIG}) def test_redirect_request_with_code_challenge(self) -> None: """The redirect request has the right arguments & generates a valid session cookie.""" From ae701e17090324ea5182450f42fbc9cfa9352835 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 14 Apr 2025 17:54:47 +0100 Subject: [PATCH 246/691] Add caches to new hot path functions (#18337) We call these two functions for every authed request when using delegated auth. --- changelog.d/18337.misc | 1 + synapse/handlers/device.py | 2 ++ synapse/storage/databases/main/devices.py | 9 ++++++- .../storage/databases/main/registration.py | 26 +++++++++---------- 4 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 changelog.d/18337.misc diff --git a/changelog.d/18337.misc b/changelog.d/18337.misc new file mode 100644 index 0000000000..b78276fe76 --- /dev/null +++ b/changelog.d/18337.misc @@ -0,0 +1 @@ +Add cache to storage functions used to auth requests when using delegated auth. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index d9622080b4..1efd039f22 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -163,6 +163,8 @@ class DeviceWorkerHandler: raise errors.NotFoundError() ips = await self.store.get_last_client_ip_by_device(user_id, device_id) + + device = dict(device) _update_device_from_client_ips(device, ips) set_tag("device", str(device)) diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 0b6d1f2b05..3f0b2f5d84 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -282,9 +282,10 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) + @cached() async def get_device( self, user_id: str, device_id: str - ) -> Optional[Dict[str, Any]]: + ) -> Optional[Mapping[str, Any]]: """Retrieve a device. Only returns devices that are not marked as hidden. @@ -1817,6 +1818,8 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): }, desc="store_device", ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + if not inserted: # if the device already exists, check if it's a real device, or # if the device ID is reserved by something else @@ -1882,6 +1885,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): values=device_ids, keyvalues={"user_id": user_id}, ) + self._invalidate_cache_and_stream_bulk( + txn, self.get_device, [(user_id, device_id) for device_id in device_ids] + ) for batch in batch_iter(device_ids, 100): await self.db_pool.runInteraction( @@ -1915,6 +1921,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): updatevalues=updates, desc="update_device", ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) async def update_remote_device_list_cache_entry( self, user_id: str, device_id: str, content: JsonDict, stream_id: str diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 8380930c70..eadbf4901c 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -759,6 +759,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): external_id: id on that system user_id: complete mxid that it is mapped to """ + self._invalidate_cache_and_stream( + txn, self.get_user_by_external_id, (auth_provider, external_id) + ) self.db_pool.simple_insert_txn( txn, @@ -789,6 +792,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): }, desc="remove_user_external_id", ) + await self.invalidate_cache_and_stream( + "get_user_by_external_id", (auth_provider, external_id) + ) async def replace_user_external_id( self, @@ -809,29 +815,20 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): ExternalIDReuseException if the new external_id could not be mapped. """ - def _remove_user_external_ids_txn( + def _replace_user_external_id_txn( txn: LoggingTransaction, - user_id: str, ) -> None: - """Remove all mappings from external user ids to a mxid - If these mappings are not found, this method does nothing. - - Args: - user_id: complete mxid that it is mapped to - """ - self.db_pool.simple_delete_txn( txn, table="user_external_ids", keyvalues={"user_id": user_id}, ) - def _replace_user_external_id_txn( - txn: LoggingTransaction, - ) -> None: - _remove_user_external_ids_txn(txn, user_id) - for auth_provider, external_id in record_external_ids: + self._invalidate_cache_and_stream( + txn, self.get_user_by_external_id, (auth_provider, external_id) + ) + self._record_user_external_id_txn( txn, auth_provider, @@ -847,6 +844,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): except self.database_engine.module.IntegrityError: raise ExternalIDReuseException() + @cached() async def get_user_by_external_id( self, auth_provider: str, external_id: str ) -> Optional[str]: From a832375bfb5b0327e73d2b5cf9104b555308e827 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Apr 2025 07:49:08 -0700 Subject: [PATCH 247/691] Add total event, unencrypted message, and e2ee event counts to stats reporting (#18260) Co-authored-by: Eric Eastwood --- changelog.d/18260.feature | 1 + .../reporting_homeserver_usage_statistics.md | 9 +- synapse/app/phone_stats_home.py | 41 ++- .../databases/main/events_bg_updates.py | 290 +++++++++++++++++- synapse/storage/databases/main/metrics.py | 38 +++ synapse/storage/schema/__init__.py | 8 +- .../schema/main/delta/92/01_event_stats.sql | 33 ++ synapse/types/storage/__init__.py | 2 + tests/metrics/test_phone_home_stats.py | 258 ++++++++++++++++ tests/storage/test_event_stats.py | 237 ++++++++++++++ 10 files changed, 907 insertions(+), 10 deletions(-) create mode 100644 changelog.d/18260.feature create mode 100644 synapse/storage/schema/main/delta/92/01_event_stats.sql create mode 100644 tests/metrics/test_phone_home_stats.py create mode 100644 tests/storage/test_event_stats.py diff --git a/changelog.d/18260.feature b/changelog.d/18260.feature new file mode 100644 index 0000000000..e44e3dc990 --- /dev/null +++ b/changelog.d/18260.feature @@ -0,0 +1 @@ +Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. diff --git a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md index 4c0dbb5acd..cdec798410 100644 --- a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md +++ b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md @@ -30,10 +30,13 @@ The following statistics are sent to the configured reporting endpoint: | `python_version` | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`. | | `total_users` | int | The number of registered users on the homeserver. | | `total_nonbridged_users` | int | The number of users, excluding those created by an Application Service. | -| `daily_user_type_native` | int | The number of native users created in the last 24 hours. | +| `daily_user_type_native` | int | The number of native, non-guest users created in the last 24 hours. | | `daily_user_type_guest` | int | The number of guest users created in the last 24 hours. | | `daily_user_type_bridged` | int | The number of users created by Application Services in the last 24 hours. | | `total_room_count` | int | The total number of rooms present on the homeserver. | +| `total_event_count` | int | The total number of events present on the homeserver. | +| `total_message_count` | int | The total number of non-state events with type `m.room.message` present on the homeserver. | +| `total_e2ee_event_count` | int | The total number of non-state events with type `m.room.encrypted` present on the homeserver. This can be used as a slight over-estimate for the number of encrypted messages. | | `daily_active_users` | int | The number of unique users[^1] that have used the homeserver in the last 24 hours. | | `monthly_active_users` | int | The number of unique users[^1] that have used the homeserver in the last 30 days. | | `daily_active_rooms` | int | The number of rooms that have had a (state) event with the type `m.room.message` sent in them in the last 24 hours. | @@ -50,8 +53,8 @@ The following statistics are sent to the configured reporting endpoint: | `cache_factor` | int | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching. | | `event_cache_size` | int | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching. | | `database_engine` | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3. | -| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | -| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | +| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | +| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | [^1]: Native matrix users and guests are always counted. If the diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index f602bbbeea..fe55838416 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -34,6 +34,22 @@ if TYPE_CHECKING: logger = logging.getLogger("synapse.app.homeserver") +ONE_MINUTE_SECONDS = 60 +ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS + +MILLISECONDS_PER_SECOND = 1000 + +INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS +""" +We wait 5 minutes to send the first set of stats as the server can be quite busy the +first few minutes +""" + +PHONE_HOME_INTERVAL_SECONDS = 3 * ONE_HOUR_SECONDS +""" +Phone home stats are sent every 3 hours +""" + # Contains the list of processes we will be monitoring # currently either 0 or 1 _stats_process: List[Tuple[int, "resource.struct_rusage"]] = [] @@ -121,6 +137,9 @@ async def phone_stats_home( room_count = await store.get_room_count() stats["total_room_count"] = room_count + stats["total_event_count"] = await store.count_total_events() + stats["total_message_count"] = await store.count_total_messages() + stats["total_e2ee_event_count"] = await store.count_total_e2ee_events() stats["daily_active_users"] = common_metrics.daily_active_users stats["monthly_active_users"] = await store.count_monthly_users() @@ -185,12 +204,14 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, 5 * 60 * 1000 + hs.get_datastores().main.generate_user_daily_visits, + 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, ) # monthly active user limiting functionality clock.looping_call( - hs.get_datastores().main.reap_monthly_active_users, 1000 * 60 * 60 + hs.get_datastores().main.reap_monthly_active_users, + ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) hs.get_datastores().main.reap_monthly_active_users() @@ -216,12 +237,20 @@ def start_phone_stats_home(hs: "HomeServer") -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000) + clock.looping_call( + generate_monthly_active_users, + 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, + ) # End of monthly active user settings if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000, hs, stats) + clock.looping_call( + phone_stats_home, + PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + hs, + stats, + ) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process @@ -229,4 +258,6 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes - clock.call_later(5 * 60, phone_stats_home, hs, stats) + clock.call_later( + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats + ) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 4b0bdd79c6..b821d1c1b4 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -47,7 +47,7 @@ from synapse.storage.databases.main.events_worker import ( ) from synapse.storage.databases.main.state_deltas import StateDeltasStore from synapse.storage.databases.main.stream import StreamWorkerStore -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.types import Cursor from synapse.types import JsonDict, RoomStreamToken, StateMap, StrCollection from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES @@ -311,6 +311,12 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS self._sliding_sync_membership_snapshots_fix_forgotten_column_bg_update, ) + # Add a background update to add triggers which track event counts. + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + self._event_stats_populate_counts_bg_update, + ) + # We want this to run on the main database at startup before we start processing # events. # @@ -2547,6 +2553,288 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS return num_rows + async def _event_stats_populate_counts_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: + """ + Background update to populate the `event_stats` table with initial + values, and register DB triggers to continue updating it. + + We first register TRIGGERs on rows being added/removed from the `events` table, + which will keep the event counts continuously updated. We also mark the stopping + point for the main population step so we don't double count events. + + Then we will iterate through the `events` table in batches and update event + counts until we reach the stopping point. + + This data is intended to be used by the phone-home stats to keep track + of total event and message counts. A trigger is preferred to counting + rows in the `events` table, as said table can grow quite large. + + It is also preferable to adding an index on the `events` table, as even + an index can grow large. And calculating total counts would require + querying that entire index. + """ + # The last event `stream_ordering` we processed (starting place of this next + # batch). + last_event_stream_ordering = progress.get( + "last_event_stream_ordering", -(1 << 31) + ) + # The event `stream_ordering` we should stop at. This is used to avoid double + # counting events that are already accounted for because of the triggers. + stop_event_stream_ordering: Optional[int] = progress.get( + "stop_event_stream_ordering", None + ) + + def _add_triggers_txn( + txn: LoggingTransaction, + ) -> Optional[int]: + """ + Adds the triggers to the `events` table to keep the `event_stats` counts + up-to-date. + + Also populates the `stop_event_stream_ordering` background update progress + value. This marks the point at which we added the triggers, so we can avoid + double counting events that are already accounted for in the population + step. + + Returns: + The latest event `stream_ordering` in the `events` table when the triggers + were added or `None` if the `events` table is empty. + """ + + # Each time an event is inserted into the `events` table, update the stats. + # + # We're using `AFTER` triggers as we want to count successful inserts/deletes and + # not the ones that could potentially fail. + if isinstance(txn.database_engine, Sqlite3Engine): + txn.execute( + """ + CREATE TRIGGER IF NOT EXISTS event_stats_events_insert_trigger + AFTER INSERT ON events + BEGIN + -- Always increment total_event_count + UPDATE event_stats SET total_event_count = total_event_count + 1; + + -- Increment unencrypted_message_count for m.room.message events + UPDATE event_stats + SET unencrypted_message_count = unencrypted_message_count + 1 + WHERE NEW.type = 'm.room.message' AND NEW.state_key IS NULL; + + -- Increment e2ee_event_count for m.room.encrypted events + UPDATE event_stats + SET e2ee_event_count = e2ee_event_count + 1 + WHERE NEW.type = 'm.room.encrypted' AND NEW.state_key IS NULL; + END; + """ + ) + + txn.execute( + """ + CREATE TRIGGER IF NOT EXISTS event_stats_events_delete_trigger + AFTER DELETE ON events + BEGIN + -- Always decrement total_event_count + UPDATE event_stats SET total_event_count = total_event_count - 1; + + -- Decrement unencrypted_message_count for m.room.message events + UPDATE event_stats + SET unencrypted_message_count = unencrypted_message_count - 1 + WHERE OLD.type = 'm.room.message' AND OLD.state_key IS NULL; + + -- Decrement e2ee_event_count for m.room.encrypted events + UPDATE event_stats + SET e2ee_event_count = e2ee_event_count - 1 + WHERE OLD.type = 'm.room.encrypted' AND OLD.state_key IS NULL; + END; + """ + ) + elif isinstance(txn.database_engine, PostgresEngine): + txn.execute( + """ + CREATE OR REPLACE FUNCTION event_stats_increment_counts() RETURNS trigger AS $BODY$ + BEGIN + IF TG_OP = 'INSERT' THEN + -- Always increment total_event_count + UPDATE event_stats SET total_event_count = total_event_count + 1; + + -- Increment unencrypted_message_count for m.room.message events + IF NEW.type = 'm.room.message' AND NEW.state_key IS NULL THEN + UPDATE event_stats SET unencrypted_message_count = unencrypted_message_count + 1; + END IF; + + -- Increment e2ee_event_count for m.room.encrypted events + IF NEW.type = 'm.room.encrypted' AND NEW.state_key IS NULL THEN + UPDATE event_stats SET e2ee_event_count = e2ee_event_count + 1; + END IF; + + -- We're not modifying the row being inserted/deleted, so we return it unchanged. + RETURN NEW; + + ELSIF TG_OP = 'DELETE' THEN + -- Always decrement total_event_count + UPDATE event_stats SET total_event_count = total_event_count - 1; + + -- Decrement unencrypted_message_count for m.room.message events + IF OLD.type = 'm.room.message' AND OLD.state_key IS NULL THEN + UPDATE event_stats SET unencrypted_message_count = unencrypted_message_count - 1; + END IF; + + -- Decrement e2ee_event_count for m.room.encrypted events + IF OLD.type = 'm.room.encrypted' AND OLD.state_key IS NULL THEN + UPDATE event_stats SET e2ee_event_count = e2ee_event_count - 1; + END IF; + + -- "The usual idiom in DELETE triggers is to return OLD." + -- (https://www.postgresql.org/docs/current/plpgsql-trigger.html) + RETURN OLD; + END IF; + + RAISE EXCEPTION 'update_event_stats() was run with unexpected operation (%%). ' + 'This indicates a trigger misconfiguration as this function should only' + 'run with INSERT/DELETE operations.', TG_OP; + END; + $BODY$ LANGUAGE plpgsql; + """ + ) + + # We could use `CREATE OR REPLACE TRIGGER` but that's only available in Postgres + # 14 (https://www.postgresql.org/docs/14/sql-createtrigger.html) + txn.execute( + """ + DO + $$BEGIN + CREATE TRIGGER event_stats_increment_counts_trigger + AFTER INSERT OR DELETE ON events + FOR EACH ROW + EXECUTE PROCEDURE event_stats_increment_counts(); + EXCEPTION + -- This acts as a "CREATE TRIGGER IF NOT EXISTS" for Postgres + WHEN duplicate_object THEN + NULL; + END;$$; + """ + ) + else: + raise NotImplementedError("Unknown database engine") + + # Find the latest `stream_ordering` in the `events` table. We need to do + # this in the same transaction as where we add the triggers so we don't miss + # any events. + txn.execute( + """ + SELECT stream_ordering + FROM events + ORDER BY stream_ordering DESC + LIMIT 1 + """ + ) + row = cast(Optional[Tuple[int]], txn.fetchone()) + + # Update the progress + if row is not None: + (max_stream_ordering,) = row + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + {"stop_event_stream_ordering": max_stream_ordering}, + ) + return max_stream_ordering + + return None + + # First, add the triggers to keep the `event_stats` values up-to-date. + # + # If we don't have a `stop_event_stream_ordering` yet, we need to add the + # triggers to the `events` table and set the stopping point so we don't + # double count `events` later. + if stop_event_stream_ordering is None: + stop_event_stream_ordering = await self.db_pool.runInteraction( + "_event_stats_populate_counts_bg_update_add_triggers", + _add_triggers_txn, + ) + + # If there is no `stop_event_stream_ordering`, then there are no events + # in the `events` table and we can end the background update altogether. + if stop_event_stream_ordering is None: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE + ) + return batch_size + + def _populate_txn( + txn: LoggingTransaction, + ) -> int: + """ + Updates the `event_stats` table from this batch of events. + """ + + # Increment the counts based on the events present in this batch. + txn.execute( + """ + WITH event_batch AS ( + SELECT * + FROM events + WHERE stream_ordering > ? AND stream_ordering <= ? + ORDER BY stream_ordering ASC + LIMIT ? + ), + batch_stats AS ( + SELECT + MAX(stream_ordering) AS max_stream_ordering, + COALESCE(COUNT(*), 0) AS total_event_count, + COALESCE(SUM(CASE WHEN type = 'm.room.message' AND state_key IS NULL THEN 1 ELSE 0 END), 0) AS unencrypted_message_count, + COALESCE(SUM(CASE WHEN type = 'm.room.encrypted' AND state_key IS NULL THEN 1 ELSE 0 END), 0) AS e2ee_event_count + FROM event_batch + + UNION ALL + + SELECT null, 0, 0, 0 + WHERE NOT EXISTS (SELECT 1 FROM event_batch) + LIMIT 1 + ) + UPDATE event_stats + SET + total_event_count = total_event_count + (SELECT total_event_count FROM batch_stats), + unencrypted_message_count = unencrypted_message_count + (SELECT unencrypted_message_count FROM batch_stats), + e2ee_event_count = e2ee_event_count + (SELECT e2ee_event_count FROM batch_stats) + RETURNING + (SELECT total_event_count FROM batch_stats) AS total_event_count, + (SELECT max_stream_ordering FROM batch_stats) AS max_stream_ordering + """, + (last_event_stream_ordering, stop_event_stream_ordering, batch_size), + ) + + # Get the results of the update + (total_event_count, max_stream_ordering) = cast( + Tuple[int, Optional[int]], txn.fetchone() + ) + + # Update the progress + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + { + "last_event_stream_ordering": max_stream_ordering, + "stop_event_stream_ordering": stop_event_stream_ordering, + }, + ) + + return total_event_count + + num_rows_processed = await self.db_pool.runInteraction( + "_event_stats_populate_counts_bg_update", + _populate_txn, + ) + + # No more rows to process, so our background update is complete. + if not num_rows_processed: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE + ) + + return batch_size + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 9ce1100b5c..a9cecc4bc1 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -126,6 +126,44 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): return await self.db_pool.runInteraction("count_e2ee_messages", _count_messages) + async def count_total_events(self) -> int: + """ + Returns the total number of events present on the server. + """ + + return await self.db_pool.simple_select_one_onecol( + table="event_stats", + keyvalues={}, + retcol="total_event_count", + desc="count_total_events", + ) + + async def count_total_messages(self) -> int: + """ + Returns the total number of `m.room.message` events present on the + server. + """ + + return await self.db_pool.simple_select_one_onecol( + table="event_stats", + keyvalues={}, + retcol="unencrypted_message_count", + desc="count_total_messages", + ) + + async def count_total_e2ee_events(self) -> int: + """ + Returns the total number of `m.room.encrypted` events present on the + server. + """ + + return await self.db_pool.simple_select_one_onecol( + table="event_stats", + keyvalues={}, + retcol="e2ee_event_count", + desc="count_total_e2ee_events", + ) + async def count_daily_sent_e2ee_messages(self) -> int: def _count_messages(txn: LoggingTransaction) -> int: # This is good enough as if you have silly characters in your own diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index ad683a3a07..7474ba4542 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 91 # remember to update the list below when updating +SCHEMA_VERSION = 92 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -162,6 +162,12 @@ Changes in SCHEMA_VERSION = 89 Changes in SCHEMA_VERSION = 90 - Add a column `participant` to `room_memberships` table - Add background update to delete unreferenced state groups. + +Changes in SCHEMA_VERSION = 91 + - TODO + +Changes in SCHEMA_VERSION = 92 + - Add `event_stats` table to store global event statistics like total counts """ diff --git a/synapse/storage/schema/main/delta/92/01_event_stats.sql b/synapse/storage/schema/main/delta/92/01_event_stats.sql new file mode 100644 index 0000000000..4bded03578 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/01_event_stats.sql @@ -0,0 +1,33 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Create the `event_stats` table to store these statistics. +CREATE TABLE event_stats ( + total_event_count INTEGER NOT NULL DEFAULT 0, + unencrypted_message_count INTEGER NOT NULL DEFAULT 0, + e2ee_event_count INTEGER NOT NULL DEFAULT 0 +); + +-- Insert initial values into the table. +INSERT INTO event_stats ( + total_event_count, + unencrypted_message_count, + e2ee_event_count +) VALUES (0, 0, 0); + +-- Add a background update to populate the `event_stats` table with the current counts +-- from the `events` table and add triggers to keep this count up-to-date. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9201, 'event_stats_populate_counts_bg_update', '{}'); + diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index e03ff7ffc8..73d19d91ed 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -52,3 +52,5 @@ class _BackgroundUpdates: MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = ( "mark_unreferenced_state_groups_for_deletion_bg_update" ) + + EVENT_STATS_POPULATE_COUNTS_BG_UPDATE = "event_stats_populate_counts_bg_update" diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py new file mode 100644 index 0000000000..1b3eafed5f --- /dev/null +++ b/tests/metrics/test_phone_home_stats.py @@ -0,0 +1,258 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging +from unittest.mock import AsyncMock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.app.phone_stats_home import ( + PHONE_HOME_INTERVAL_SECONDS, + start_phone_stats_home, +) +from synapse.rest import admin, login, register, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest +from tests.server import ThreadedMemoryReactorClock + +TEST_REPORT_STATS_ENDPOINT = "https://fake.endpoint/stats" +TEST_SERVER_CONTEXT = "test-server-context" + + +class PhoneHomeStatsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + register.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + # Configure the homeserver to enable stats reporting. + config = self.default_config() + config["report_stats"] = True + config["report_stats_endpoint"] = TEST_REPORT_STATS_ENDPOINT + + # Configure the server context so we can check it ends up being reported + config["server_context"] = TEST_SERVER_CONTEXT + + # Allow guests to be registered + config["allow_guest_access"] = True + + hs = self.setup_test_homeserver(config=config) + + # Replace the proxied http client with a mock, so we can inspect outbound requests to + # the configured stats endpoint. + self.put_json_mock = AsyncMock(return_value={}) + hs.get_proxied_http_client().put_json = self.put_json_mock # type: ignore[method-assign] + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + # Wait for the background updates to add the database triggers that keep the + # `event_stats` table up-to-date. + self.wait_for_background_updates() + + # Force stats reporting to occur + start_phone_stats_home(hs=hs) + + super().prepare(reactor, clock, hs) + + def _get_latest_phone_home_stats(self) -> JsonDict: + # Wait for `phone_stats_home` to be called again + a healthy margin (50s). + self.reactor.advance(2 * PHONE_HOME_INTERVAL_SECONDS + 50) + + # Extract the reported stats from our http client mock + mock_calls = self.put_json_mock.call_args_list + report_stats_calls = [] + for call in mock_calls: + if call.args[0] == TEST_REPORT_STATS_ENDPOINT: + report_stats_calls.append(call) + + self.assertGreaterEqual( + (len(report_stats_calls)), + 1, + "Expected at-least one call to the report_stats endpoint", + ) + + # Extract the phone home stats from the call + phone_home_stats = report_stats_calls[0].args[1] + + return phone_home_stats + + def _perform_user_actions(self) -> None: + """ + Perform some actions on the homeserver that would bump the phone home + stats. + """ + + # Create some users + user_1_mxid = self.register_user( + username="test_user_1", + password="test", + ) + user_2_mxid = self.register_user( + username="test_user_2", + password="test", + ) + # Note: `self.register_user` does not support guest registration, and updating the + # Admin API it calls to add a new parameter would cause the `mac` parameter to fail + # in a backwards-incompatible manner. Hence, we make a manual request here. + _guest_user_mxid = self.make_request( + method="POST", + path="/_matrix/client/v3/register?kind=guest", + content={ + "username": "guest_user", + "password": "test", + }, + shorthand=False, + ) + + # Log in to each user + user_1_token = self.login(username=user_1_mxid, password="test") + user_2_token = self.login(username=user_2_mxid, password="test") + + # Create a room between the two users + room_1_id = self.helper.create_room_as( + is_public=False, + tok=user_1_token, + ) + + # Mark this room as end-to-end encrypted + self.helper.send_state( + room_id=room_1_id, + event_type="m.room.encryption", + body={ + "algorithm": "m.megolm.v1.aes-sha2", + "rotation_period_ms": 604800000, + "rotation_period_msgs": 100, + }, + state_key="", + tok=user_1_token, + ) + + # User 1 invites user 2 + self.helper.invite( + room=room_1_id, + src=user_1_mxid, + targ=user_2_mxid, + tok=user_1_token, + ) + + # User 2 joins + self.helper.join( + room=room_1_id, + user=user_2_mxid, + tok=user_2_token, + ) + + # User 1 sends 10 unencrypted messages + for _ in range(10): + self.helper.send( + room_id=room_1_id, + body="Zoinks Scoob! A message!", + tok=user_1_token, + ) + + # User 2 sends 5 encrypted "messages" + for _ in range(5): + self.helper.send_event( + room_id=room_1_id, + type="m.room.encrypted", + content={ + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "some_key", + "ciphertext": { + "some_key": { + "type": 0, + "body": "encrypted_payload", + }, + }, + }, + tok=user_2_token, + ) + + def test_phone_home_stats(self) -> None: + """ + Test that the phone home stats contain the stats we expect based on + the scenario carried out in `prepare` + """ + # Do things to bump the stats + self._perform_user_actions() + + # Wait for the stats to be reported + phone_home_stats = self._get_latest_phone_home_stats() + + self.assertEqual( + phone_home_stats["homeserver"], self.hs.config.server.server_name + ) + + self.assertTrue(isinstance(phone_home_stats["memory_rss"], int)) + self.assertTrue(isinstance(phone_home_stats["cpu_average"], int)) + + self.assertEqual(phone_home_stats["server_context"], TEST_SERVER_CONTEXT) + + self.assertTrue(isinstance(phone_home_stats["timestamp"], int)) + self.assertTrue(isinstance(phone_home_stats["uptime_seconds"], int)) + self.assertTrue(isinstance(phone_home_stats["python_version"], str)) + + # We expect only our test users to exist on the homeserver + self.assertEqual(phone_home_stats["total_users"], 3) + self.assertEqual(phone_home_stats["total_nonbridged_users"], 3) + self.assertEqual(phone_home_stats["daily_user_type_native"], 2) + self.assertEqual(phone_home_stats["daily_user_type_guest"], 1) + self.assertEqual(phone_home_stats["daily_user_type_bridged"], 0) + self.assertEqual(phone_home_stats["total_room_count"], 1) + self.assertEqual(phone_home_stats["total_event_count"], 24) + self.assertEqual(phone_home_stats["total_message_count"], 10) + self.assertEqual(phone_home_stats["total_e2ee_event_count"], 5) + self.assertEqual(phone_home_stats["daily_active_users"], 2) + self.assertEqual(phone_home_stats["monthly_active_users"], 2) + self.assertEqual(phone_home_stats["daily_active_rooms"], 1) + self.assertEqual(phone_home_stats["daily_active_e2ee_rooms"], 1) + self.assertEqual(phone_home_stats["daily_messages"], 10) + self.assertEqual(phone_home_stats["daily_e2ee_messages"], 5) + self.assertEqual(phone_home_stats["daily_sent_messages"], 10) + self.assertEqual(phone_home_stats["daily_sent_e2ee_messages"], 5) + + # Our users have not been around for >30 days, hence these are all 0. + self.assertEqual(phone_home_stats["r30v2_users_all"], 0) + self.assertEqual(phone_home_stats["r30v2_users_android"], 0) + self.assertEqual(phone_home_stats["r30v2_users_ios"], 0) + self.assertEqual(phone_home_stats["r30v2_users_electron"], 0) + self.assertEqual(phone_home_stats["r30v2_users_web"], 0) + self.assertEqual( + phone_home_stats["cache_factor"], self.hs.config.caches.global_factor + ) + self.assertEqual( + phone_home_stats["event_cache_size"], + self.hs.config.caches.event_cache_size, + ) + self.assertEqual( + phone_home_stats["database_engine"], + self.hs.config.database.databases[0].config["name"], + ) + self.assertEqual( + phone_home_stats["database_server_version"], + self.hs.get_datastores().main.database_engine.server_version, + ) + + synapse_logger = logging.getLogger("synapse") + log_level = synapse_logger.getEffectiveLevel() + self.assertEqual(phone_home_stats["log_level"], logging.getLevelName(log_level)) diff --git a/tests/storage/test_event_stats.py b/tests/storage/test_event_stats.py new file mode 100644 index 0000000000..791ed27018 --- /dev/null +++ b/tests/storage/test_event_stats.py @@ -0,0 +1,237 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.rest import admin, login, register, room +from synapse.server import HomeServer +from synapse.types.storage import _BackgroundUpdates +from synapse.util import Clock + +from tests import unittest + + +class EventStatsTestCase(unittest.HomeserverTestCase): + """ + Tests for the `event_stats` table + """ + + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + register.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + # Wait for the background updates to add the database triggers that keep the + # `event_stats` table up-to-date. + # + # This also prevents background updates running during the tests and messing + # with the results. + self.wait_for_background_updates() + + super().prepare(reactor, clock, hs) + + def _perform_user_actions(self) -> None: + """ + Perform some actions on the homeserver that would bump the event counts. + """ + # Create some users + user_1_mxid = self.register_user( + username="test_user_1", + password="test", + ) + user_2_mxid = self.register_user( + username="test_user_2", + password="test", + ) + # Note: `self.register_user` does not support guest registration, and updating the + # Admin API it calls to add a new parameter would cause the `mac` parameter to fail + # in a backwards-incompatible manner. Hence, we make a manual request here. + _guest_user_mxid = self.make_request( + method="POST", + path="/_matrix/client/v3/register?kind=guest", + content={ + "username": "guest_user", + "password": "test", + }, + shorthand=False, + ) + + # Log in to each user + user_1_token = self.login(username=user_1_mxid, password="test") + user_2_token = self.login(username=user_2_mxid, password="test") + + # Create a room between the two users + room_1_id = self.helper.create_room_as( + is_public=False, + tok=user_1_token, + ) + + # Mark this room as end-to-end encrypted + self.helper.send_state( + room_id=room_1_id, + event_type="m.room.encryption", + body={ + "algorithm": "m.megolm.v1.aes-sha2", + "rotation_period_ms": 604800000, + "rotation_period_msgs": 100, + }, + state_key="", + tok=user_1_token, + ) + + # User 1 invites user 2 + self.helper.invite( + room=room_1_id, + src=user_1_mxid, + targ=user_2_mxid, + tok=user_1_token, + ) + + # User 2 joins + self.helper.join( + room=room_1_id, + user=user_2_mxid, + tok=user_2_token, + ) + + # User 1 sends 10 unencrypted messages + for _ in range(10): + self.helper.send( + room_id=room_1_id, + body="Zoinks Scoob! A message!", + tok=user_1_token, + ) + + # User 2 sends 5 encrypted "messages" + for _ in range(5): + self.helper.send_event( + room_id=room_1_id, + type="m.room.encrypted", + content={ + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "some_key", + "ciphertext": { + "some_key": { + "type": 0, + "body": "encrypted_payload", + }, + }, + }, + tok=user_2_token, + ) + + def test_background_update_with_events(self) -> None: + """ + Test that the background update to populate the `event_stats` table works + correctly when there are events in the database. + """ + # Do things to bump the stats + self._perform_user_actions() + + # Keep in mind: These are already populated as the background update has already + # ran once when Synapse started and added the database triggers which are + # incrementing things as new events come in. + self.assertEqual(self.get_success(self.store.count_total_events()), 24) + self.assertEqual(self.get_success(self.store.count_total_messages()), 10) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 5) + + # Run the background update again + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # We expect these values to double as the background update is being run *again* + # and will double-count the `events`. + self.assertEqual(self.get_success(self.store.count_total_events()), 48) + self.assertEqual(self.get_success(self.store.count_total_messages()), 20) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 10) + + def test_background_update_without_events(self) -> None: + """ + Test that the background update to populate the `event_stats` table works + correctly without events in the database. + """ + # Keep in mind: These are already populated as the background update has already + # ran once when Synapse started and added the database triggers which are + # incrementing things as new events come in. + # + # In this case, no events have been sent, so we expect the counts to be 0. + self.assertEqual(self.get_success(self.store.count_total_events()), 0) + self.assertEqual(self.get_success(self.store.count_total_messages()), 0) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 0) + + # Run the background update again + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + self.assertEqual(self.get_success(self.store.count_total_events()), 0) + self.assertEqual(self.get_success(self.store.count_total_messages()), 0) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 0) + + def test_background_update_resume_progress(self) -> None: + """ + Test that the background update to populate the `event_stats` table works + correctly to resume from `progress_json`. + """ + # Do things to bump the stats + self._perform_user_actions() + + # Keep in mind: These are already populated as the background update has already + # ran once when Synapse started and added the database triggers which are + # incrementing things as new events come in. + self.assertEqual(self.get_success(self.store.count_total_events()), 24) + self.assertEqual(self.get_success(self.store.count_total_messages()), 10) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 5) + + # Run the background update again + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, + "progress_json": '{ "last_event_stream_ordering": 14, "stop_event_stream_ordering": 21 }', + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # We expect these values to increase as the background update is being run + # *again* and will double-count some of the `events` over the range specified + # by the `progress_json`. + self.assertEqual(self.get_success(self.store.count_total_events()), 24 + 7) + self.assertEqual(self.get_success(self.store.count_total_messages()), 16) + self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 6) From 19b0e23c3d0af4a372194a6510281bd4ca3c1489 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 15 Apr 2025 14:58:30 +0000 Subject: [PATCH 248/691] Fix the token introspection cache logging access tokens when MAS integration is in use. (#18335) The `ResponseCache` logs keys by default. Let's not do that for access tokens. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18335.bugfix | 1 + synapse/api/auth/msc3861_delegated.py | 2 ++ synapse/util/caches/response_cache.py | 33 +++++++++++++++++++-------- 3 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 changelog.d/18335.bugfix diff --git a/changelog.d/18335.bugfix b/changelog.d/18335.bugfix new file mode 100644 index 0000000000..50df5a1b1d --- /dev/null +++ b/changelog.d/18335.bugfix @@ -0,0 +1 @@ +Fix the token introspection cache logging access tokens when MAS integration is in use. \ No newline at end of file diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 74e526123f..cc2c79fa96 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -201,6 +201,8 @@ class MSC3861DelegatedAuth(BaseAuth): self._clock, "token_introspection", timeout_ms=120_000, + # don't log because the keys are access tokens + enable_logging=False, ) self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 96b7ca83dc..54b99134b9 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -101,7 +101,13 @@ class ResponseCache(Generic[KV]): used rather than trying to compute a new response. """ - def __init__(self, clock: Clock, name: str, timeout_ms: float = 0): + def __init__( + self, + clock: Clock, + name: str, + timeout_ms: float = 0, + enable_logging: bool = True, + ): self._result_cache: Dict[KV, ResponseCacheEntry] = {} self.clock = clock @@ -109,6 +115,7 @@ class ResponseCache(Generic[KV]): self._name = name self._metrics = register_cache("response_cache", name, self, resizable=False) + self._enable_logging = enable_logging def size(self) -> int: return len(self._result_cache) @@ -246,9 +253,12 @@ class ResponseCache(Generic[KV]): """ entry = self._get(key) if not entry: - logger.debug( - "[%s]: no cached result for [%s], calculating new one", self._name, key - ) + if self._enable_logging: + logger.debug( + "[%s]: no cached result for [%s], calculating new one", + self._name, + key, + ) context = ResponseCacheContext(cache_key=key) if cache_context: kwargs["cache_context"] = context @@ -269,12 +279,15 @@ class ResponseCache(Generic[KV]): return await make_deferred_yieldable(entry.result.observe()) result = entry.result.observe() - if result.called: - logger.info("[%s]: using completed cached result for [%s]", self._name, key) - else: - logger.info( - "[%s]: using incomplete cached result for [%s]", self._name, key - ) + if self._enable_logging: + if result.called: + logger.info( + "[%s]: using completed cached result for [%s]", self._name, key + ) + else: + logger.info( + "[%s]: using incomplete cached result for [%s]", self._name, key + ) span_context = entry.opentracing_span_context with start_active_span_follows_from( From 45420b1d42416a3461f08aa3e6752c160c93366a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Apr 2025 16:02:27 +0100 Subject: [PATCH 249/691] Fix `force_tracing_for_users` config when using MAS (#18334) This is a copy of what we do for internal auth, and we should figure out a way to deduplicate some of this stuff: https://github.com/element-hq/synapse/blob/dd05cc55eedbf086ae224a13c9ae9f0332d96b1f/synapse/api/auth/internal.py#L62-L110 --- changelog.d/18334.bugfix | 1 + synapse/api/auth/msc3861_delegated.py | 51 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 changelog.d/18334.bugfix diff --git a/changelog.d/18334.bugfix b/changelog.d/18334.bugfix new file mode 100644 index 0000000000..d82e522cb8 --- /dev/null +++ b/changelog.d/18334.bugfix @@ -0,0 +1 @@ +Fix `force_tracing_for_users` config when using delegated auth. diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index cc2c79fa96..0598286cf4 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -45,6 +45,7 @@ from synapse.api.errors import ( ) from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable +from synapse.logging.opentracing import active_span, force_tracing, start_active_span from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall @@ -177,6 +178,7 @@ class MSC3861DelegatedAuth(BaseAuth): self._http_client = hs.get_proxied_http_client() self._hostname = hs.hostname self._admin_token: Callable[[], Optional[str]] = self._config.admin_token + self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users # # Token Introspection Cache # This remembers what users/devices are represented by which access tokens, @@ -363,6 +365,55 @@ class MSC3861DelegatedAuth(BaseAuth): allow_guest: bool = False, allow_expired: bool = False, allow_locked: bool = False, + ) -> Requester: + """Get a registered user's ID. + + Args: + request: An HTTP request with an access_token query parameter. + allow_guest: If False, will raise an AuthError if the user making the + request is a guest. + allow_expired: If True, allow the request through even if the account + is expired, or session token lifetime has ended. Note that + /login will deliver access tokens regardless of expiration. + + Returns: + Resolves to the requester + Raises: + InvalidClientCredentialsError if no user by that token exists or the token + is invalid. + AuthError if access is denied for the user in the access token + """ + parent_span = active_span() + with start_active_span("get_user_by_req"): + requester = await self._wrapped_get_user_by_req( + request, allow_guest, allow_expired, allow_locked + ) + + if parent_span: + if requester.authenticated_entity in self._force_tracing_for_users: + # request tracing is enabled for this user, so we need to force it + # tracing on for the parent span (which will be the servlet span). + # + # It's too late for the get_user_by_req span to inherit the setting, + # so we also force it on for that. + force_tracing() + force_tracing(parent_span) + parent_span.set_tag( + "authenticated_entity", requester.authenticated_entity + ) + parent_span.set_tag("user_id", requester.user.to_string()) + if requester.device_id is not None: + parent_span.set_tag("device_id", requester.device_id) + if requester.app_service is not None: + parent_span.set_tag("appservice_id", requester.app_service.id) + return requester + + async def _wrapped_get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, ) -> Requester: access_token = self.get_access_token_from_request(request) From 2c7a61e311002ebec0e3f5aff054f46dfb0015c5 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 15 Apr 2025 17:30:45 +0200 Subject: [PATCH 250/691] Don't cache introspection failures (#18339) --- changelog.d/18339.bugfix | 1 + synapse/api/auth/msc3861_delegated.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18339.bugfix diff --git a/changelog.d/18339.bugfix b/changelog.d/18339.bugfix new file mode 100644 index 0000000000..09d6d73420 --- /dev/null +++ b/changelog.d/18339.bugfix @@ -0,0 +1 @@ +Stop caching introspection failures when delegating auth to MAS. diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 0598286cf4..9ded3366e3 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -49,7 +49,7 @@ from synapse.logging.opentracing import active_span, force_tracing, start_active from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall -from synapse.util.caches.response_cache import ResponseCache +from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext if TYPE_CHECKING: from synapse.rest.admin.experimental_features import ExperimentalFeature @@ -279,7 +279,9 @@ class MSC3861DelegatedAuth(BaseAuth): metadata = await self._issuer_metadata.get() return metadata.get("introspection_endpoint") - async def _introspect_token(self, token: str) -> IntrospectionResult: + async def _introspect_token( + self, token: str, cache_context: ResponseCacheContext[str] + ) -> IntrospectionResult: """ Send a token to the introspection endpoint and returns the introspection response @@ -295,6 +297,8 @@ class MSC3861DelegatedAuth(BaseAuth): Returns: The introspection response """ + # By default, we shouldn't cache the result unless we know it's valid + cache_context.should_cache = False introspection_endpoint = await self._introspection_endpoint() raw_headers: Dict[str, str] = { "Content-Type": "application/x-www-form-urlencoded", @@ -352,6 +356,8 @@ class MSC3861DelegatedAuth(BaseAuth): "The introspection endpoint returned an invalid JSON response." ) + # We had a valid response, so we can cache it + cache_context.should_cache = True return IntrospectionResult( IntrospectionToken(**resp), retrieved_at_ms=self._clock.time_msec() ) @@ -482,7 +488,7 @@ class MSC3861DelegatedAuth(BaseAuth): try: introspection_result = await self._introspection_cache.wrap( - token, self._introspect_token, token + token, self._introspect_token, token, cache_context=True ) except Exception: logger.exception("Failed to introspect token") From 5ced4efe1d9ef45f05a0d013fa509ca8534a7ef2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 15 Apr 2025 10:48:32 -0600 Subject: [PATCH 251/691] 1.129.0rc1 --- CHANGES.md | 25 +++++++++++++++++++++++++ changelog.d/18133.misc | 1 - changelog.d/18232.feature | 1 - changelog.d/18260.feature | 1 - changelog.d/18294.docker | 1 - changelog.d/18334.bugfix | 1 - changelog.d/18335.bugfix | 1 - changelog.d/18337.misc | 1 - changelog.d/18339.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 11 files changed, 32 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/18133.misc delete mode 100644 changelog.d/18232.feature delete mode 100644 changelog.d/18260.feature delete mode 100644 changelog.d/18294.docker delete mode 100644 changelog.d/18334.bugfix delete mode 100644 changelog.d/18335.bugfix delete mode 100644 changelog.d/18337.misc delete mode 100644 changelog.d/18339.bugfix diff --git a/CHANGES.md b/CHANGES.md index 4ac25a3786..fcd4b0b9af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,28 @@ +# Synapse 1.129.0rc1 (2025-04-15) + +### Features + +- Add `passthrough_authorization_parameters` in OIDC configuration to allow to pass parameters to the authorization grant URL. ([\#18232](https://github.com/element-hq/synapse/issues/18232)) +- Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. ([\#18260](https://github.com/element-hq/synapse/issues/18260)) + +### Bugfixes + +- Fix `force_tracing_for_users` config when using delegated auth. ([\#18334](https://github.com/element-hq/synapse/issues/18334)) +- Fix the token introspection cache logging access tokens when MAS integration is in use. ([\#18335](https://github.com/element-hq/synapse/issues/18335)) +- Stop caching introspection failures when delegating auth to MAS. ([\#18339](https://github.com/element-hq/synapse/issues/18339)) + +### Updates to the Docker image + +- Optimize the build of the complement-synapse image. ([\#18294](https://github.com/element-hq/synapse/issues/18294)) + +### Internal Changes + +- Disable statement timeout during room purge. ([\#18133](https://github.com/element-hq/synapse/issues/18133)) +- Add cache to storage functions used to auth requests when using delegated auth. ([\#18337](https://github.com/element-hq/synapse/issues/18337)) + + + + # Synapse 1.128.0 (2025-04-08) No significant changes since 1.128.0rc1. diff --git a/changelog.d/18133.misc b/changelog.d/18133.misc deleted file mode 100644 index 151ceb2cab..0000000000 --- a/changelog.d/18133.misc +++ /dev/null @@ -1 +0,0 @@ -Disable statement timeout during room purge. diff --git a/changelog.d/18232.feature b/changelog.d/18232.feature deleted file mode 100644 index ba5059ba80..0000000000 --- a/changelog.d/18232.feature +++ /dev/null @@ -1 +0,0 @@ -Add `passthrough_authorization_parameters` in OIDC configuration to allow to pass parameters to the authorization grant URL. diff --git a/changelog.d/18260.feature b/changelog.d/18260.feature deleted file mode 100644 index e44e3dc990..0000000000 --- a/changelog.d/18260.feature +++ /dev/null @@ -1 +0,0 @@ -Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. diff --git a/changelog.d/18294.docker b/changelog.d/18294.docker deleted file mode 100644 index cc40ca90c0..0000000000 --- a/changelog.d/18294.docker +++ /dev/null @@ -1 +0,0 @@ -Optimize the build of the complement-synapse image. diff --git a/changelog.d/18334.bugfix b/changelog.d/18334.bugfix deleted file mode 100644 index d82e522cb8..0000000000 --- a/changelog.d/18334.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `force_tracing_for_users` config when using delegated auth. diff --git a/changelog.d/18335.bugfix b/changelog.d/18335.bugfix deleted file mode 100644 index 50df5a1b1d..0000000000 --- a/changelog.d/18335.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the token introspection cache logging access tokens when MAS integration is in use. \ No newline at end of file diff --git a/changelog.d/18337.misc b/changelog.d/18337.misc deleted file mode 100644 index b78276fe76..0000000000 --- a/changelog.d/18337.misc +++ /dev/null @@ -1 +0,0 @@ -Add cache to storage functions used to auth requests when using delegated auth. diff --git a/changelog.d/18339.bugfix b/changelog.d/18339.bugfix deleted file mode 100644 index 09d6d73420..0000000000 --- a/changelog.d/18339.bugfix +++ /dev/null @@ -1 +0,0 @@ -Stop caching introspection failures when delegating auth to MAS. diff --git a/debian/changelog b/debian/changelog index 56839ac5b4..e9eb331477 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.129.0~rc1) stable; urgency=medium + + * New Synapse release 1.129.0rc1. + + -- Synapse Packaging team Tue, 15 Apr 2025 10:47:43 -0600 + matrix-synapse-py3 (1.128.0) stable; urgency=medium * New Synapse release 1.128.0. diff --git a/pyproject.toml b/pyproject.toml index 8f8eb46e68..76a9cf4a70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.128.0" +version = "1.129.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 9b8eebbe4e765409b18c26c0c18d02e27ad79d12 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 15 Apr 2025 11:12:04 -0600 Subject: [PATCH 252/691] Changelog tweaks --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fcd4b0b9af..7fcb1fb4c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ### Features -- Add `passthrough_authorization_parameters` in OIDC configuration to allow to pass parameters to the authorization grant URL. ([\#18232](https://github.com/element-hq/synapse/issues/18232)) +- Add `passthrough_authorization_parameters` in OIDC configuration to allow passing parameters to the authorization grant URL. ([\#18232](https://github.com/element-hq/synapse/issues/18232)) - Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. ([\#18260](https://github.com/element-hq/synapse/issues/18260)) ### Bugfixes From 0046d7278bd8e350dcef40b95a05e116e6e66d90 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 16 Apr 2025 09:34:58 +0200 Subject: [PATCH 253/691] Fix ExternalIDReuse exception for concurrent transactions (#18342) --- changelog.d/18342.bugfix | 1 + .../storage/databases/main/registration.py | 29 +++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18342.bugfix diff --git a/changelog.d/18342.bugfix b/changelog.d/18342.bugfix new file mode 100644 index 0000000000..6fa2fa679a --- /dev/null +++ b/changelog.d/18342.bugfix @@ -0,0 +1 @@ +Fix `ExternalIDReuse` exception after migrating to MAS on workers with a high traffic. diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index eadbf4901c..c43f31353b 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -763,16 +763,33 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): txn, self.get_user_by_external_id, (auth_provider, external_id) ) - self.db_pool.simple_insert_txn( + # This INSERT ... ON CONFLICT DO NOTHING statement will cause a + # 'could not serialize access due to concurrent update' + # if the row is added concurrently by another transaction. + # This is exactly what we want, as it makes the transaction get retried + # in a new snapshot where we can check for a genuine conflict. + was_inserted = self.db_pool.simple_upsert_txn( txn, table="user_external_ids", - values={ - "auth_provider": auth_provider, - "external_id": external_id, - "user_id": user_id, - }, + keyvalues={"auth_provider": auth_provider, "external_id": external_id}, + values={}, + insertion_values={"user_id": user_id}, ) + if not was_inserted: + existing_id = self.db_pool.simple_select_one_onecol_txn( + txn, + table="user_external_ids", + keyvalues={"auth_provider": auth_provider, "user_id": user_id}, + retcol="external_id", + allow_none=True, + ) + + if existing_id != external_id: + raise ExternalIDReuseException( + f"{user_id!r} has external id {existing_id!r} for {auth_provider} but trying to add {external_id!r}" + ) + async def remove_user_external_id( self, auth_provider: str, external_id: str, user_id: str ) -> None: From c16a981f22dd559b56caa94a46392c206be9a265 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Apr 2025 14:14:56 +0100 Subject: [PATCH 254/691] Fix query for room participation (#18345) Follow on from #18068 Currently the subquery in `UPDATE` is pointless, as it will still just update all `room_membership` rows. Instead, we should look at the current membership event ID (which is easily retrieved from `local_current_membership`). We also add a `AND NOT participant` to noop the `UPDATE` when the `participant` flag is already set. cc @H-Shay --- changelog.d/18345.bugfix | 1 + synapse/storage/databases/main/roommember.py | 20 ++++++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 changelog.d/18345.bugfix diff --git a/changelog.d/18345.bugfix b/changelog.d/18345.bugfix new file mode 100644 index 0000000000..c8a001d4a3 --- /dev/null +++ b/changelog.d/18345.bugfix @@ -0,0 +1 @@ +Fix minor performance regression caused by tracking of room participation. Regressed in v1.128.0. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index a0a6dcd04e..dfa7dd48d9 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1622,14 +1622,11 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): sql = """ UPDATE room_memberships SET participant = true - WHERE (user_id, room_id) IN ( - SELECT user_id, room_id - FROM room_memberships - WHERE user_id = ? - AND room_id = ? - ORDER BY event_stream_ordering DESC - LIMIT 1 + WHERE event_id IN ( + SELECT event_id FROM local_current_membership + WHERE user_id = ? AND room_id = ? ) + AND NOT participant """ txn.execute(sql, (user_id, room_id)) @@ -1651,11 +1648,10 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): ) -> bool: sql = """ SELECT participant - FROM room_memberships - WHERE user_id = ? - AND room_id = ? - ORDER BY event_stream_ordering DESC - LIMIT 1 + FROM local_current_membership AS l + INNER JOIN room_memberships AS r USING (event_id) + WHERE l.user_id = ? + AND l.room_id = ? """ txn.execute(sql, (user_id, room_id)) res = txn.fetchone() From d67e9c5367c39b977c881800a2451e5bf0c5b713 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 16 Apr 2025 07:19:27 -0600 Subject: [PATCH 255/691] Update changelog --- CHANGES.md | 2 ++ changelog.d/18342.bugfix | 1 - changelog.d/18345.bugfix | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18342.bugfix delete mode 100644 changelog.d/18345.bugfix diff --git a/CHANGES.md b/CHANGES.md index 7fcb1fb4c7..37ab5faf67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ - Fix `force_tracing_for_users` config when using delegated auth. ([\#18334](https://github.com/element-hq/synapse/issues/18334)) - Fix the token introspection cache logging access tokens when MAS integration is in use. ([\#18335](https://github.com/element-hq/synapse/issues/18335)) - Stop caching introspection failures when delegating auth to MAS. ([\#18339](https://github.com/element-hq/synapse/issues/18339)) +- Fix `ExternalIDReuse` exception after migrating to MAS on workers with a high traffic. ([\#18342](https://github.com/element-hq/synapse/issues/18342)) +- Fix minor performance regression caused by tracking of room participation. Regressed in v1.128.0. ([\#18345](https://github.com/element-hq/synapse/issues/18345)) ### Updates to the Docker image diff --git a/changelog.d/18342.bugfix b/changelog.d/18342.bugfix deleted file mode 100644 index 6fa2fa679a..0000000000 --- a/changelog.d/18342.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `ExternalIDReuse` exception after migrating to MAS on workers with a high traffic. diff --git a/changelog.d/18345.bugfix b/changelog.d/18345.bugfix deleted file mode 100644 index c8a001d4a3..0000000000 --- a/changelog.d/18345.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix minor performance regression caused by tracking of room participation. Regressed in v1.128.0. From 89cb613a4ef321d2eb52f13b94d1f1fc3205bad1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 16 Apr 2025 16:41:41 +0000 Subject: [PATCH 256/691] Revert "Add total event, unencrypted message, and e2ee event counts to stats reporting" (#18346) Reverts element-hq/synapse#18260 It is causing a failure when building release debs for `debian:bullseye` with the following error: ``` sqlite3.OperationalError: near "RETURNING": syntax error ``` --- changelog.d/18260.feature | 1 - .../reporting_homeserver_usage_statistics.md | 9 +- synapse/app/phone_stats_home.py | 41 +-- .../databases/main/events_bg_updates.py | 290 +----------------- synapse/storage/databases/main/metrics.py | 38 --- synapse/storage/schema/__init__.py | 8 +- .../schema/main/delta/92/01_event_stats.sql | 33 -- synapse/types/storage/__init__.py | 2 - tests/metrics/test_phone_home_stats.py | 258 ---------------- tests/storage/test_event_stats.py | 237 -------------- 10 files changed, 10 insertions(+), 907 deletions(-) delete mode 100644 changelog.d/18260.feature delete mode 100644 synapse/storage/schema/main/delta/92/01_event_stats.sql delete mode 100644 tests/metrics/test_phone_home_stats.py delete mode 100644 tests/storage/test_event_stats.py diff --git a/changelog.d/18260.feature b/changelog.d/18260.feature deleted file mode 100644 index e44e3dc990..0000000000 --- a/changelog.d/18260.feature +++ /dev/null @@ -1 +0,0 @@ -Add `total_event_count`, `total_message_count`, and `total_e2ee_event_count` fields to the homeserver usage statistics. diff --git a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md index cdec798410..4c0dbb5acd 100644 --- a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md +++ b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md @@ -30,13 +30,10 @@ The following statistics are sent to the configured reporting endpoint: | `python_version` | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`. | | `total_users` | int | The number of registered users on the homeserver. | | `total_nonbridged_users` | int | The number of users, excluding those created by an Application Service. | -| `daily_user_type_native` | int | The number of native, non-guest users created in the last 24 hours. | +| `daily_user_type_native` | int | The number of native users created in the last 24 hours. | | `daily_user_type_guest` | int | The number of guest users created in the last 24 hours. | | `daily_user_type_bridged` | int | The number of users created by Application Services in the last 24 hours. | | `total_room_count` | int | The total number of rooms present on the homeserver. | -| `total_event_count` | int | The total number of events present on the homeserver. | -| `total_message_count` | int | The total number of non-state events with type `m.room.message` present on the homeserver. | -| `total_e2ee_event_count` | int | The total number of non-state events with type `m.room.encrypted` present on the homeserver. This can be used as a slight over-estimate for the number of encrypted messages. | | `daily_active_users` | int | The number of unique users[^1] that have used the homeserver in the last 24 hours. | | `monthly_active_users` | int | The number of unique users[^1] that have used the homeserver in the last 30 days. | | `daily_active_rooms` | int | The number of rooms that have had a (state) event with the type `m.room.message` sent in them in the last 24 hours. | @@ -53,8 +50,8 @@ The following statistics are sent to the configured reporting endpoint: | `cache_factor` | int | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching. | | `event_cache_size` | int | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching. | | `database_engine` | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3. | -| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | -| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | +| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | +| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | [^1]: Native matrix users and guests are always counted. If the diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index fe55838416..f602bbbeea 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -34,22 +34,6 @@ if TYPE_CHECKING: logger = logging.getLogger("synapse.app.homeserver") -ONE_MINUTE_SECONDS = 60 -ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS - -MILLISECONDS_PER_SECOND = 1000 - -INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS -""" -We wait 5 minutes to send the first set of stats as the server can be quite busy the -first few minutes -""" - -PHONE_HOME_INTERVAL_SECONDS = 3 * ONE_HOUR_SECONDS -""" -Phone home stats are sent every 3 hours -""" - # Contains the list of processes we will be monitoring # currently either 0 or 1 _stats_process: List[Tuple[int, "resource.struct_rusage"]] = [] @@ -137,9 +121,6 @@ async def phone_stats_home( room_count = await store.get_room_count() stats["total_room_count"] = room_count - stats["total_event_count"] = await store.count_total_events() - stats["total_message_count"] = await store.count_total_messages() - stats["total_e2ee_event_count"] = await store.count_total_e2ee_events() stats["daily_active_users"] = common_metrics.daily_active_users stats["monthly_active_users"] = await store.count_monthly_users() @@ -204,14 +185,12 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, - 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, + hs.get_datastores().main.generate_user_daily_visits, 5 * 60 * 1000 ) # monthly active user limiting functionality clock.looping_call( - hs.get_datastores().main.reap_monthly_active_users, - ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, + hs.get_datastores().main.reap_monthly_active_users, 1000 * 60 * 60 ) hs.get_datastores().main.reap_monthly_active_users() @@ -237,20 +216,12 @@ def start_phone_stats_home(hs: "HomeServer") -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - clock.looping_call( - generate_monthly_active_users, - 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, - ) + clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000) # End of monthly active user settings if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - clock.looping_call( - phone_stats_home, - PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, - hs, - stats, - ) + clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000, hs, stats) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process @@ -258,6 +229,4 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes - clock.call_later( - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats - ) + clock.call_later(5 * 60, phone_stats_home, hs, stats) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index b821d1c1b4..4b0bdd79c6 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -47,7 +47,7 @@ from synapse.storage.databases.main.events_worker import ( ) from synapse.storage.databases.main.state_deltas import StateDeltasStore from synapse.storage.databases.main.stream import StreamWorkerStore -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor from synapse.types import JsonDict, RoomStreamToken, StateMap, StrCollection from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES @@ -311,12 +311,6 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS self._sliding_sync_membership_snapshots_fix_forgotten_column_bg_update, ) - # Add a background update to add triggers which track event counts. - self.db_pool.updates.register_background_update_handler( - _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - self._event_stats_populate_counts_bg_update, - ) - # We want this to run on the main database at startup before we start processing # events. # @@ -2553,288 +2547,6 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS return num_rows - async def _event_stats_populate_counts_bg_update( - self, progress: JsonDict, batch_size: int - ) -> int: - """ - Background update to populate the `event_stats` table with initial - values, and register DB triggers to continue updating it. - - We first register TRIGGERs on rows being added/removed from the `events` table, - which will keep the event counts continuously updated. We also mark the stopping - point for the main population step so we don't double count events. - - Then we will iterate through the `events` table in batches and update event - counts until we reach the stopping point. - - This data is intended to be used by the phone-home stats to keep track - of total event and message counts. A trigger is preferred to counting - rows in the `events` table, as said table can grow quite large. - - It is also preferable to adding an index on the `events` table, as even - an index can grow large. And calculating total counts would require - querying that entire index. - """ - # The last event `stream_ordering` we processed (starting place of this next - # batch). - last_event_stream_ordering = progress.get( - "last_event_stream_ordering", -(1 << 31) - ) - # The event `stream_ordering` we should stop at. This is used to avoid double - # counting events that are already accounted for because of the triggers. - stop_event_stream_ordering: Optional[int] = progress.get( - "stop_event_stream_ordering", None - ) - - def _add_triggers_txn( - txn: LoggingTransaction, - ) -> Optional[int]: - """ - Adds the triggers to the `events` table to keep the `event_stats` counts - up-to-date. - - Also populates the `stop_event_stream_ordering` background update progress - value. This marks the point at which we added the triggers, so we can avoid - double counting events that are already accounted for in the population - step. - - Returns: - The latest event `stream_ordering` in the `events` table when the triggers - were added or `None` if the `events` table is empty. - """ - - # Each time an event is inserted into the `events` table, update the stats. - # - # We're using `AFTER` triggers as we want to count successful inserts/deletes and - # not the ones that could potentially fail. - if isinstance(txn.database_engine, Sqlite3Engine): - txn.execute( - """ - CREATE TRIGGER IF NOT EXISTS event_stats_events_insert_trigger - AFTER INSERT ON events - BEGIN - -- Always increment total_event_count - UPDATE event_stats SET total_event_count = total_event_count + 1; - - -- Increment unencrypted_message_count for m.room.message events - UPDATE event_stats - SET unencrypted_message_count = unencrypted_message_count + 1 - WHERE NEW.type = 'm.room.message' AND NEW.state_key IS NULL; - - -- Increment e2ee_event_count for m.room.encrypted events - UPDATE event_stats - SET e2ee_event_count = e2ee_event_count + 1 - WHERE NEW.type = 'm.room.encrypted' AND NEW.state_key IS NULL; - END; - """ - ) - - txn.execute( - """ - CREATE TRIGGER IF NOT EXISTS event_stats_events_delete_trigger - AFTER DELETE ON events - BEGIN - -- Always decrement total_event_count - UPDATE event_stats SET total_event_count = total_event_count - 1; - - -- Decrement unencrypted_message_count for m.room.message events - UPDATE event_stats - SET unencrypted_message_count = unencrypted_message_count - 1 - WHERE OLD.type = 'm.room.message' AND OLD.state_key IS NULL; - - -- Decrement e2ee_event_count for m.room.encrypted events - UPDATE event_stats - SET e2ee_event_count = e2ee_event_count - 1 - WHERE OLD.type = 'm.room.encrypted' AND OLD.state_key IS NULL; - END; - """ - ) - elif isinstance(txn.database_engine, PostgresEngine): - txn.execute( - """ - CREATE OR REPLACE FUNCTION event_stats_increment_counts() RETURNS trigger AS $BODY$ - BEGIN - IF TG_OP = 'INSERT' THEN - -- Always increment total_event_count - UPDATE event_stats SET total_event_count = total_event_count + 1; - - -- Increment unencrypted_message_count for m.room.message events - IF NEW.type = 'm.room.message' AND NEW.state_key IS NULL THEN - UPDATE event_stats SET unencrypted_message_count = unencrypted_message_count + 1; - END IF; - - -- Increment e2ee_event_count for m.room.encrypted events - IF NEW.type = 'm.room.encrypted' AND NEW.state_key IS NULL THEN - UPDATE event_stats SET e2ee_event_count = e2ee_event_count + 1; - END IF; - - -- We're not modifying the row being inserted/deleted, so we return it unchanged. - RETURN NEW; - - ELSIF TG_OP = 'DELETE' THEN - -- Always decrement total_event_count - UPDATE event_stats SET total_event_count = total_event_count - 1; - - -- Decrement unencrypted_message_count for m.room.message events - IF OLD.type = 'm.room.message' AND OLD.state_key IS NULL THEN - UPDATE event_stats SET unencrypted_message_count = unencrypted_message_count - 1; - END IF; - - -- Decrement e2ee_event_count for m.room.encrypted events - IF OLD.type = 'm.room.encrypted' AND OLD.state_key IS NULL THEN - UPDATE event_stats SET e2ee_event_count = e2ee_event_count - 1; - END IF; - - -- "The usual idiom in DELETE triggers is to return OLD." - -- (https://www.postgresql.org/docs/current/plpgsql-trigger.html) - RETURN OLD; - END IF; - - RAISE EXCEPTION 'update_event_stats() was run with unexpected operation (%%). ' - 'This indicates a trigger misconfiguration as this function should only' - 'run with INSERT/DELETE operations.', TG_OP; - END; - $BODY$ LANGUAGE plpgsql; - """ - ) - - # We could use `CREATE OR REPLACE TRIGGER` but that's only available in Postgres - # 14 (https://www.postgresql.org/docs/14/sql-createtrigger.html) - txn.execute( - """ - DO - $$BEGIN - CREATE TRIGGER event_stats_increment_counts_trigger - AFTER INSERT OR DELETE ON events - FOR EACH ROW - EXECUTE PROCEDURE event_stats_increment_counts(); - EXCEPTION - -- This acts as a "CREATE TRIGGER IF NOT EXISTS" for Postgres - WHEN duplicate_object THEN - NULL; - END;$$; - """ - ) - else: - raise NotImplementedError("Unknown database engine") - - # Find the latest `stream_ordering` in the `events` table. We need to do - # this in the same transaction as where we add the triggers so we don't miss - # any events. - txn.execute( - """ - SELECT stream_ordering - FROM events - ORDER BY stream_ordering DESC - LIMIT 1 - """ - ) - row = cast(Optional[Tuple[int]], txn.fetchone()) - - # Update the progress - if row is not None: - (max_stream_ordering,) = row - self.db_pool.updates._background_update_progress_txn( - txn, - _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - {"stop_event_stream_ordering": max_stream_ordering}, - ) - return max_stream_ordering - - return None - - # First, add the triggers to keep the `event_stats` values up-to-date. - # - # If we don't have a `stop_event_stream_ordering` yet, we need to add the - # triggers to the `events` table and set the stopping point so we don't - # double count `events` later. - if stop_event_stream_ordering is None: - stop_event_stream_ordering = await self.db_pool.runInteraction( - "_event_stats_populate_counts_bg_update_add_triggers", - _add_triggers_txn, - ) - - # If there is no `stop_event_stream_ordering`, then there are no events - # in the `events` table and we can end the background update altogether. - if stop_event_stream_ordering is None: - await self.db_pool.updates._end_background_update( - _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE - ) - return batch_size - - def _populate_txn( - txn: LoggingTransaction, - ) -> int: - """ - Updates the `event_stats` table from this batch of events. - """ - - # Increment the counts based on the events present in this batch. - txn.execute( - """ - WITH event_batch AS ( - SELECT * - FROM events - WHERE stream_ordering > ? AND stream_ordering <= ? - ORDER BY stream_ordering ASC - LIMIT ? - ), - batch_stats AS ( - SELECT - MAX(stream_ordering) AS max_stream_ordering, - COALESCE(COUNT(*), 0) AS total_event_count, - COALESCE(SUM(CASE WHEN type = 'm.room.message' AND state_key IS NULL THEN 1 ELSE 0 END), 0) AS unencrypted_message_count, - COALESCE(SUM(CASE WHEN type = 'm.room.encrypted' AND state_key IS NULL THEN 1 ELSE 0 END), 0) AS e2ee_event_count - FROM event_batch - - UNION ALL - - SELECT null, 0, 0, 0 - WHERE NOT EXISTS (SELECT 1 FROM event_batch) - LIMIT 1 - ) - UPDATE event_stats - SET - total_event_count = total_event_count + (SELECT total_event_count FROM batch_stats), - unencrypted_message_count = unencrypted_message_count + (SELECT unencrypted_message_count FROM batch_stats), - e2ee_event_count = e2ee_event_count + (SELECT e2ee_event_count FROM batch_stats) - RETURNING - (SELECT total_event_count FROM batch_stats) AS total_event_count, - (SELECT max_stream_ordering FROM batch_stats) AS max_stream_ordering - """, - (last_event_stream_ordering, stop_event_stream_ordering, batch_size), - ) - - # Get the results of the update - (total_event_count, max_stream_ordering) = cast( - Tuple[int, Optional[int]], txn.fetchone() - ) - - # Update the progress - self.db_pool.updates._background_update_progress_txn( - txn, - _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - { - "last_event_stream_ordering": max_stream_ordering, - "stop_event_stream_ordering": stop_event_stream_ordering, - }, - ) - - return total_event_count - - num_rows_processed = await self.db_pool.runInteraction( - "_event_stats_populate_counts_bg_update", - _populate_txn, - ) - - # No more rows to process, so our background update is complete. - if not num_rows_processed: - await self.db_pool.updates._end_background_update( - _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE - ) - - return batch_size - def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index a9cecc4bc1..9ce1100b5c 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -126,44 +126,6 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): return await self.db_pool.runInteraction("count_e2ee_messages", _count_messages) - async def count_total_events(self) -> int: - """ - Returns the total number of events present on the server. - """ - - return await self.db_pool.simple_select_one_onecol( - table="event_stats", - keyvalues={}, - retcol="total_event_count", - desc="count_total_events", - ) - - async def count_total_messages(self) -> int: - """ - Returns the total number of `m.room.message` events present on the - server. - """ - - return await self.db_pool.simple_select_one_onecol( - table="event_stats", - keyvalues={}, - retcol="unencrypted_message_count", - desc="count_total_messages", - ) - - async def count_total_e2ee_events(self) -> int: - """ - Returns the total number of `m.room.encrypted` events present on the - server. - """ - - return await self.db_pool.simple_select_one_onecol( - table="event_stats", - keyvalues={}, - retcol="e2ee_event_count", - desc="count_total_e2ee_events", - ) - async def count_daily_sent_e2ee_messages(self) -> int: def _count_messages(txn: LoggingTransaction) -> int: # This is good enough as if you have silly characters in your own diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 7474ba4542..ad683a3a07 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 92 # remember to update the list below when updating +SCHEMA_VERSION = 91 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -162,12 +162,6 @@ Changes in SCHEMA_VERSION = 89 Changes in SCHEMA_VERSION = 90 - Add a column `participant` to `room_memberships` table - Add background update to delete unreferenced state groups. - -Changes in SCHEMA_VERSION = 91 - - TODO - -Changes in SCHEMA_VERSION = 92 - - Add `event_stats` table to store global event statistics like total counts """ diff --git a/synapse/storage/schema/main/delta/92/01_event_stats.sql b/synapse/storage/schema/main/delta/92/01_event_stats.sql deleted file mode 100644 index 4bded03578..0000000000 --- a/synapse/storage/schema/main/delta/92/01_event_stats.sql +++ /dev/null @@ -1,33 +0,0 @@ --- --- This file is licensed under the Affero General Public License (AGPL) version 3. --- --- Copyright (C) 2025 New Vector, Ltd --- --- This program is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation, either version 3 of the --- License, or (at your option) any later version. --- --- See the GNU Affero General Public License for more details: --- . - - --- Create the `event_stats` table to store these statistics. -CREATE TABLE event_stats ( - total_event_count INTEGER NOT NULL DEFAULT 0, - unencrypted_message_count INTEGER NOT NULL DEFAULT 0, - e2ee_event_count INTEGER NOT NULL DEFAULT 0 -); - --- Insert initial values into the table. -INSERT INTO event_stats ( - total_event_count, - unencrypted_message_count, - e2ee_event_count -) VALUES (0, 0, 0); - --- Add a background update to populate the `event_stats` table with the current counts --- from the `events` table and add triggers to keep this count up-to-date. -INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (9201, 'event_stats_populate_counts_bg_update', '{}'); - diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index 73d19d91ed..e03ff7ffc8 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -52,5 +52,3 @@ class _BackgroundUpdates: MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = ( "mark_unreferenced_state_groups_for_deletion_bg_update" ) - - EVENT_STATS_POPULATE_COUNTS_BG_UPDATE = "event_stats_populate_counts_bg_update" diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py deleted file mode 100644 index 1b3eafed5f..0000000000 --- a/tests/metrics/test_phone_home_stats.py +++ /dev/null @@ -1,258 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . - -import logging -from unittest.mock import AsyncMock - -from twisted.test.proto_helpers import MemoryReactor - -from synapse.app.phone_stats_home import ( - PHONE_HOME_INTERVAL_SECONDS, - start_phone_stats_home, -) -from synapse.rest import admin, login, register, room -from synapse.server import HomeServer -from synapse.types import JsonDict -from synapse.util import Clock - -from tests import unittest -from tests.server import ThreadedMemoryReactorClock - -TEST_REPORT_STATS_ENDPOINT = "https://fake.endpoint/stats" -TEST_SERVER_CONTEXT = "test-server-context" - - -class PhoneHomeStatsTestCase(unittest.HomeserverTestCase): - servlets = [ - admin.register_servlets_for_client_rest_resource, - room.register_servlets, - register.register_servlets, - login.register_servlets, - ] - - def make_homeserver( - self, reactor: ThreadedMemoryReactorClock, clock: Clock - ) -> HomeServer: - # Configure the homeserver to enable stats reporting. - config = self.default_config() - config["report_stats"] = True - config["report_stats_endpoint"] = TEST_REPORT_STATS_ENDPOINT - - # Configure the server context so we can check it ends up being reported - config["server_context"] = TEST_SERVER_CONTEXT - - # Allow guests to be registered - config["allow_guest_access"] = True - - hs = self.setup_test_homeserver(config=config) - - # Replace the proxied http client with a mock, so we can inspect outbound requests to - # the configured stats endpoint. - self.put_json_mock = AsyncMock(return_value={}) - hs.get_proxied_http_client().put_json = self.put_json_mock # type: ignore[method-assign] - return hs - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.store = hs.get_datastores().main - - # Wait for the background updates to add the database triggers that keep the - # `event_stats` table up-to-date. - self.wait_for_background_updates() - - # Force stats reporting to occur - start_phone_stats_home(hs=hs) - - super().prepare(reactor, clock, hs) - - def _get_latest_phone_home_stats(self) -> JsonDict: - # Wait for `phone_stats_home` to be called again + a healthy margin (50s). - self.reactor.advance(2 * PHONE_HOME_INTERVAL_SECONDS + 50) - - # Extract the reported stats from our http client mock - mock_calls = self.put_json_mock.call_args_list - report_stats_calls = [] - for call in mock_calls: - if call.args[0] == TEST_REPORT_STATS_ENDPOINT: - report_stats_calls.append(call) - - self.assertGreaterEqual( - (len(report_stats_calls)), - 1, - "Expected at-least one call to the report_stats endpoint", - ) - - # Extract the phone home stats from the call - phone_home_stats = report_stats_calls[0].args[1] - - return phone_home_stats - - def _perform_user_actions(self) -> None: - """ - Perform some actions on the homeserver that would bump the phone home - stats. - """ - - # Create some users - user_1_mxid = self.register_user( - username="test_user_1", - password="test", - ) - user_2_mxid = self.register_user( - username="test_user_2", - password="test", - ) - # Note: `self.register_user` does not support guest registration, and updating the - # Admin API it calls to add a new parameter would cause the `mac` parameter to fail - # in a backwards-incompatible manner. Hence, we make a manual request here. - _guest_user_mxid = self.make_request( - method="POST", - path="/_matrix/client/v3/register?kind=guest", - content={ - "username": "guest_user", - "password": "test", - }, - shorthand=False, - ) - - # Log in to each user - user_1_token = self.login(username=user_1_mxid, password="test") - user_2_token = self.login(username=user_2_mxid, password="test") - - # Create a room between the two users - room_1_id = self.helper.create_room_as( - is_public=False, - tok=user_1_token, - ) - - # Mark this room as end-to-end encrypted - self.helper.send_state( - room_id=room_1_id, - event_type="m.room.encryption", - body={ - "algorithm": "m.megolm.v1.aes-sha2", - "rotation_period_ms": 604800000, - "rotation_period_msgs": 100, - }, - state_key="", - tok=user_1_token, - ) - - # User 1 invites user 2 - self.helper.invite( - room=room_1_id, - src=user_1_mxid, - targ=user_2_mxid, - tok=user_1_token, - ) - - # User 2 joins - self.helper.join( - room=room_1_id, - user=user_2_mxid, - tok=user_2_token, - ) - - # User 1 sends 10 unencrypted messages - for _ in range(10): - self.helper.send( - room_id=room_1_id, - body="Zoinks Scoob! A message!", - tok=user_1_token, - ) - - # User 2 sends 5 encrypted "messages" - for _ in range(5): - self.helper.send_event( - room_id=room_1_id, - type="m.room.encrypted", - content={ - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": "some_key", - "ciphertext": { - "some_key": { - "type": 0, - "body": "encrypted_payload", - }, - }, - }, - tok=user_2_token, - ) - - def test_phone_home_stats(self) -> None: - """ - Test that the phone home stats contain the stats we expect based on - the scenario carried out in `prepare` - """ - # Do things to bump the stats - self._perform_user_actions() - - # Wait for the stats to be reported - phone_home_stats = self._get_latest_phone_home_stats() - - self.assertEqual( - phone_home_stats["homeserver"], self.hs.config.server.server_name - ) - - self.assertTrue(isinstance(phone_home_stats["memory_rss"], int)) - self.assertTrue(isinstance(phone_home_stats["cpu_average"], int)) - - self.assertEqual(phone_home_stats["server_context"], TEST_SERVER_CONTEXT) - - self.assertTrue(isinstance(phone_home_stats["timestamp"], int)) - self.assertTrue(isinstance(phone_home_stats["uptime_seconds"], int)) - self.assertTrue(isinstance(phone_home_stats["python_version"], str)) - - # We expect only our test users to exist on the homeserver - self.assertEqual(phone_home_stats["total_users"], 3) - self.assertEqual(phone_home_stats["total_nonbridged_users"], 3) - self.assertEqual(phone_home_stats["daily_user_type_native"], 2) - self.assertEqual(phone_home_stats["daily_user_type_guest"], 1) - self.assertEqual(phone_home_stats["daily_user_type_bridged"], 0) - self.assertEqual(phone_home_stats["total_room_count"], 1) - self.assertEqual(phone_home_stats["total_event_count"], 24) - self.assertEqual(phone_home_stats["total_message_count"], 10) - self.assertEqual(phone_home_stats["total_e2ee_event_count"], 5) - self.assertEqual(phone_home_stats["daily_active_users"], 2) - self.assertEqual(phone_home_stats["monthly_active_users"], 2) - self.assertEqual(phone_home_stats["daily_active_rooms"], 1) - self.assertEqual(phone_home_stats["daily_active_e2ee_rooms"], 1) - self.assertEqual(phone_home_stats["daily_messages"], 10) - self.assertEqual(phone_home_stats["daily_e2ee_messages"], 5) - self.assertEqual(phone_home_stats["daily_sent_messages"], 10) - self.assertEqual(phone_home_stats["daily_sent_e2ee_messages"], 5) - - # Our users have not been around for >30 days, hence these are all 0. - self.assertEqual(phone_home_stats["r30v2_users_all"], 0) - self.assertEqual(phone_home_stats["r30v2_users_android"], 0) - self.assertEqual(phone_home_stats["r30v2_users_ios"], 0) - self.assertEqual(phone_home_stats["r30v2_users_electron"], 0) - self.assertEqual(phone_home_stats["r30v2_users_web"], 0) - self.assertEqual( - phone_home_stats["cache_factor"], self.hs.config.caches.global_factor - ) - self.assertEqual( - phone_home_stats["event_cache_size"], - self.hs.config.caches.event_cache_size, - ) - self.assertEqual( - phone_home_stats["database_engine"], - self.hs.config.database.databases[0].config["name"], - ) - self.assertEqual( - phone_home_stats["database_server_version"], - self.hs.get_datastores().main.database_engine.server_version, - ) - - synapse_logger = logging.getLogger("synapse") - log_level = synapse_logger.getEffectiveLevel() - self.assertEqual(phone_home_stats["log_level"], logging.getLevelName(log_level)) diff --git a/tests/storage/test_event_stats.py b/tests/storage/test_event_stats.py deleted file mode 100644 index 791ed27018..0000000000 --- a/tests/storage/test_event_stats.py +++ /dev/null @@ -1,237 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . - - -from twisted.test.proto_helpers import MemoryReactor - -from synapse.rest import admin, login, register, room -from synapse.server import HomeServer -from synapse.types.storage import _BackgroundUpdates -from synapse.util import Clock - -from tests import unittest - - -class EventStatsTestCase(unittest.HomeserverTestCase): - """ - Tests for the `event_stats` table - """ - - servlets = [ - admin.register_servlets_for_client_rest_resource, - room.register_servlets, - register.register_servlets, - login.register_servlets, - ] - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.store = hs.get_datastores().main - - # Wait for the background updates to add the database triggers that keep the - # `event_stats` table up-to-date. - # - # This also prevents background updates running during the tests and messing - # with the results. - self.wait_for_background_updates() - - super().prepare(reactor, clock, hs) - - def _perform_user_actions(self) -> None: - """ - Perform some actions on the homeserver that would bump the event counts. - """ - # Create some users - user_1_mxid = self.register_user( - username="test_user_1", - password="test", - ) - user_2_mxid = self.register_user( - username="test_user_2", - password="test", - ) - # Note: `self.register_user` does not support guest registration, and updating the - # Admin API it calls to add a new parameter would cause the `mac` parameter to fail - # in a backwards-incompatible manner. Hence, we make a manual request here. - _guest_user_mxid = self.make_request( - method="POST", - path="/_matrix/client/v3/register?kind=guest", - content={ - "username": "guest_user", - "password": "test", - }, - shorthand=False, - ) - - # Log in to each user - user_1_token = self.login(username=user_1_mxid, password="test") - user_2_token = self.login(username=user_2_mxid, password="test") - - # Create a room between the two users - room_1_id = self.helper.create_room_as( - is_public=False, - tok=user_1_token, - ) - - # Mark this room as end-to-end encrypted - self.helper.send_state( - room_id=room_1_id, - event_type="m.room.encryption", - body={ - "algorithm": "m.megolm.v1.aes-sha2", - "rotation_period_ms": 604800000, - "rotation_period_msgs": 100, - }, - state_key="", - tok=user_1_token, - ) - - # User 1 invites user 2 - self.helper.invite( - room=room_1_id, - src=user_1_mxid, - targ=user_2_mxid, - tok=user_1_token, - ) - - # User 2 joins - self.helper.join( - room=room_1_id, - user=user_2_mxid, - tok=user_2_token, - ) - - # User 1 sends 10 unencrypted messages - for _ in range(10): - self.helper.send( - room_id=room_1_id, - body="Zoinks Scoob! A message!", - tok=user_1_token, - ) - - # User 2 sends 5 encrypted "messages" - for _ in range(5): - self.helper.send_event( - room_id=room_1_id, - type="m.room.encrypted", - content={ - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": "some_key", - "ciphertext": { - "some_key": { - "type": 0, - "body": "encrypted_payload", - }, - }, - }, - tok=user_2_token, - ) - - def test_background_update_with_events(self) -> None: - """ - Test that the background update to populate the `event_stats` table works - correctly when there are events in the database. - """ - # Do things to bump the stats - self._perform_user_actions() - - # Keep in mind: These are already populated as the background update has already - # ran once when Synapse started and added the database triggers which are - # incrementing things as new events come in. - self.assertEqual(self.get_success(self.store.count_total_events()), 24) - self.assertEqual(self.get_success(self.store.count_total_messages()), 10) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 5) - - # Run the background update again - self.get_success( - self.store.db_pool.simple_insert( - "background_updates", - { - "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - "progress_json": "{}", - }, - ) - ) - self.store.db_pool.updates._all_done = False - self.wait_for_background_updates() - - # We expect these values to double as the background update is being run *again* - # and will double-count the `events`. - self.assertEqual(self.get_success(self.store.count_total_events()), 48) - self.assertEqual(self.get_success(self.store.count_total_messages()), 20) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 10) - - def test_background_update_without_events(self) -> None: - """ - Test that the background update to populate the `event_stats` table works - correctly without events in the database. - """ - # Keep in mind: These are already populated as the background update has already - # ran once when Synapse started and added the database triggers which are - # incrementing things as new events come in. - # - # In this case, no events have been sent, so we expect the counts to be 0. - self.assertEqual(self.get_success(self.store.count_total_events()), 0) - self.assertEqual(self.get_success(self.store.count_total_messages()), 0) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 0) - - # Run the background update again - self.get_success( - self.store.db_pool.simple_insert( - "background_updates", - { - "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - "progress_json": "{}", - }, - ) - ) - self.store.db_pool.updates._all_done = False - self.wait_for_background_updates() - - self.assertEqual(self.get_success(self.store.count_total_events()), 0) - self.assertEqual(self.get_success(self.store.count_total_messages()), 0) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 0) - - def test_background_update_resume_progress(self) -> None: - """ - Test that the background update to populate the `event_stats` table works - correctly to resume from `progress_json`. - """ - # Do things to bump the stats - self._perform_user_actions() - - # Keep in mind: These are already populated as the background update has already - # ran once when Synapse started and added the database triggers which are - # incrementing things as new events come in. - self.assertEqual(self.get_success(self.store.count_total_events()), 24) - self.assertEqual(self.get_success(self.store.count_total_messages()), 10) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 5) - - # Run the background update again - self.get_success( - self.store.db_pool.simple_insert( - "background_updates", - { - "update_name": _BackgroundUpdates.EVENT_STATS_POPULATE_COUNTS_BG_UPDATE, - "progress_json": '{ "last_event_stream_ordering": 14, "stop_event_stream_ordering": 21 }', - }, - ) - ) - self.store.db_pool.updates._all_done = False - self.wait_for_background_updates() - - # We expect these values to increase as the background update is being run - # *again* and will double-count some of the `events` over the range specified - # by the `progress_json`. - self.assertEqual(self.get_success(self.store.count_total_events()), 24 + 7) - self.assertEqual(self.get_success(self.store.count_total_messages()), 16) - self.assertEqual(self.get_success(self.store.count_total_e2ee_events()), 6) From 33824495ba520f57eae3687db847175b40f71d73 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Apr 2025 15:08:33 +0100 Subject: [PATCH 257/691] Move GET /devices/ off main process (#18355) We can't move PUT/DELETE as they do need to happen on main process (due to notification of device changes). --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18355.feature | 1 + docs/workers.md | 1 + synapse/rest/client/devices.py | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18355.feature diff --git a/changelog.d/18355.feature b/changelog.d/18355.feature new file mode 100644 index 0000000000..4813f0a291 --- /dev/null +++ b/changelog.d/18355.feature @@ -0,0 +1 @@ +Add support for handling `GET /devices/` on workers. diff --git a/docs/workers.md b/docs/workers.md index 08ee493da9..def902d24c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -280,6 +280,7 @@ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices/ # Account data requests ^/_matrix/client/(r0|v3|unstable)/.*/tags diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 4607b23494..0b075cc2f2 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -143,11 +143,11 @@ class DeviceRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler self.auth_handler = hs.get_auth_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled + self._is_main_process = hs.config.worker.worker_app is None async def on_GET( self, request: SynapseRequest, device_id: str @@ -179,6 +179,14 @@ class DeviceRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: + # Can only be run on main process, as changes to device lists must + # happen on main. + if not self._is_main_process: + error_message = "DELETE on /devices/ must be routed to main process" + logger.error(error_message) + raise SynapseError(500, error_message) + assert isinstance(self.device_handler, DeviceHandler) + requester = await self.auth.get_user_by_req(request) try: @@ -223,6 +231,14 @@ class DeviceRestServlet(RestServlet): async def on_PUT( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: + # Can only be run on main process, as changes to device lists must + # happen on main. + if not self._is_main_process: + error_message = "PUT on /devices/ must be routed to main process" + logger.error(error_message) + raise SynapseError(500, error_message) + assert isinstance(self.device_handler, DeviceHandler) + requester = await self.auth.get_user_by_req(request, allow_guest=True) body = parse_and_validate_json_object_from_request(request, self.PutBody) @@ -585,9 +601,9 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ): DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) + DeviceRestServlet(hs).register(http_server) if hs.config.worker.worker_app is None: - DeviceRestServlet(hs).register(http_server) if hs.config.experimental.msc2697_enabled: DehydratedDeviceServlet(hs).register(http_server) ClaimDehydratedDeviceServlet(hs).register(http_server) From 5b89c9264380da8f9cc55460f8215758fe570010 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Apr 2025 15:18:22 +0100 Subject: [PATCH 258/691] Allow /rooms/ admin API to be on workers (#18360) Tested by https://github.com/matrix-org/sytest/pull/1400 --- changelog.d/18360.misc | 1 + docs/workers.md | 1 + synapse/app/generic_worker.py | 3 ++- synapse/rest/admin/__init__.py | 5 +++-- 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18360.misc diff --git a/changelog.d/18360.misc b/changelog.d/18360.misc new file mode 100644 index 0000000000..e5bf4f536f --- /dev/null +++ b/changelog.d/18360.misc @@ -0,0 +1 @@ +Allow `/rooms/` admin API to be run on workers. diff --git a/docs/workers.md b/docs/workers.md index def902d24c..9ebcc886b1 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -249,6 +249,7 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$ ^/_matrix/client/(r0|v3|unstable)/capabilities$ ^/_matrix/client/(r0|v3|unstable)/notifications$ + ^/_synapse/admin/v1/rooms/ # Encryption requests ^/_matrix/client/(r0|v3|unstable)/keys/query$ diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index a528c3890d..e4120ed424 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -52,7 +52,7 @@ from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.rest import ClientRestResource -from synapse.rest.admin import register_servlets_for_media_repo +from synapse.rest.admin import AdminRestResource, register_servlets_for_media_repo from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree @@ -190,6 +190,7 @@ class GenericWorkerServer(HomeServer): resources.update(build_synapse_client_resource_tree(self)) resources["/.well-known"] = well_known_resource(self) + resources["/_synapse/admin"] = AdminRestResource(self) elif name == "federation": resources[FEDERATION_PREFIX] = TransportLayerServer(self) diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index f3c99663e8..5977ded4a0 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -275,7 +275,9 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: """ Register all the admin servlets. """ - # Admin servlets aren't registered on workers. + RoomRestServlet(hs).register(http_server) + + # Admin servlets below may not work on workers. if hs.config.worker.worker_app is not None: return @@ -283,7 +285,6 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: BlockRoomRestServlet(hs).register(http_server) ListRoomRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) - RoomRestServlet(hs).register(http_server) RoomRestV2Servlet(hs).register(http_server) RoomMembersRestServlet(hs).register(http_server) DeleteRoomStatusByDeleteIdRestServlet(hs).register(http_server) From 1482ad1917ef5e022b2d2238d30be74f50b47953 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:05:43 +0100 Subject: [PATCH 259/691] Bump sigstore/cosign-installer from 3.8.1 to 3.8.2 (#18366) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 052dcf800b..c617753c7a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1 + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From 2ff977a6c39caa24f35c58f2f5acd948dbdf122b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:05:55 +0100 Subject: [PATCH 260/691] Bump actions/add-to-project from 280af8ae1f83a494cfad2cb10f02f6d13529caa9 to 5b1a254a3546aef88e0a7724a77a623fa2e47c36 (#18365) --- .github/workflows/triage_labelled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index feab5906e0..e506be393f 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -11,7 +11,7 @@ jobs: if: > contains(github.event.issue.labels.*.name, 'X-Needs-Info') steps: - - uses: actions/add-to-project@280af8ae1f83a494cfad2cb10f02f6d13529caa9 # main (v1.0.2 + 10 commits) + - uses: actions/add-to-project@5b1a254a3546aef88e0a7724a77a623fa2e47c36 # main (v1.0.2 + 10 commits) id: add_project with: project-url: "https://github.com/orgs/matrix-org/projects/67" From a87981f673fe944690202cc4067a02f0c666eee4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:06:13 +0100 Subject: [PATCH 261/691] Bump actions/download-artifact from 4.2.1 to 4.3.0 (#18364) --- .github/workflows/release-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 573264229f..e0b8f2faf4 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -203,7 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. From 4c958c679a9c20930adfa25e64fc237fbf526591 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:06:26 +0100 Subject: [PATCH 262/691] Bump stefanzweifel/git-auto-commit-action from 5.1.0 to 5.2.0 (#18354) --- .github/workflows/fix_lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index fe699c1b2f..923e96a624 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -44,6 +44,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # v5.1.0 + - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5.2.0 with: commit_message: "Attempt to fix linting" From 39e17856a37570bda2fa912c6751e31bad6f970b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:06:36 +0100 Subject: [PATCH 263/691] Bump anyhow from 1.0.97 to 1.0.98 (#18336) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b17e9910a..e1c381e273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arc-swap" From 2ef782462011044718b0b3848f0cd33e5b2e1827 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:07:06 +0100 Subject: [PATCH 264/691] Bump pyo3-log from 0.12.2 to 0.12.3 (#18317) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1c381e273..822eb2cdba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b78e4983ba15bc62833a0e0941d965bc03690163f1127864f1408db25063466" +checksum = "7079e412e909af5d6be7c04a7f29f6a2837a080410e1c529c9dee2c367383db4" dependencies = [ "arc-swap", "log", From b0795d0cb670b2e8e66839e729cce42eb681832e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:07:15 +0100 Subject: [PATCH 265/691] Bump types-psycopg2 from 2.9.21.20250121 to 2.9.21.20250318 (#18316) Bumps [types-psycopg2](https://github.com/python/typeshed) from 2.9.21.20250121 to 2.9.21.20250318.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-psycopg2&package-manager=pip&previous-version=2.9.21.20250121&new-version=2.9.21.20250318)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2bf511e8a6..51e73bae54 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3007,14 +3007,14 @@ files = [ [[package]] name = "types-psycopg2" -version = "2.9.21.20250121" +version = "2.9.21.20250318" description = "Typing stubs for psycopg2" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_psycopg2-2.9.21.20250121-py3-none-any.whl", hash = "sha256:b890dc6f5a08b6433f0ff73a4ec9a834deedad3e914f2a4a6fd43df021f745f1"}, - {file = "types_psycopg2-2.9.21.20250121.tar.gz", hash = "sha256:2b0e2cd0f3747af1ae25a7027898716d80209604770ef3cbf350fe055b9c349b"}, + {file = "types_psycopg2-2.9.21.20250318-py3-none-any.whl", hash = "sha256:7296d111ad950bbd2fc979a1ab0572acae69047f922280e77db657c00d2c79c0"}, + {file = "types_psycopg2-2.9.21.20250318.tar.gz", hash = "sha256:eb6eac5bfb16adfd5f16b818918b9e26a40ede147e0f2bbffdf53a6ef7025a87"}, ] [[package]] From 7346760aed018eaf46a0bff2d0459b39881d2af5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:07:33 +0100 Subject: [PATCH 266/691] Bump pyopenssl from 24.3.0 to 25.0.0 (#18315) --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 51e73bae54..c6a6ce9826 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2053,18 +2053,19 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" -version = "24.3.0" +version = "25.0.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, - {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, + {file = "pyOpenSSL-25.0.0-py3-none-any.whl", hash = "sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90"}, + {file = "pyopenssl-25.0.0.tar.gz", hash = "sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16"}, ] [package.dependencies] cryptography = ">=41.0.5,<45" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] From 75832f25b08a058d01acde334033f76edc131ad5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:07:49 +0100 Subject: [PATCH 267/691] Bump types-jsonschema from 4.23.0.20240813 to 4.23.0.20241208 (#18305) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index c6a6ce9826..abd97a785b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2957,14 +2957,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.23.0.20240813" +version = "4.23.0.20241208" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types-jsonschema-4.23.0.20240813.tar.gz", hash = "sha256:c93f48206f209a5bc4608d295ac39f172fb98b9e24159ce577dbd25ddb79a1c0"}, - {file = "types_jsonschema-4.23.0.20240813-py3-none-any.whl", hash = "sha256:be283e23f0b87547316c2ee6b0fd36d95ea30e921db06478029e10b5b6aa6ac3"}, + {file = "types_jsonschema-4.23.0.20241208-py3-none-any.whl", hash = "sha256:87934bd9231c99d8eff94cacfc06ba668f7973577a9bd9e1f9de957c5737313e"}, + {file = "types_jsonschema-4.23.0.20241208.tar.gz", hash = "sha256:e8b15ad01f290ecf6aea53f93fbdf7d4730e4600313e89e8a7f95622f7e87b7c"}, ] [package.dependencies] From 0384fd72eeaa77dd56b52f38f7b339b95babe8dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:08:20 +0100 Subject: [PATCH 268/691] Bump softprops/action-gh-release from 1 to 2 (#18264) --- .github/workflows/release-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index e0b8f2faf4..e03c9d2bd5 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -213,7 +213,7 @@ jobs: tar -cvJf debs.tar.xz debs - name: Attach to release # Pinned to work around https://github.com/softprops/action-gh-release/issues/445 - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 + uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From e47de2b32de6183fd0cb91dda9b232de5d263345 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Apr 2025 14:08:11 +0100 Subject: [PATCH 269/691] Do not retry push during backoff period (#18363) This fixes a bug where if a pusher gets told about a new event to push it will ignore the backoff and immediately retry sending any pending push. --- changelog.d/18363.bugfix | 1 + synapse/push/httppusher.py | 6 +++ tests/push/test_http.py | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 changelog.d/18363.bugfix diff --git a/changelog.d/18363.bugfix b/changelog.d/18363.bugfix new file mode 100644 index 0000000000..bfa336d52f --- /dev/null +++ b/changelog.d/18363.bugfix @@ -0,0 +1 @@ +Fix longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 69790ecab5..7df8a128c9 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -205,6 +205,12 @@ class HttpPusher(Pusher): if self._is_processing: return + # Check if we are trying, but failing, to contact the pusher. If so, we + # don't try and start processing immediately and instead wait for the + # retry loop to try again later (which is controlled by the timer). + if self.failing_since and self.timed_call and self.timed_call.active(): + return + run_as_background_process("httppush.process", self._process) async def _process(self) -> None: diff --git a/tests/push/test_http.py b/tests/push/test_http.py index 5c235bbe53..b42fd284b6 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -1167,3 +1167,81 @@ class HTTPPusherTests(HomeserverTestCase): self.assertEqual( self.push_attempts[0][2]["notification"]["counts"]["unread"], 1 ) + + def test_push_backoff(self) -> None: + """ + The HTTP pusher will backoff correctly if it fails to contact the pusher. + """ + + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Register the pusher + user_tuple = self.get_success( + self.hs.get_datastores().main.get_user_by_access_token(access_token) + ) + assert user_tuple is not None + device_id = user_tuple.device_id + + self.get_success( + self.hs.get_pusherpool().add_or_update_pusher( + user_id=user_id, + device_id=device_id, + kind="http", + app_id="m.http", + app_display_name="HTTP Push Notifications", + device_display_name="pushy push", + pushkey="a@example.com", + lang=None, + data={"url": "http://example.com/_matrix/push/v1/notify"}, + ) + ) + + # Create a room with the other user + room = self.helper.create_room_as(user_id, tok=access_token) + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends some messages + self.helper.send(room, body="Message 1", tok=other_access_token) + + # One push was attempted to be sent + self.assertEqual(len(self.push_attempts), 1) + self.assertEqual( + self.push_attempts[0][1], "http://example.com/_matrix/push/v1/notify" + ) + self.assertEqual( + self.push_attempts[0][2]["notification"]["content"]["body"], "Message 1" + ) + self.push_attempts[0][0].callback({}) + self.pump() + + # Send another message, this time it fails + self.helper.send(room, body="Message 2", tok=other_access_token) + self.assertEqual(len(self.push_attempts), 2) + self.push_attempts[1][0].errback(Exception("couldn't connect")) + self.pump() + + # Sending yet another message doesn't trigger a push immediately + self.helper.send(room, body="Message 3", tok=other_access_token) + self.pump() + self.assertEqual(len(self.push_attempts), 2) + + # .. but waiting for a bit will cause more pushes + self.reactor.advance(10) + self.assertEqual(len(self.push_attempts), 3) + self.assertEqual( + self.push_attempts[2][2]["notification"]["content"]["body"], "Message 2" + ) + self.push_attempts[2][0].callback({}) + self.pump() + + self.assertEqual(len(self.push_attempts), 4) + self.assertEqual( + self.push_attempts[3][2]["notification"]["content"]["body"], "Message 3" + ) + self.push_attempts[3][0].callback({}) From ad140130cc3db503de3fd15aa2923417f46b700b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Apr 2025 14:08:22 +0100 Subject: [PATCH 270/691] Slight performance increase when using the ratelimiter (#18369) See the commits. --- changelog.d/18369.misc | 1 + synapse/api/ratelimiting.py | 19 ++++++++----------- synapse/rest/client/sync.py | 7 +++---- tests/api/test_ratelimiting.py | 4 +--- 4 files changed, 13 insertions(+), 18 deletions(-) create mode 100644 changelog.d/18369.misc diff --git a/changelog.d/18369.misc b/changelog.d/18369.misc new file mode 100644 index 0000000000..f4c0e5f006 --- /dev/null +++ b/changelog.d/18369.misc @@ -0,0 +1 @@ +Slight performance increase when using the ratelimiter. diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 229329a5ae..8665b3b765 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -20,8 +20,7 @@ # # -from collections import OrderedDict -from typing import Hashable, Optional, Tuple +from typing import Dict, Hashable, Optional, Tuple from synapse.api.errors import LimitExceededError from synapse.config.ratelimiting import RatelimitSettings @@ -80,12 +79,14 @@ class Ratelimiter: self.store = store self._limiter_name = cfg.key - # An ordered dictionary representing the token buckets tracked by this rate + # A dictionary representing the token buckets tracked by this rate # limiter. Each entry maps a key of arbitrary type to a tuple representing: # * The number of tokens currently in the bucket, # * The time point when the bucket was last completely empty, and # * The rate_hz (leak rate) of this particular bucket. - self.actions: OrderedDict[Hashable, Tuple[float, float, float]] = OrderedDict() + self.actions: Dict[Hashable, Tuple[float, float, float]] = {} + + self.clock.looping_call(self._prune_message_counts, 60 * 1000) def _get_key( self, requester: Optional[Requester], key: Optional[Hashable] @@ -169,9 +170,6 @@ class Ratelimiter: rate_hz = rate_hz if rate_hz is not None else self.rate_hz burst_count = burst_count if burst_count is not None else self.burst_count - # Remove any expired entries - self._prune_message_counts(time_now_s) - # Check if there is an existing count entry for this key action_count, time_start, _ = self._get_action_counts(key, time_now_s) @@ -246,13 +244,12 @@ class Ratelimiter: action_count, time_start, rate_hz = self._get_action_counts(key, time_now_s) self.actions[key] = (action_count + n_actions, time_start, rate_hz) - def _prune_message_counts(self, time_now_s: float) -> None: + def _prune_message_counts(self) -> None: """Remove message count entries that have not exceeded their defined rate_hz limit - - Args: - time_now_s: The current time """ + time_now_s = self.clock.time() + # We create a copy of the key list here as the dictionary is modified during # the loop for key in list(self.actions.keys()): diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 4fb9c0c8e7..bac02122d0 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -24,7 +24,7 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState -from synapse.api.errors import Codes, LimitExceededError, StoreError, SynapseError +from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import FilterCollection from synapse.api.presence import UserPresenceState from synapse.api.ratelimiting import Ratelimiter @@ -248,9 +248,8 @@ class SyncRestServlet(RestServlet): await self._server_notices_sender.on_user_syncing(user.to_string()) # ignore the presence update if the ratelimit is exceeded but do not pause the request - try: - await self._presence_per_user_limiter.ratelimit(requester, pause=0.0) - except LimitExceededError: + allowed, _ = await self._presence_per_user_limiter.can_do_action(requester) + if not allowed: affect_presence = False logger.debug("User set_presence ratelimit exceeded; ignoring it.") else: diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index a59e168db1..1a1cbde74e 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -220,9 +220,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): self.assertIn("test_id_1", limiter.actions) - self.get_success_or_raise( - limiter.can_do_action(None, key="test_id_2", _time_now_s=10) - ) + self.reactor.advance(60) self.assertNotIn("test_id_1", limiter.actions) From 4eaab31757f096a04f4278d722cdef1eb92a1743 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Apr 2025 14:08:32 +0100 Subject: [PATCH 271/691] Minor performance improvements to notifier/replication (#18367) These are some improvements to `on_new_event` which is a hot path. Not sure how much this will save, but maybe like ~5%? Possibly easier to review commit-by-commit --- changelog.d/18367.misc | 1 + synapse/notifier.py | 59 +++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 changelog.d/18367.misc diff --git a/changelog.d/18367.misc b/changelog.d/18367.misc new file mode 100644 index 0000000000..2e8b897fa6 --- /dev/null +++ b/changelog.d/18367.misc @@ -0,0 +1 @@ +Minor performance improvements to the notifier. diff --git a/synapse/notifier.py b/synapse/notifier.py index 88f531182a..1914d0c914 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -66,7 +66,6 @@ from synapse.types import ( from synapse.util.async_helpers import ( timeout_deferred, ) -from synapse.util.metrics import Measure from synapse.util.stringutils import shortstr from synapse.visibility import filter_events_for_client @@ -520,20 +519,22 @@ class Notifier: users = users or [] rooms = rooms or [] - with Measure(self.clock, "on_new_event"): - user_streams: Set[_NotifierUserStream] = set() + user_streams: Set[_NotifierUserStream] = set() - log_kv( - { - "waking_up_explicit_users": len(users), - "waking_up_explicit_rooms": len(rooms), - "users": shortstr(users), - "rooms": shortstr(rooms), - "stream": stream_key, - "stream_id": new_token, - } - ) + log_kv( + { + "waking_up_explicit_users": len(users), + "waking_up_explicit_rooms": len(rooms), + "users": shortstr(users), + "rooms": shortstr(rooms), + "stream": stream_key, + "stream_id": new_token, + } + ) + # Only calculate which user streams to wake up if there are, in fact, + # any user streams registered. + if self.user_to_user_stream or self.room_to_user_streams: for user in users: user_stream = self.user_to_user_stream.get(str(user)) if user_stream is not None: @@ -565,25 +566,25 @@ class Notifier: # We resolve all these deferreds in one go so that we only need to # call `PreserveLoggingContext` once, as it has a bunch of overhead # (to calculate performance stats) - with PreserveLoggingContext(): - for listener in listeners: - listener.callback(current_token) + if listeners: + with PreserveLoggingContext(): + for listener in listeners: + listener.callback(current_token) - users_woken_by_stream_counter.labels(stream_key).inc(len(user_streams)) + if user_streams: + users_woken_by_stream_counter.labels(stream_key).inc(len(user_streams)) - self.notify_replication() + self.notify_replication() - # Notify appservices. - try: - self.appservice_handler.notify_interested_services_ephemeral( - stream_key, - new_token, - users, - ) - except Exception: - logger.exception( - "Error notifying application services of ephemeral events" - ) + # Notify appservices. + try: + self.appservice_handler.notify_interested_services_ephemeral( + stream_key, + new_token, + users, + ) + except Exception: + logger.exception("Error notifying application services of ephemeral events") def on_new_replication_data(self) -> None: """Used to inform replication listeners that something has happened From dbf5b0be67b52dac11b376f52f0e5831ec60c4bc Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 30 Apr 2025 14:07:21 +0200 Subject: [PATCH 272/691] Remove the trigger added in #18260 and then reverted (#18373) See #18260 This is useful for anyone who tried Synapse v1.129.0rc1 out Fixes #18349 To test: - checkout v1.129.0rc1 and start - check that the events table has the trigger (`\dS events` with postgres) - checkout this PR and start - check that the events table doesn't have the trigger anymore --- changelog.d/18373.misc | 1 + synapse/storage/schema/__init__.py | 8 +++++++- .../main/delta/92/01_remove_trigger.sql.postgres | 16 ++++++++++++++++ .../main/delta/92/01_remove_trigger.sql.sqlite | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18373.misc create mode 100644 synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres create mode 100644 synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite diff --git a/changelog.d/18373.misc b/changelog.d/18373.misc new file mode 100644 index 0000000000..31817bbdf6 --- /dev/null +++ b/changelog.d/18373.misc @@ -0,0 +1 @@ +Revert "Add total event, unencrypted message, and e2ee event counts to stats reporting". diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index ad683a3a07..3c3b13437e 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 91 # remember to update the list below when updating +SCHEMA_VERSION = 92 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -162,6 +162,12 @@ Changes in SCHEMA_VERSION = 89 Changes in SCHEMA_VERSION = 90 - Add a column `participant` to `room_memberships` table - Add background update to delete unreferenced state groups. + +Changes in SCHEMA_VERSION = 91 + - Add a `sha256` column to the `local_media_repository` and `remote_media_cache` tables. + +Changes in SCHEMA_VERSION = 92 + - Cleaned up a trigger that was added in #18260 and then reverted. """ diff --git a/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres new file mode 100644 index 0000000000..e9f160cdcc --- /dev/null +++ b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Removes the trigger that was added in #18260 and then reverted +DROP TRIGGER IF EXISTS event_stats_increment_counts_trigger ON events; +DROP FUNCTION IF EXISTS event_stats_increment_counts(); diff --git a/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite new file mode 100644 index 0000000000..b5f084dde8 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Removes the trigger that was added in #18260 and then reverted +DROP TRIGGER IF EXISTS event_stats_events_insert_trigger; +DROP TRIGGER IF EXISTS event_stats_events_delete_trigger; From 02dca7c67a63ac6d61c3258d8d955ba92842b1b7 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 30 Apr 2025 14:35:32 +0200 Subject: [PATCH 273/691] Unschedule the background update scheduled in #18068. (#18372) Fixes #18356 --- changelog.d/18372.misc | 1 + synapse/storage/databases/main/roommember.py | 87 ------------------- ...umn_participant_room_memberships_table.sql | 6 +- ..._remove_populate_participant_bg_update.sql | 17 ++++ 4 files changed, 19 insertions(+), 92 deletions(-) create mode 100644 changelog.d/18372.misc create mode 100644 synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql diff --git a/changelog.d/18372.misc b/changelog.d/18372.misc new file mode 100644 index 0000000000..d835f44752 --- /dev/null +++ b/changelog.d/18372.misc @@ -0,0 +1 @@ +Add a column `participant` to `room_memberships` table. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index dfa7dd48d9..b8c78baa6c 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1693,93 +1693,6 @@ class RoomMemberBackgroundUpdateStore(SQLBaseStore): columns=["user_id", "room_id"], ) - self.db_pool.updates.register_background_update_handler( - "populate_participant_bg_update", self._populate_participant - ) - - async def _populate_participant(self, progress: JsonDict, batch_size: int) -> int: - """ - Background update to populate column `participant` on `room_memberships` table - - A 'participant' is someone who is currently joined to a room and has sent at least - one `m.room.message` or `m.room.encrypted` event. - - This background update will set the `participant` column across all rows in - `room_memberships` based on the user's *current* join status, and if - they've *ever* sent a message or encrypted event. Therefore one should - never assume the `participant` column's value is based solely on whether - the user participated in a previous "session" (where a "session" is defined - as a period between the user joining and leaving). See - https://github.com/element-hq/synapse/pull/18068#discussion_r1931070291 - for further detail. - """ - stream_token = progress.get("last_stream_token", None) - - def _get_max_stream_token_txn(txn: LoggingTransaction) -> int: - sql = """ - SELECT event_stream_ordering from room_memberships - ORDER BY event_stream_ordering DESC - LIMIT 1; - """ - txn.execute(sql) - res = txn.fetchone() - if not res or not res[0]: - return 0 - return res[0] - - def _background_populate_participant_txn( - txn: LoggingTransaction, stream_token: str - ) -> None: - sql = """ - UPDATE room_memberships - SET participant = True - FROM ( - SELECT DISTINCT c.state_key, e.room_id - FROM current_state_events AS c - INNER JOIN events AS e ON c.room_id = e.room_id - WHERE c.membership = 'join' - AND c.state_key = e.sender - AND ( - e.type = 'm.room.message' - OR e.type = 'm.room.encrypted' - ) - ) AS subquery - WHERE room_memberships.user_id = subquery.state_key - AND room_memberships.room_id = subquery.room_id - AND room_memberships.event_stream_ordering <= ? - AND room_memberships.event_stream_ordering > ?; - """ - batch = int(stream_token) - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE - txn.execute(sql, (stream_token, batch)) - - if stream_token is None: - stream_token = await self.db_pool.runInteraction( - "_get_max_stream_token", _get_max_stream_token_txn - ) - - if stream_token < 0: - await self.db_pool.updates._end_background_update( - "populate_participant_bg_update" - ) - return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE - - await self.db_pool.runInteraction( - "_background_populate_participant_txn", - _background_populate_participant_txn, - stream_token, - ) - - progress["last_stream_token"] = ( - stream_token - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE - ) - await self.db_pool.runInteraction( - "populate_participant_bg_update", - self.db_pool.updates._background_update_progress_txn, - "populate_participant_bg_update", - progress, - ) - return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE - async def _background_add_membership_profile( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql index 672be1031e..dafd046499 100644 --- a/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql +++ b/synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql @@ -13,8 +13,4 @@ -- Add a column `participant` to `room_memberships` table to track whether a room member has sent -- a `m.room.message` or `m.room.encrypted` event into a room they are a member of -ALTER TABLE room_memberships ADD COLUMN participant BOOLEAN DEFAULT FALSE; - --- Add a background update to populate `participant` column -INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (9001, 'populate_participant_bg_update', '{}'); \ No newline at end of file +ALTER TABLE room_memberships ADD COLUMN participant BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql b/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql new file mode 100644 index 0000000000..e1f377c37d --- /dev/null +++ b/synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove the background update if it was scheduled, as it is not rollback-safe +-- See https://github.com/element-hq/synapse/issues/18356 for context +DELETE FROM background_updates +WHERE update_name = 'populate_participant_bg_update'; \ No newline at end of file From a4ec96ca34c02b14ec6a48902ec396681481a7d2 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 30 Apr 2025 15:17:19 +0200 Subject: [PATCH 274/691] 1.129.0rc2 --- CHANGES.md | 10 ++++++++++ changelog.d/18372.misc | 1 - changelog.d/18373.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/18372.misc delete mode 100644 changelog.d/18373.misc diff --git a/CHANGES.md b/CHANGES.md index 37ab5faf67..6b24733aa8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +# Synapse 1.129.0rc2 (2025-04-30) + +### Internal Changes + +- Revert the slow background update introduced by [\#18068](https://github.com/element-hq/synapse/issues/18068) in v1.128.0. ([\#18372](https://github.com/element-hq/synapse/issues/18372)) +- Revert "Add total event, unencrypted message, and e2ee event counts to stats reporting", added in v1.129.0rc1. ([\#18373](https://github.com/element-hq/synapse/issues/18373)) + + + + # Synapse 1.129.0rc1 (2025-04-15) ### Features diff --git a/changelog.d/18372.misc b/changelog.d/18372.misc deleted file mode 100644 index d835f44752..0000000000 --- a/changelog.d/18372.misc +++ /dev/null @@ -1 +0,0 @@ -Add a column `participant` to `room_memberships` table. diff --git a/changelog.d/18373.misc b/changelog.d/18373.misc deleted file mode 100644 index 31817bbdf6..0000000000 --- a/changelog.d/18373.misc +++ /dev/null @@ -1 +0,0 @@ -Revert "Add total event, unencrypted message, and e2ee event counts to stats reporting". diff --git a/debian/changelog b/debian/changelog index e9eb331477..c4bc3cc966 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.129.0~rc2) stable; urgency=medium + + * New synapse release 1.129.0rc2. + + -- Synapse Packaging team Wed, 30 Apr 2025 13:13:16 +0000 + matrix-synapse-py3 (1.129.0~rc1) stable; urgency=medium * New Synapse release 1.129.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 76a9cf4a70..3005f71a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.129.0rc1" +version = "1.129.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 5f587dfd38e8b5c7e8ab71004054143bd2749f23 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 30 Apr 2025 15:25:59 +0200 Subject: [PATCH 275/691] Adjust changelog Co-Authored-By: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6b24733aa8..71da6c29f5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Synapse 1.129.0rc2 (2025-04-30) +Synapse 1.129.0rc1 was never formally released due to regressions discovered during the release process. 1.129.0rc2 fixes those regressions by reverting the affected PRs. + ### Internal Changes - Revert the slow background update introduced by [\#18068](https://github.com/element-hq/synapse/issues/18068) in v1.128.0. ([\#18372](https://github.com/element-hq/synapse/issues/18372)) From f79811ed80bebaa5b187637af6d16d413b07166e Mon Sep 17 00:00:00 2001 From: Kim Brose <2803622+HarHarLinks@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:27:08 +0200 Subject: [PATCH 276/691] Fix typo in docs about `push` (#18320) --- changelog.d/18320.doc | 1 + docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18320.doc diff --git a/changelog.d/18320.doc b/changelog.d/18320.doc new file mode 100644 index 0000000000..d84c279940 --- /dev/null +++ b/changelog.d/18320.doc @@ -0,0 +1 @@ +Fix typo in docs about the `push` config option. Contributed by @HarHarLinks. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 73fd9622ce..19dc9dd356 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4018,7 +4018,7 @@ This option has a number of sub-options. They are as follows: * `include_content`: Clients requesting push notifications can either have the body of the message sent in the notification poke along with other details like the sender, or just the event ID and room ID (`event_id_only`). - If clients choose the to have the body sent, this option controls whether the + If clients choose to have the body sent, this option controls whether the notification request includes the content of the event (other details like the sender are still included). If `event_id_only` is enabled, it has no effect. From 4097ada89fefe12e7ec6d2b7a3bfbc61e64e14a0 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 30 Apr 2025 09:54:30 -0400 Subject: [PATCH 277/691] Optimize `Dockerfile-workers` (#18292) - Use a `uv:python` image for the first build layer, to reduce the number of intermediate images required, as the main Dockerfile uses that image already - Use a cache mount for `apt` commands - Skip a pointless install of `redis-server`, since the redis Docker image is copied from instead - Move some RUN steps out of the final image layer & into the build layer Depends on https://github.com/element-hq/synapse/pull/18275 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18292.docker | 1 + docker/Dockerfile-workers | 50 +++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 changelog.d/18292.docker diff --git a/changelog.d/18292.docker b/changelog.d/18292.docker new file mode 100644 index 0000000000..cdb95b369b --- /dev/null +++ b/changelog.d/18292.docker @@ -0,0 +1 @@ +Optimize the build of the workers image. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index dd0bf59994..a7f576184d 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -3,18 +3,37 @@ ARG SYNAPSE_VERSION=latest ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION ARG DEBIAN_VERSION=bookworm +ARG PYTHON_VERSION=3.12 -# first of all, we create a base image with an nginx which we can copy into the +# first of all, we create a base image with dependencies which we can copy into the # target image. For repeated rebuilds, this is much faster than apt installing # each time. -FROM docker.io/library/debian:${DEBIAN_VERSION}-slim AS deps_base +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS deps_base + + # Tell apt to keep downloaded package files, as we're using cache mounts. + RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \ - redis-server nginx-light + nginx-light + + RUN \ + # remove default page + rm /etc/nginx/sites-enabled/default && \ + # have nginx log to stderr/out + ln -sf /dev/stdout /var/log/nginx/access.log && \ + ln -sf /dev/stderr /var/log/nginx/error.log + + # --link-mode=copy silences a warning as uv isn't able to do hardlinks between its cache + # (mounted as --mount=type=cache) and the target directory. + RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --link-mode=copy --prefix="/uv/usr/local" supervisor~=4.2 + + RUN mkdir -p /uv/etc/supervisor/conf.d # Similarly, a base to copy the redis server from. # @@ -27,31 +46,16 @@ FROM docker.io/library/redis:7-${DEBIAN_VERSION} AS redis_base # now build the final image, based on the the regular Synapse docker image FROM $FROM - # Install supervisord with uv pip instead of apt, to avoid installing a second - # copy of python. - # --link-mode=copy silences a warning as uv isn't able to do hardlinks between its cache - # (mounted as --mount=type=cache) and the target directory. - RUN \ - --mount=type=bind,from=ghcr.io/astral-sh/uv:0.6.8,source=/uv,target=/uv \ - --mount=type=cache,target=/root/.cache/uv \ - /uv pip install --link-mode=copy --prefix="/usr/local" supervisor~=4.2 - - RUN mkdir -p /etc/supervisor/conf.d - - # Copy over redis and nginx + # Copy over dependencies COPY --from=redis_base /usr/local/bin/redis-server /usr/local/bin - + COPY --from=deps_base /uv / COPY --from=deps_base /usr/sbin/nginx /usr/sbin COPY --from=deps_base /usr/share/nginx /usr/share/nginx COPY --from=deps_base /usr/lib/nginx /usr/lib/nginx COPY --from=deps_base /etc/nginx /etc/nginx - RUN rm /etc/nginx/sites-enabled/default - RUN mkdir /var/log/nginx /var/lib/nginx - RUN chown www-data /var/lib/nginx - - # have nginx log to stderr/out - RUN ln -sf /dev/stdout /var/log/nginx/access.log - RUN ln -sf /dev/stderr /var/log/nginx/error.log + COPY --from=deps_base /var/log/nginx /var/log/nginx + # chown to allow non-root user to write to http-*-temp-path dirs + COPY --from=deps_base --chown=www-data:root /var/lib/nginx /var/lib/nginx # Copy Synapse worker, nginx and supervisord configuration template files COPY ./docker/conf-workers/* /conf/ From 7563b2a2a316a7b249ef847ddbf5b63064eb1cc2 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 30 Apr 2025 10:22:09 -0400 Subject: [PATCH 278/691] configure_workers_and_start.py: unify python path (#18291) Use absolute path for python in script shebang, and invoke child python processes with sys.executable. This is consistent with the absolute path used to invoke python elsewhere (like in the supervisor config). ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18291.docker | 1 + docker/configure_workers_and_start.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18291.docker diff --git a/changelog.d/18291.docker b/changelog.d/18291.docker new file mode 100644 index 0000000000..b94c0e80e3 --- /dev/null +++ b/changelog.d/18291.docker @@ -0,0 +1 @@ +In configure_workers_and_start.py, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 6d73e8feaa..ff5cff3221 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/local/bin/python # # This file is licensed under the Affero General Public License (AGPL) version 3. # @@ -604,7 +604,7 @@ def generate_base_homeserver_config() -> None: # start.py already does this for us, so just call that. # note that this script is copied in in the official, monolith dockerfile os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT) - subprocess.run(["/usr/local/bin/python", "/start.py", "migrate_config"], check=True) + subprocess.run([sys.executable, "/start.py", "migrate_config"], check=True) def parse_worker_types( From 5ab05e7b95a687967fe99be33cb33a9c62fee34b Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 30 Apr 2025 10:26:08 -0400 Subject: [PATCH 279/691] docker: use shebangs to invoke generated scripts (#18295) When generating scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18295.docker | 1 + docker/Dockerfile-workers | 2 +- docker/complement/Dockerfile | 2 +- docker/configure_workers_and_start.py | 5 ++++- 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18295.docker diff --git a/changelog.d/18295.docker b/changelog.d/18295.docker new file mode 100644 index 0000000000..239def1f54 --- /dev/null +++ b/changelog.d/18295.docker @@ -0,0 +1 @@ +When generating container scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index a7f576184d..6d0fc1440b 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -74,4 +74,4 @@ FROM $FROM # Replace the healthcheck with one which checks *all* the workers. The script # is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ - CMD /bin/sh /healthcheck.sh + CMD ["/healthcheck.sh"] diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index dd029c5fbc..6ed084fe5d 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -58,4 +58,4 @@ ENTRYPOINT ["/start_for_complement.sh"] # Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ - CMD /bin/sh /healthcheck.sh + CMD ["/healthcheck.sh"] diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index ff5cff3221..8f96e57e50 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -376,9 +376,11 @@ def convert(src: str, dst: str, **template_vars: object) -> None: # # We use append mode in case the files have already been written to by something else # (for instance, as part of the instructions in a dockerfile). + exists = os.path.isfile(dst) with open(dst, "a") as outfile: # In case the existing file doesn't end with a newline - outfile.write("\n") + if exists: + outfile.write("\n") outfile.write(rendered) @@ -998,6 +1000,7 @@ def generate_worker_files( "/healthcheck.sh", healthcheck_urls=healthcheck_urls, ) + os.chmod("/healthcheck.sh", 0o755) # Ensure the logging directory exists log_dir = data_dir + "/logs" From 7be6c711d4a57f990003613c0b9715e3ac1502cb Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 30 Apr 2025 11:53:15 -0400 Subject: [PATCH 280/691] start_for_complement.sh: use more shell builtins (#18293) Avoid calling external tools when shell builtins suffice. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18293.docker | 1 + docker/complement/conf/start_for_complement.sh | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18293.docker diff --git a/changelog.d/18293.docker b/changelog.d/18293.docker new file mode 100644 index 0000000000..df47a68bfe --- /dev/null +++ b/changelog.d/18293.docker @@ -0,0 +1 @@ +In start_for_complement.sh, replace some external program calls with shell builtins. diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index 59b30e2051..a5e06396e2 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -9,7 +9,7 @@ echo " Args: $*" echo " Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR" function log { - d=$(date +"%Y-%m-%d %H:%M:%S,%3N") + d=$(printf '%(%Y-%m-%d %H:%M:%S)T,%.3s\n' ${EPOCHREALTIME/./ }) echo "$d $*" } @@ -103,12 +103,11 @@ fi # Note that both the key and certificate are in PEM format (not DER). # First generate a configuration file to set up a Subject Alternative Name. -cat > /conf/server.tls.conf < /conf/server.tls.conf # Generate an RSA key openssl genrsa -out /conf/server.tls.key 2048 @@ -123,8 +122,8 @@ openssl x509 -req -in /conf/server.tls.csr \ -out /conf/server.tls.crt -extfile /conf/server.tls.conf -extensions SAN # Assert that we have a Subject Alternative Name in the certificate. -# (grep will exit with 1 here if there isn't a SAN in the certificate.) -openssl x509 -in /conf/server.tls.crt -noout -text | grep DNS: +# (the test will exit with 1 here if there isn't a SAN in the certificate.) +[[ $(openssl x509 -in /conf/server.tls.crt -noout -text) == *DNS:* ]] export SYNAPSE_TLS_CERT=/conf/server.tls.crt export SYNAPSE_TLS_KEY=/conf/server.tls.key From d59bbd8b6b342d41641fddf99035d38e3939f18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Lav=C3=A9n?= Date: Wed, 30 Apr 2025 18:13:09 +0200 Subject: [PATCH 281/691] Added Pocket ID to openid.md (#18237) --- changelog.d/18237.doc | 1 + docs/openid.md | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 changelog.d/18237.doc diff --git a/changelog.d/18237.doc b/changelog.d/18237.doc new file mode 100644 index 0000000000..872f7cab7d --- /dev/null +++ b/changelog.d/18237.doc @@ -0,0 +1 @@ +Add documentation for configuring [Pocket ID](https://github.com/pocket-id/pocket-id) as an OIDC provider. \ No newline at end of file diff --git a/docs/openid.md b/docs/openid.md index 5a3d7e9fba..f86ba189c7 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -23,6 +23,7 @@ such as [Github][github-idp]. [auth0]: https://auth0.com/ [authentik]: https://goauthentik.io/ [lemonldap]: https://lemonldap-ng.org/ +[pocket-id]: https://pocket-id.org/ [okta]: https://www.okta.com/ [dex-idp]: https://github.com/dexidp/dex [keycloak-idp]: https://www.keycloak.org/docs/latest/server_admin/#sso-protocols @@ -624,6 +625,32 @@ oidc_providers: Note that the fields `client_id` and `client_secret` are taken from the CURL response above. +### Pocket ID + +[Pocket ID][pocket-id] is a simple OIDC provider that allows users to authenticate with their passkeys. +1. Go to `OIDC Clients` +2. Click on `Add OIDC Client` +3. Add a name, for example `Synapse` +4. Add `"https://auth.example.org/_synapse/client/oidc/callback` to `Callback URLs` # Replace `auth.example.org` with your domain +5. Click on `Save` +6. Note down your `Client ID` and `Client secret`, these will be used later + +Synapse config: + +```yaml +oidc_providers: + - idp_id: pocket_id + idp_name: Pocket ID + issuer: "https://auth.example.org/" # Replace with your domain + client_id: "your-client-id" # Replace with the "Client ID" you noted down before + client_secret: "your-client-secret" # Replace with the "Client secret" you noted down before + scopes: ["openid", "profile"] + user_mapping_provider: + config: + localpart_template: "{{ user.preferred_username }}" + display_name_template: "{{ user.name }}" +``` + ### Shibboleth with OIDC Plugin [Shibboleth](https://www.shibboleth.net/) is an open Standard IdP solution widely used by Universities. From 2965c9970c0b2742885dc345f6d70df7d5686423 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 May 2025 16:11:59 +0200 Subject: [PATCH 282/691] docs/workers.md: Add ^/_matrix/federation/v1/event/ to list of delegatable endpoints (#18377) --- changelog.d/18377.doc | 1 + docker/configure_workers_and_start.py | 1 + docs/upgrade.md | 10 ++++++++++ docs/workers.md | 1 + 4 files changed, 13 insertions(+) create mode 100644 changelog.d/18377.doc diff --git a/changelog.d/18377.doc b/changelog.d/18377.doc new file mode 100644 index 0000000000..ceb2b64e5d --- /dev/null +++ b/changelog.d/18377.doc @@ -0,0 +1 @@ +Add `/_matrix/federation/v1/version` to list of federation endpoints that can be handled by workers. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 8f96e57e50..df34d51f77 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -202,6 +202,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "app": "synapse.app.generic_worker", "listener_resources": ["federation"], "endpoint_patterns": [ + "^/_matrix/federation/v1/version$", "^/_matrix/federation/(v1|v2)/event/", "^/_matrix/federation/(v1|v2)/state/", "^/_matrix/federation/(v1|v2)/state_ids/", diff --git a/docs/upgrade.md b/docs/upgrade.md index 07a9641fdd..d508e2231e 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,16 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.130.0 + +## Documented endpoint which can be delegated to a federation worker + +The endpoint `^/_matrix/federation/v1/version$` can be delegated to a federation +worker. This is not new behaviour, but had not been documented yet. The +[list of delegatable endpoints](workers.md#synapseappgeneric_worker) has +been updated to include it. Make sure to check your reverse proxy rules if you +are using workers. + # Upgrading to v1.126.0 ## Room list publication rules change diff --git a/docs/workers.md b/docs/workers.md index 9ebcc886b1..2597e78217 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -200,6 +200,7 @@ information. ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ # Federation requests + ^/_matrix/federation/v1/version$ ^/_matrix/federation/v1/event/ ^/_matrix/federation/v1/state/ ^/_matrix/federation/v1/state_ids/ From 6dc1ecd35972c95ce62c5e0563245845c9c64e49 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 1 May 2025 11:30:00 -0700 Subject: [PATCH 283/691] Add an Admin API endpoint to fetch scheduled tasks (#18214) --- changelog.d/18214.feature | 1 + docs/admin_api/scheduled_tasks.md | 54 +++++++ synapse/rest/admin/__init__.py | 2 + synapse/rest/admin/scheduled_tasks.py | 70 +++++++++ tests/rest/admin/test_scheduled_tasks.py | 192 +++++++++++++++++++++++ 5 files changed, 319 insertions(+) create mode 100644 changelog.d/18214.feature create mode 100644 docs/admin_api/scheduled_tasks.md create mode 100644 synapse/rest/admin/scheduled_tasks.py create mode 100644 tests/rest/admin/test_scheduled_tasks.py diff --git a/changelog.d/18214.feature b/changelog.d/18214.feature new file mode 100644 index 0000000000..751cb7d383 --- /dev/null +++ b/changelog.d/18214.feature @@ -0,0 +1 @@ +Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. \ No newline at end of file diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md new file mode 100644 index 0000000000..1708871a6d --- /dev/null +++ b/docs/admin_api/scheduled_tasks.md @@ -0,0 +1,54 @@ +# Show scheduled tasks + +This API returns information about scheduled tasks. + +To use it, you will need to authenticate by providing an `access_token` +for a server admin: see [Admin API](../usage/administration/admin_api/). + +The api is: +``` +GET /_synapse/admin/v1/scheduled_tasks +``` + +It returns a JSON body like the following: + +```json +{ + "scheduled_tasks": [ + { + "id": "GSA124oegf1", + "action": "shutdown_room", + "status": "complete", + "timestamp": 23423523, + "resource_id": "!roomid", + "result": "some result", + "error": null + } + ] +} +``` + +**Query parameters:** + +* `action_name`: string - Is optional. Returns only the scheduled tasks with the given action name. +* `resource_id`: string - Is optional. Returns only the scheduled tasks with the given resource id. +* `status`: string - Is optional. Returns only the scheduled tasks matching the given status, one of + - "scheduled" - Task is scheduled but not active + - "active" - Task is active and probably running, and if not will be run on next scheduler loop run + - "complete" - Task has completed successfully + - "failed" - Task is over and either returned a failed status, or had an exception + +* `max_timestamp`: int - Is optional. Returns only the scheduled tasks with a timestamp inferior to the specified one. + +**Response** + +The following fields are returned in the JSON response body along with a `200` HTTP status code: + +* `id`: string - ID of scheduled task. +* `action`: string - The name of the scheduled task's action. +* `status`: string - The status of the scheduled task. +* `timestamp_ms`: integer - The timestamp (in milliseconds since the unix epoch) of the given task - If the status is "scheduled" then this represents when it should be launched. + Otherwise it represents the last time this task got a change of state. +* `resource_id`: Optional string - The resource id of the scheduled task, if it possesses one +* `result`: Optional Json - Any result of the scheduled task, if given +* `error`: Optional string - If the task has the status "failed", the error associated with this failure diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 5977ded4a0..cf809d1a27 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -86,6 +86,7 @@ from synapse.rest.admin.rooms import ( RoomStateRestServlet, RoomTimestampToEventRestServlet, ) +from synapse.rest.admin.scheduled_tasks import ScheduledTasksRestServlet from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.rest.admin.statistics import ( LargestRoomsStatistics, @@ -338,6 +339,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: BackgroundUpdateStartJobRestServlet(hs).register(http_server) ExperimentalFeaturesRestServlet(hs).register(http_server) SuspendAccountRestServlet(hs).register(http_server) + ScheduledTasksRestServlet(hs).register(http_server) def register_servlets_for_client_rest_resource( diff --git a/synapse/rest/admin/scheduled_tasks.py b/synapse/rest/admin/scheduled_tasks.py new file mode 100644 index 0000000000..2ae13021b9 --- /dev/null +++ b/synapse/rest/admin/scheduled_tasks.py @@ -0,0 +1,70 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +# +from typing import TYPE_CHECKING, Tuple + +from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.admin import admin_patterns, assert_requester_is_admin +from synapse.types import JsonDict, TaskStatus + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class ScheduledTasksRestServlet(RestServlet): + """Get a list of scheduled tasks and their statuses + optionally filtered by action name, resource id, status, and max timestamp + """ + + PATTERNS = admin_patterns("/scheduled_tasks$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + # extract query params + action_name = parse_string(request, "action_name") + resource_id = parse_string(request, "resource_id") + status = parse_string(request, "job_status") + max_timestamp = parse_integer(request, "max_timestamp") + + actions = [action_name] if action_name else None + statuses = [TaskStatus(status)] if status else None + + tasks = await self._store.get_scheduled_tasks( + actions=actions, + resource_id=resource_id, + statuses=statuses, + max_timestamp=max_timestamp, + ) + + json_tasks = [] + for task in tasks: + result_task = { + "id": task.id, + "action": task.action, + "status": task.status, + "timestamp_ms": task.timestamp, + "resource_id": task.resource_id, + "result": task.result, + "error": task.error, + } + json_tasks.append(result_task) + + return 200, {"scheduled_tasks": json_tasks} diff --git a/tests/rest/admin/test_scheduled_tasks.py b/tests/rest/admin/test_scheduled_tasks.py new file mode 100644 index 0000000000..9654e9322b --- /dev/null +++ b/tests/rest/admin/test_scheduled_tasks.py @@ -0,0 +1,192 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +# +from typing import Mapping, Optional, Tuple + +from twisted.test.proto_helpers import MemoryReactor + +import synapse.rest.admin +from synapse.api.errors import Codes +from synapse.rest.client import login +from synapse.server import HomeServer +from synapse.types import JsonMapping, ScheduledTask, TaskStatus +from synapse.util import Clock + +from tests import unittest + + +class ScheduledTasksAdminApiTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + self._task_scheduler = hs.get_task_scheduler() + + # create and schedule a few tasks + async def _test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.ACTIVE, None, None + + async def _finished_test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.COMPLETE, None, None + + async def _failed_test_task( + task: ScheduledTask, + ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]: + return TaskStatus.FAILED, None, "Everything failed" + + self._task_scheduler.register_action(_test_task, "test_task") + self.get_success( + self._task_scheduler.schedule_task("test_task", resource_id="test") + ) + + self._task_scheduler.register_action(_finished_test_task, "finished_test_task") + self.get_success( + self._task_scheduler.schedule_task( + "finished_test_task", resource_id="finished_task" + ) + ) + + self._task_scheduler.register_action(_failed_test_task, "failed_test_task") + self.get_success( + self._task_scheduler.schedule_task( + "failed_test_task", resource_id="failed_task" + ) + ) + + def check_scheduled_tasks_response(self, scheduled_tasks: Mapping) -> list: + result = [] + for task in scheduled_tasks: + if task["resource_id"] == "test": + self.assertEqual(task["status"], TaskStatus.ACTIVE) + self.assertEqual(task["action"], "test_task") + result.append(task) + if task["resource_id"] == "finished_task": + self.assertEqual(task["status"], TaskStatus.COMPLETE) + self.assertEqual(task["action"], "finished_test_task") + result.append(task) + if task["resource_id"] == "failed_task": + self.assertEqual(task["status"], TaskStatus.FAILED) + self.assertEqual(task["action"], "failed_test_task") + result.append(task) + + return result + + def test_requester_is_not_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + self.register_user("user", "pass", admin=False) + other_user_tok = self.login("user", "pass") + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks", + content={}, + access_token=other_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_scheduled_tasks(self) -> None: + """ + Test that endpoint returns scheduled tasks. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + + # make sure we got back all the scheduled tasks + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + self.assertEqual(len(found_tasks), 3) + + def test_filtering_scheduled_tasks(self) -> None: + """ + Test that filtering the scheduled tasks response via query params works as expected. + """ + # filter via job_status + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?job_status=active", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # only the active task should have been returned + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["status"], "active") + + # filter via action_name + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?action_name=test_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + + # only test_task should have been returned + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["action"], "test_task") + + # filter via max_timestamp + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?max_timestamp=0", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # none should have been returned + self.assertEqual(len(found_tasks), 0) + + # filter via resource id + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?resource_id=failed_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + scheduled_tasks = channel.json_body["scheduled_tasks"] + found_tasks = self.check_scheduled_tasks_response(scheduled_tasks) + + # only the task with the matching resource id should have been returned + self.assertEqual(len(found_tasks), 1) + self.assertEqual(found_tasks[0]["resource_id"], "failed_task") From f2ca2e31f7c5b1627554f85adead37212736bf5a Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 2 May 2025 06:11:48 -0400 Subject: [PATCH 284/691] Readme tweaks (#18218) --- README.rst | 12 +++++++----- changelog.d/18218.doc | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18218.doc diff --git a/README.rst b/README.rst index 77f861e788..8974990ed1 100644 --- a/README.rst +++ b/README.rst @@ -253,15 +253,17 @@ Alongside all that, join our developer community on Matrix: Copyright and Licensing ======================= -Copyright 2014-2017 OpenMarket Ltd -Copyright 2017 Vector Creations Ltd -Copyright 2017-2025 New Vector Ltd +| Copyright 2014-2017 OpenMarket Ltd +| Copyright 2017 Vector Creations Ltd +| Copyright 2017-2025 New Vector Ltd +| This software is dual-licensed by New Vector Ltd (Element). It can be used either: - + (1) for free under the terms of the GNU Affero General Public License (as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version); OR - + (2) under the terms of a paid-for Element Commercial License agreement between you and Element (the terms of which may vary depending on what you and Element have agreed to). + Unless required by applicable law or agreed to in writing, software distributed under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the specific language governing permissions and limitations under the Licenses. diff --git a/changelog.d/18218.doc b/changelog.d/18218.doc new file mode 100644 index 0000000000..f62da6a0b9 --- /dev/null +++ b/changelog.d/18218.doc @@ -0,0 +1 @@ +Improve formatting of the README file. From 74be5cfdbc2208f0b34d9ab75f99994bd8ed217d Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 2 May 2025 12:13:26 +0200 Subject: [PATCH 285/691] Do not auto-provision missing users & devices when delegating auth to MAS (#18181) Since MAS 0.13.0, the provisionning of devices and users is done synchronously and reliably enough that we don't need to auto-provision on the Synapse side anymore. It's important to remove this behaviour if we want to start caching token introspection results. --- changelog.d/18181.misc | 1 + synapse/api/auth/msc3861_delegated.py | 39 +++++++------------------ tests/handlers/test_oauth_delegation.py | 10 +++++++ 3 files changed, 22 insertions(+), 28 deletions(-) create mode 100644 changelog.d/18181.misc diff --git a/changelog.d/18181.misc b/changelog.d/18181.misc new file mode 100644 index 0000000000..d9ba2f1dd1 --- /dev/null +++ b/changelog.d/18181.misc @@ -0,0 +1 @@ +Stop auto-provisionning missing users & devices when delegating auth to Matrix Authentication Service. Requires MAS 0.13.0 or later. diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 9ded3366e3..e500a06afe 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -39,7 +39,6 @@ from synapse.api.errors import ( HttpResponseException, InvalidClientTokenError, OAuthInsufficientScopeError, - StoreError, SynapseError, UnrecognizedRequestError, ) @@ -512,7 +511,7 @@ class MSC3861DelegatedAuth(BaseAuth): raise InvalidClientTokenError("No scope in token granting user rights") # Match via the sub claim - sub: Optional[str] = introspection_result.get_sub() + sub = introspection_result.get_sub() if sub is None: raise InvalidClientTokenError( "Invalid sub claim in the introspection result" @@ -525,29 +524,20 @@ class MSC3861DelegatedAuth(BaseAuth): # If we could not find a user via the external_id, it either does not exist, # or the external_id was never recorded - # TODO: claim mapping should be configurable - username: Optional[str] = introspection_result.get_username() - if username is None or not isinstance(username, str): + username = introspection_result.get_username() + if username is None: raise AuthError( 500, "Invalid username claim in the introspection result", ) user_id = UserID(username, self._hostname) - # First try to find a user from the username claim + # Try to find a user from the username claim user_info = await self.store.get_user_by_id(user_id=user_id.to_string()) if user_info is None: - # If the user does not exist, we should create it on the fly - # TODO: we could use SCIM to provision users ahead of time and listen - # for SCIM SET events if those ever become standard: - # https://datatracker.ietf.org/doc/html/draft-hunt-scim-notify-00 - - # TODO: claim mapping should be configurable - # If present, use the name claim as the displayname - name: Optional[str] = introspection_result.get_name() - - await self.store.register_user( - user_id=user_id.to_string(), create_profile_with_displayname=name + raise AuthError( + 500, + "User not found", ) # And record the sub as external_id @@ -587,17 +577,10 @@ class MSC3861DelegatedAuth(BaseAuth): "Invalid device ID in introspection result", ) - # Create the device on the fly if it does not exist - try: - await self.store.get_device( - user_id=user_id.to_string(), device_id=device_id - ) - except StoreError: - await self.store.store_device( - user_id=user_id.to_string(), - device_id=device_id, - initial_device_display_name="OIDC-native client", - ) + # Make sure the device exists + await self.store.get_device( + user_id=user_id.to_string(), device_id=device_id + ) # TODO: there is a few things missing in the requester here, which still need # to be figured out, like: diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 034a1594d9..934bfee0bc 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -147,6 +147,16 @@ class MSC3861OAuthDelegation(HomeserverTestCase): return hs + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision the user and the device we use in the tests. + store = homeserver.get_datastores().main + self.get_success(store.register_user(USER_ID)) + self.get_success( + store.store_device(USER_ID, DEVICE, initial_device_display_name=None) + ) + def _assertParams(self) -> None: """Assert that the request parameters are correct.""" params = parse_qs(self.http_client.request.call_args[1]["data"].decode("utf-8")) From ea376126a0b7e3fbc0df6ac827eba87d98e479de Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 2 May 2025 04:14:31 -0700 Subject: [PATCH 286/691] Fix typo in doc for Scheduled Tasks Admin API (#18384) --- changelog.d/18384.doc | 1 + docs/admin_api/scheduled_tasks.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18384.doc diff --git a/changelog.d/18384.doc b/changelog.d/18384.doc new file mode 100644 index 0000000000..ebcd029639 --- /dev/null +++ b/changelog.d/18384.doc @@ -0,0 +1 @@ +Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md index 1708871a6d..b80da5083c 100644 --- a/docs/admin_api/scheduled_tasks.md +++ b/docs/admin_api/scheduled_tasks.md @@ -19,7 +19,7 @@ It returns a JSON body like the following: "id": "GSA124oegf1", "action": "shutdown_room", "status": "complete", - "timestamp": 23423523, + "timestamp_ms": 23423523, "resource_id": "!roomid", "result": "some result", "error": null From fd5d3d852df9dbbac13b406144be7ec5a807078d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 2 May 2025 12:16:14 +0100 Subject: [PATCH 287/691] Don't check the `at_hash` (access token hash) in OIDC ID Tokens if we don't use the access token (#18374) Co-authored-by: Eric Eastwood --- changelog.d/18374.misc | 1 + synapse/handlers/oidc.py | 29 ++++++++++++++++++++++-- tests/handlers/test_oidc.py | 44 +++++++++++++++++++++++++++++++++++++ tests/test_utils/oidc.py | 19 ++++++++++++++-- 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18374.misc diff --git a/changelog.d/18374.misc b/changelog.d/18374.misc new file mode 100644 index 0000000000..a8efca68d0 --- /dev/null +++ b/changelog.d/18374.misc @@ -0,0 +1 @@ +Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. \ No newline at end of file diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index c4cf0636a3..fb759172b3 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -586,6 +586,24 @@ class OidcProvider: or self._user_profile_method == "userinfo_endpoint" ) + @property + def _uses_access_token(self) -> bool: + """Return True if the `access_token` will be used during the login process. + + This is useful to determine whether the access token + returned by the identity provider, and + any related metadata (such as the `at_hash` field in + the ID token), should be validated. + """ + # Currently, Synapse only uses the access_token to fetch user metadata + # from the userinfo endpoint. Therefore we only have a single criteria + # to check right now but this may change in the future and this function + # should be updated if more usages are introduced. + # + # For example, if we start to use the access_token given to us by the + # IdP for more things, such as accessing Resource Server APIs. + return self._uses_userinfo + @property def issuer(self) -> str: """The issuer identifying this provider.""" @@ -957,9 +975,16 @@ class OidcProvider: "nonce": nonce, "client_id": self._client_auth.client_id, } - if "access_token" in token: + if self._uses_access_token and "access_token" in token: # If we got an `access_token`, there should be an `at_hash` claim - # in the `id_token` that we can check against. + # in the `id_token` that we can check against. Setting this + # instructs authlib to check the value of `at_hash` in the + # ID token. + # + # We only need to verify the access token if we actually make + # use of it. Which currently only happens when we need to fetch + # the user's information from the userinfo_endpoint. Thus, this + # check is also gated on self._uses_userinfo. claims_params["access_token"] = token["access_token"] claims_options = {"iss": {"values": [metadata["issuer"]]}} diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index a7cead83d0..e5f31d57ca 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -1029,6 +1029,50 @@ class OidcHandlerTestCase(HomeserverTestCase): args = parse_qs(kwargs["data"].decode("utf-8")) self.assertEqual(args["redirect_uri"], [TEST_REDIRECT_URI]) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "redirect_uri": TEST_REDIRECT_URI, + } + } + ) + def test_code_exchange_ignores_access_token(self) -> None: + """ + Code exchange completes successfully and doesn't validate the `at_hash` + (access token hash) field of an ID token when the access token isn't + going to be used. + + The access token won't be used in this test because Synapse (currently) + only needs it to fetch a user's metadata if it isn't included in the ID + token itself. + + Because we have included "openid" in the requested scopes for this IdP + (see `SCOPES`), user metadata is be included in the ID token. Thus the + access token isn't needed, and it's unnecessary for Synapse to validate + the access token. + + This is a regression test for a situation where an upstream identity + provider was providing an invalid `at_hash` value, which Synapse errored + on, yet Synapse wasn't using the access token for anything. + """ + # Exchange the code against the fake IdP. + userinfo = { + "sub": "foo", + "username": "foo", + "phone": "1234567", + } + with self.fake_server.id_token_override( + { + "at_hash": "invalid-hash", + } + ): + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # If no error was rendered, then we have success. + self.render_error.assert_not_called() + @override_config( { "oidc_config": { diff --git a/tests/test_utils/oidc.py b/tests/test_utils/oidc.py index 6c4be1c1f8..5bf5e5cb0c 100644 --- a/tests/test_utils/oidc.py +++ b/tests/test_utils/oidc.py @@ -20,7 +20,9 @@ # +import base64 import json +from hashlib import sha256 from typing import Any, ContextManager, Dict, List, Optional, Tuple from unittest.mock import Mock, patch from urllib.parse import parse_qs @@ -154,10 +156,23 @@ class FakeOidcServer: json_payload = json.dumps(payload) return jws.serialize_compact(protected, json_payload, self._key).decode("utf-8") - def generate_id_token(self, grant: FakeAuthorizationGrant) -> str: + def generate_id_token( + self, grant: FakeAuthorizationGrant, access_token: str + ) -> str: + # Generate a hash of the access token for the optional + # `at_hash` field in an ID Token. + # + # 3.1.3.6. ID Token, https://openid.net/specs/openid-connect-core-1_0.html#CodeIDToken + at_hash = ( + base64.urlsafe_b64encode(sha256(access_token.encode("ascii")).digest()[:16]) + .rstrip(b"=") + .decode("ascii") + ) + now = int(self._clock.time()) id_token = { **grant.userinfo, + "at_hash": at_hash, "iss": self.issuer, "aud": grant.client_id, "iat": now, @@ -243,7 +258,7 @@ class FakeOidcServer: } if "openid" in grant.scope: - token["id_token"] = self.generate_id_token(grant) + token["id_token"] = self.generate_id_token(grant, access_token) return dict(token) From d18edf67d6f444c8dfa6a46e8769bbfa8d22f57b Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 2 May 2025 14:07:23 +0200 Subject: [PATCH 288/691] Fix lint which broke in #18374 (#18385) https://github.com/element-hq/synapse/pull/18374 did not pass linting but was merged --- changelog.d/18385.misc | 1 + synapse/handlers/oidc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18385.misc diff --git a/changelog.d/18385.misc b/changelog.d/18385.misc new file mode 100644 index 0000000000..a8efca68d0 --- /dev/null +++ b/changelog.d/18385.misc @@ -0,0 +1 @@ +Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. \ No newline at end of file diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index fb759172b3..acf2d4bc8b 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -599,7 +599,7 @@ class OidcProvider: # from the userinfo endpoint. Therefore we only have a single criteria # to check right now but this may change in the future and this function # should be updated if more usages are introduced. - # + # # For example, if we start to use the access_token given to us by the # IdP for more things, such as accessing Resource Server APIs. return self._uses_userinfo From 411d239db47158cc14f94c94a86a5c713d783821 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 2 May 2025 06:04:01 -0700 Subject: [PATCH 289/691] Apply `should_drop_federated_event` to federation invites (#18330) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18330.misc | 1 + docs/modules/spam_checker_callbacks.md | 2 ++ synapse/federation/federation_server.py | 6 ++++++ 3 files changed, 9 insertions(+) create mode 100644 changelog.d/18330.misc diff --git a/changelog.d/18330.misc b/changelog.d/18330.misc new file mode 100644 index 0000000000..dcf341fa34 --- /dev/null +++ b/changelog.d/18330.misc @@ -0,0 +1 @@ +Apply `should_drop_federated_event` to federation invites. diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index c7f8606fd0..063099a127 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -353,6 +353,8 @@ callback returns `False`, Synapse falls through to the next one. The value of th callback that does not return `False` will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. +Note that this check is applied to federation invites as of Synapse v1.130.0. + ### `check_login_for_spam` diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index f9e97ea13e..2f2c78babc 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -701,6 +701,12 @@ class FederationServer(FederationBase): pdu = event_from_pdu_json(content, room_version) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) + if await self._spam_checker_module_callbacks.should_drop_federated_event(pdu): + logger.info( + "Federated event contains spam, dropping %s", + pdu.event_id, + ) + raise SynapseError(403, Codes.FORBIDDEN) try: pdu = await self._check_sigs_and_hash(room_version, pdu) except InvalidEventSignatureError as e: From b8146d4b03d89a9407125b5934bd7accbe0680e0 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 2 May 2025 15:37:58 +0200 Subject: [PATCH 290/691] Allow a few admin APIs used by MAS to run on workers (#18313) This should be reviewed commit by commit. It adds a few admin servlets that are used by MAS when in delegation mode to workers --------- Co-authored-by: Olivier 'reivilibre Co-authored-by: Devon Hudson Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18313.misc | 1 + docs/workers.md | 9 + synapse/app/generic_worker.py | 18 +- synapse/app/homeserver.py | 9 +- synapse/handlers/set_password.py | 18 +- synapse/rest/__init__.py | 1 - synapse/rest/admin/__init__.py | 29 +- synapse/rest/admin/devices.py | 26 +- .../storage/databases/main/end_to_end_keys.py | 78 +++--- .../storage/databases/main/registration.py | 260 +++++++++--------- 10 files changed, 249 insertions(+), 200 deletions(-) create mode 100644 changelog.d/18313.misc diff --git a/changelog.d/18313.misc b/changelog.d/18313.misc new file mode 100644 index 0000000000..febf3ac06e --- /dev/null +++ b/changelog.d/18313.misc @@ -0,0 +1 @@ +Allow a few admin APIs used by matrix-authentication-service to run on workers. diff --git a/docs/workers.md b/docs/workers.md index 2597e78217..45a00696f3 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -323,6 +323,15 @@ For multiple workers not handling the SSO endpoints properly, see [#7530](https://github.com/matrix-org/synapse/issues/7530) and [#9427](https://github.com/matrix-org/synapse/issues/9427). +Additionally, when MSC3861 is enabled (`experimental_features.msc3861.enabled` +set to `true`), the following endpoints can be handled by the worker: + + ^/_synapse/admin/v2/users/[^/]+$ + ^/_synapse/admin/v1/username_available$ + ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ + # Only the GET method: + ^/_synapse/admin/v1/users/[^/]+/devices$ + Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) with `client` and `federation` `resources` must be configured in the [`worker_listeners`](usage/configuration/config_documentation.md#worker_listeners) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index e4120ed424..f495d5b7e4 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -51,8 +51,7 @@ from synapse.http.server import JsonResource, OptionsResource from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource -from synapse.rest import ClientRestResource -from synapse.rest.admin import AdminRestResource, register_servlets_for_media_repo +from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree @@ -176,8 +175,13 @@ class GenericWorkerServer(HomeServer): def _listen_http(self, listener_config: ListenerConfig) -> None: assert listener_config.http_options is not None - # We always include a health resource. - resources: Dict[str, Resource] = {"/health": HealthResource()} + # We always include an admin resource that we populate with servlets as needed + admin_resource = JsonResource(self, canonical_json=False) + resources: Dict[str, Resource] = { + # We always include a health resource. + "/health": HealthResource(), + "/_synapse/admin": admin_resource, + } for res in listener_config.http_options.resources: for name in res.names: @@ -190,7 +194,7 @@ class GenericWorkerServer(HomeServer): resources.update(build_synapse_client_resource_tree(self)) resources["/.well-known"] = well_known_resource(self) - resources["/_synapse/admin"] = AdminRestResource(self) + admin.register_servlets(self, admin_resource) elif name == "federation": resources[FEDERATION_PREFIX] = TransportLayerServer(self) @@ -200,15 +204,13 @@ class GenericWorkerServer(HomeServer): # We need to serve the admin servlets for media on the # worker. - admin_resource = JsonResource(self, canonical_json=False) - register_servlets_for_media_repo(self, admin_resource) + admin.register_servlets_for_media_repo(self, admin_resource) resources.update( { MEDIA_R0_PREFIX: media_repo, MEDIA_V3_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, - "/_synapse/admin": admin_resource, } ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2a824e8457..6da2194cf7 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -54,6 +54,7 @@ from synapse.config.server import ListenerConfig, TCPListenerConfig from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import ( + JsonResource, OptionsResource, RootOptionsRedirectResource, StaticResource, @@ -61,8 +62,7 @@ from synapse.http.server import ( from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource -from synapse.rest import ClientRestResource -from synapse.rest.admin import AdminRestResource +from synapse.rest import ClientRestResource, admin from synapse.rest.health import HealthResource from synapse.rest.key.v2 import KeyResource from synapse.rest.synapse.client import build_synapse_client_resource_tree @@ -180,11 +180,14 @@ class SynapseHomeServer(HomeServer): if compress: client_resource = gz_wrap(client_resource) + admin_resource = JsonResource(self, canonical_json=False) + admin.register_servlets(self, admin_resource) + resources.update( { CLIENT_API_PREFIX: client_resource, "/.well-known": well_known_resource(self), - "/_synapse/admin": AdminRestResource(self), + "/_synapse/admin": admin_resource, **build_synapse_client_resource_tree(self), } ) diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 29cc03d71d..94301add9e 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -36,10 +36,17 @@ class SetPasswordHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() - # This can only be instantiated on the main process. - device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) - self._device_handler = device_handler + + # We don't need the device handler if password changing is disabled. + # This allows us to instantiate the SetPasswordHandler on the workers + # that have admin APIs for MAS + if self._auth_handler.can_change_password(): + # This can only be instantiated on the main process. + device_handler = hs.get_device_handler() + assert isinstance(device_handler, DeviceHandler) + self._device_handler: Optional[DeviceHandler] = device_handler + else: + self._device_handler = None async def set_password( self, @@ -51,6 +58,9 @@ class SetPasswordHandler: if not self._auth_handler.can_change_password(): raise SynapseError(403, "Password change disabled", errcode=Codes.FORBIDDEN) + # We should have this available only if password changing is enabled. + assert self._device_handler is not None + try: await self.store.user_set_password_hash(user_id, password_hash) except StoreError as e: diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 2f1ef84e26..00f108de08 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -187,7 +187,6 @@ class ClientRestResource(JsonResource): mutual_rooms.register_servlets, login_token_request.register_servlets, rendezvous.register_servlets, - auth_metadata.register_servlets, ]: continue diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index cf809d1a27..b1335fed66 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -39,7 +39,7 @@ from typing import TYPE_CHECKING, Optional, Tuple from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.handlers.pagination import PURGE_HISTORY_ACTION_NAME -from synapse.http.server import HttpServer, JsonResource +from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin @@ -51,6 +51,7 @@ from synapse.rest.admin.background_updates import ( from synapse.rest.admin.devices import ( DeleteDevicesRestServlet, DeviceRestServlet, + DevicesGetRestServlet, DevicesRestServlet, ) from synapse.rest.admin.event_reports import ( @@ -264,14 +265,6 @@ class PurgeHistoryStatusRestServlet(RestServlet): ######################################################################################## -class AdminRestResource(JsonResource): - """The REST resource which gets mounted at /_synapse/admin""" - - def __init__(self, hs: "HomeServer"): - JsonResource.__init__(self, hs, canonical_json=False) - register_servlets(hs, self) - - def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: """ Register all the admin servlets. @@ -280,6 +273,10 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: # Admin servlets below may not work on workers. if hs.config.worker.worker_app is not None: + # Some admin servlets can be mounted on workers when MSC3861 is enabled. + if hs.config.experimental.msc3861.enabled: + register_servlets_for_msc3861_delegation(hs, http_server) + return register_servlets_for_client_rest_resource(hs, http_server) @@ -367,4 +364,16 @@ def register_servlets_for_client_rest_resource( ListMediaInRoom(hs).register(http_server) # don't add more things here: new servlets should only be exposed on - # /_synapse/admin so should not go here. Instead register them in AdminRestResource. + # /_synapse/admin so should not go here. Instead register them in register_servlets. + + +def register_servlets_for_msc3861_delegation( + hs: "HomeServer", http_server: HttpServer +) -> None: + """Register servlets needed by MAS when MSC3861 is enabled""" + assert hs.config.experimental.msc3861.enabled + + UserRestServletV2(hs).register(http_server) + UsernameAvailableRestServlet(hs).register(http_server) + UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) + DevicesGetRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 449b066923..125ed8c491 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -113,18 +113,19 @@ class DeviceRestServlet(RestServlet): return HTTPStatus.OK, {} -class DevicesRestServlet(RestServlet): +class DevicesGetRestServlet(RestServlet): """ Retrieve the given user's devices + + This can be mounted on workers as it is read-only, as opposed + to `DevicesRestServlet`. """ PATTERNS = admin_patterns("/users/(?P[^/]*)/devices$", "v2") def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_worker_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine @@ -141,9 +142,24 @@ class DevicesRestServlet(RestServlet): if u is None: raise NotFoundError("Unknown user") - devices = await self.device_handler.get_devices_by_user(target_user.to_string()) + devices = await self.device_worker_handler.get_devices_by_user( + target_user.to_string() + ) return HTTPStatus.OK, {"devices": devices, "total": len(devices)} + +class DevicesRestServlet(DevicesGetRestServlet): + """ + Retrieve the given user's devices + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/devices$", "v2") + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + assert isinstance(self.device_worker_handler, DeviceHandler) + self.device_handler = self.device_worker_handler + async def on_POST( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index b4c7069958..341e7014d6 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -1501,6 +1501,45 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker "delete_old_otks_for_next_user_batch", impl ) + async def allow_master_cross_signing_key_replacement_without_uia( + self, user_id: str, duration_ms: int + ) -> Optional[int]: + """Mark this user's latest master key as being replaceable without UIA. + + Said replacement will only be permitted for a short time after calling this + function. That time period is controlled by the duration argument. + + Returns: + None, if there is no such key. + Otherwise, the timestamp before which replacement is allowed without UIA. + """ + timestamp = self._clock.time_msec() + duration_ms + + def impl(txn: LoggingTransaction) -> Optional[int]: + txn.execute( + """ + UPDATE e2e_cross_signing_keys + SET updatable_without_uia_before_ms = ? + WHERE stream_id = ( + SELECT stream_id + FROM e2e_cross_signing_keys + WHERE user_id = ? AND keytype = 'master' + ORDER BY stream_id DESC + LIMIT 1 + ) + """, + (timestamp, user_id), + ) + if txn.rowcount == 0: + return None + + return timestamp + + return await self.db_pool.runInteraction( + "allow_master_cross_signing_key_replacement_without_uia", + impl, + ) + class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): def __init__( @@ -1755,42 +1794,3 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): ], desc="add_e2e_signing_key", ) - - async def allow_master_cross_signing_key_replacement_without_uia( - self, user_id: str, duration_ms: int - ) -> Optional[int]: - """Mark this user's latest master key as being replaceable without UIA. - - Said replacement will only be permitted for a short time after calling this - function. That time period is controlled by the duration argument. - - Returns: - None, if there is no such key. - Otherwise, the timestamp before which replacement is allowed without UIA. - """ - timestamp = self._clock.time_msec() + duration_ms - - def impl(txn: LoggingTransaction) -> Optional[int]: - txn.execute( - """ - UPDATE e2e_cross_signing_keys - SET updatable_without_uia_before_ms = ? - WHERE stream_id = ( - SELECT stream_id - FROM e2e_cross_signing_keys - WHERE user_id = ? AND keytype = 'master' - ORDER BY stream_id DESC - LIMIT 1 - ) - """, - (timestamp, user_id), - ) - if txn.rowcount == 0: - return None - - return timestamp - - return await self.db_pool.runInteraction( - "allow_master_cross_signing_key_replacement_without_uia", - impl, - ) diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index c43f31353b..1aeae951c5 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2105,6 +2105,136 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): func=is_user_approved_txn, ) + async def set_user_deactivated_status( + self, user_id: str, deactivated: bool + ) -> None: + """Set the `deactivated` property for the provided user to the provided value. + + Args: + user_id: The ID of the user to set the status for. + deactivated: The value to set for `deactivated`. + """ + + await self.db_pool.runInteraction( + "set_user_deactivated_status", + self.set_user_deactivated_status_txn, + user_id, + deactivated, + ) + + def set_user_deactivated_status_txn( + self, txn: LoggingTransaction, user_id: str, deactivated: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"deactivated": 1 if deactivated else 0}, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_deactivated_status, (user_id,) + ) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + self._invalidate_cache_and_stream(txn, self.is_guest, (user_id,)) + + async def set_user_suspended_status(self, user_id: str, suspended: bool) -> None: + """ + Set whether the user's account is suspended in the `users` table. + + Args: + user_id: The user ID of the user in question + suspended: True if the user is suspended, false if not + """ + await self.db_pool.runInteraction( + "set_user_suspended_status", + self.set_user_suspended_status_txn, + user_id, + suspended, + ) + + def set_user_suspended_status_txn( + self, txn: LoggingTransaction, user_id: str, suspended: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"suspended": suspended}, + ) + self._invalidate_cache_and_stream( + txn, self.get_user_suspended_status, (user_id,) + ) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + + async def set_user_locked_status(self, user_id: str, locked: bool) -> None: + """Set the `locked` property for the provided user to the provided value. + + Args: + user_id: The ID of the user to set the status for. + locked: The value to set for `locked`. + """ + + await self.db_pool.runInteraction( + "set_user_locked_status", + self.set_user_locked_status_txn, + user_id, + locked, + ) + + def set_user_locked_status_txn( + self, txn: LoggingTransaction, user_id: str, locked: bool + ) -> None: + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"locked": locked}, + ) + self._invalidate_cache_and_stream(txn, self.get_user_locked_status, (user_id,)) + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + + async def update_user_approval_status( + self, user_id: UserID, approved: bool + ) -> None: + """Set the user's 'approved' flag to the given value. + + The boolean will be turned into an int (in update_user_approval_status_txn) + because the column is a smallint. + + Args: + user_id: the user to update the flag for. + approved: the value to set the flag to. + """ + await self.db_pool.runInteraction( + "update_user_approval_status", + self.update_user_approval_status_txn, + user_id.to_string(), + approved, + ) + + def update_user_approval_status_txn( + self, txn: LoggingTransaction, user_id: str, approved: bool + ) -> None: + """Set the user's 'approved' flag to the given value. + + The boolean is turned into an int because the column is a smallint. + + Args: + txn: the current database transaction. + user_id: the user to update the flag for. + approved: the value to set the flag to. + """ + self.db_pool.simple_update_one_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + updatevalues={"approved": approved}, + ) + + # Invalidate the caches of methods that read the value of the 'approved' flag. + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) + class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): def __init__( @@ -2217,117 +2347,6 @@ class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): return nb_processed - async def set_user_deactivated_status( - self, user_id: str, deactivated: bool - ) -> None: - """Set the `deactivated` property for the provided user to the provided value. - - Args: - user_id: The ID of the user to set the status for. - deactivated: The value to set for `deactivated`. - """ - - await self.db_pool.runInteraction( - "set_user_deactivated_status", - self.set_user_deactivated_status_txn, - user_id, - deactivated, - ) - - def set_user_deactivated_status_txn( - self, txn: LoggingTransaction, user_id: str, deactivated: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"deactivated": 1 if deactivated else 0}, - ) - self._invalidate_cache_and_stream( - txn, self.get_user_deactivated_status, (user_id,) - ) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - txn.call_after(self.is_guest.invalidate, (user_id,)) - - async def set_user_suspended_status(self, user_id: str, suspended: bool) -> None: - """ - Set whether the user's account is suspended in the `users` table. - - Args: - user_id: The user ID of the user in question - suspended: True if the user is suspended, false if not - """ - await self.db_pool.runInteraction( - "set_user_suspended_status", - self.set_user_suspended_status_txn, - user_id, - suspended, - ) - - def set_user_suspended_status_txn( - self, txn: LoggingTransaction, user_id: str, suspended: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"suspended": suspended}, - ) - self._invalidate_cache_and_stream( - txn, self.get_user_suspended_status, (user_id,) - ) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - async def set_user_locked_status(self, user_id: str, locked: bool) -> None: - """Set the `locked` property for the provided user to the provided value. - - Args: - user_id: The ID of the user to set the status for. - locked: The value to set for `locked`. - """ - - await self.db_pool.runInteraction( - "set_user_locked_status", - self.set_user_locked_status_txn, - user_id, - locked, - ) - - def set_user_locked_status_txn( - self, txn: LoggingTransaction, user_id: str, locked: bool - ) -> None: - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"locked": locked}, - ) - self._invalidate_cache_and_stream(txn, self.get_user_locked_status, (user_id,)) - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - - def update_user_approval_status_txn( - self, txn: LoggingTransaction, user_id: str, approved: bool - ) -> None: - """Set the user's 'approved' flag to the given value. - - The boolean is turned into an int because the column is a smallint. - - Args: - txn: the current database transaction. - user_id: the user to update the flag for. - approved: the value to set the flag to. - """ - self.db_pool.simple_update_one_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - updatevalues={"approved": approved}, - ) - - # Invalidate the caches of methods that read the value of the 'approved' flag. - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) - class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): def __init__( @@ -2956,25 +2975,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): start_or_continue_validation_session_txn, ) - async def update_user_approval_status( - self, user_id: UserID, approved: bool - ) -> None: - """Set the user's 'approved' flag to the given value. - - The boolean will be turned into an int (in update_user_approval_status_txn) - because the column is a smallint. - - Args: - user_id: the user to update the flag for. - approved: the value to set the flag to. - """ - await self.db_pool.runInteraction( - "update_user_approval_status", - self.update_user_approval_status_txn, - user_id.to_string(), - approved, - ) - @wrap_as_background_process("delete_expired_login_tokens") async def _delete_expired_login_tokens(self) -> None: """Remove login tokens with expiry dates that have passed.""" From fe8bb620de8e5830328c6d23127657560f449af0 Mon Sep 17 00:00:00 2001 From: Will Lewis <1543626+wrjlewis@users.noreply.github.com> Date: Fri, 2 May 2025 15:38:02 +0100 Subject: [PATCH 291/691] Add the ability to exclude remote users in user directory search results (#18300) This change adds a new configuration `user_directory.exclude_remote_users`, which defaults to False. When set to True, remote users will not appear in user directory search results. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18300.feature | 1 + .../configuration/config_documentation.md | 2 + synapse/config/user_directory.py | 3 + synapse/handlers/user_directory.py | 3 + .../storage/databases/main/user_directory.py | 18 ++++-- tests/handlers/test_user_directory.py | 61 +++++++++++++++++++ 6 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18300.feature diff --git a/changelog.d/18300.feature b/changelog.d/18300.feature new file mode 100644 index 0000000000..92bea77556 --- /dev/null +++ b/changelog.d/18300.feature @@ -0,0 +1 @@ +Add config option `user_directory.exclude_remote_users` which, when enabled, excludes remote users from user directory search results. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 19dc9dd356..5351bef83a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4095,6 +4095,7 @@ This option has the following sub-options: * `prefer_local_users`: Defines whether to prefer local users in search query results. If set to true, local users are more likely to appear above remote users when searching the user directory. Defaults to false. +* `exclude_remote_users`: If set to true, the search will only return local users. Defaults to false. * `show_locked_users`: Defines whether to show locked users in search query results. Defaults to false. Example configuration: @@ -4103,6 +4104,7 @@ user_directory: enabled: false search_all_users: true prefer_local_users: true + exclude_remote_users: false show_locked_users: true ``` --- diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py index c67796906f..fe4e2dc65c 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py @@ -38,6 +38,9 @@ class UserDirectoryConfig(Config): self.user_directory_search_all_users = user_directory_config.get( "search_all_users", False ) + self.user_directory_exclude_remote_users = user_directory_config.get( + "exclude_remote_users", False + ) self.user_directory_search_prefer_local_users = user_directory_config.get( "prefer_local_users", False ) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index f88d39b38f..33edef5f14 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -108,6 +108,9 @@ class UserDirectoryHandler(StateDeltasHandler): self.is_mine_id = hs.is_mine_id self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users + self.exclude_remote_users = ( + hs.config.userdirectory.user_directory_exclude_remote_users + ) self.show_locked_users = hs.config.userdirectory.show_locked_users self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._hs = hs diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index d6cd0774a8..391f0dd638 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -1037,11 +1037,11 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): } """ + join_args: Tuple[str, ...] = (user_id,) + if self.hs.config.userdirectory.user_directory_search_all_users: - join_args = (user_id,) where_clause = "user_id != ?" else: - join_args = (user_id,) where_clause = """ ( EXISTS (select 1 from users_in_public_rooms WHERE user_id = t.user_id) @@ -1055,6 +1055,14 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): if not show_locked_users: where_clause += " AND (u.locked IS NULL OR u.locked = FALSE)" + # Adjust the JOIN type based on the exclude_remote_users flag (the users + # table only contains local users so an inner join is a good way to + # to exclude remote users) + if self.hs.config.userdirectory.user_directory_exclude_remote_users: + join_type = "JOIN" + else: + join_type = "LEFT JOIN" + # We allow manipulating the ranking algorithm by injecting statements # based on config options. additional_ordering_statements = [] @@ -1086,7 +1094,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): SELECT d.user_id AS user_id, display_name, avatar_url FROM matching_users as t INNER JOIN user_directory AS d USING (user_id) - LEFT JOIN users AS u ON t.user_id = u.name + %(join_type)s users AS u ON t.user_id = u.name WHERE %(where_clause)s ORDER BY @@ -1115,6 +1123,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): """ % { "where_clause": where_clause, "order_case_statements": " ".join(additional_ordering_statements), + "join_type": join_type, } args = ( (full_query,) @@ -1142,7 +1151,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): SELECT d.user_id AS user_id, display_name, avatar_url FROM user_directory_search as t INNER JOIN user_directory AS d USING (user_id) - LEFT JOIN users AS u ON t.user_id = u.name + %(join_type)s users AS u ON t.user_id = u.name WHERE %(where_clause)s AND value MATCH ? @@ -1155,6 +1164,7 @@ class UserDirectoryStore(UserDirectoryBackgroundUpdateStore): """ % { "where_clause": where_clause, "order_statements": " ".join(additional_ordering_statements), + "join_type": join_type, } args = join_args + (search_query,) + ordering_arguments + (limit + 1,) else: diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index a75095a79f..a9e9d7d7ea 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -992,6 +992,67 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): [self.assertIn(user, local_users) for user in received_user_id_ordering[:3]] [self.assertIn(user, remote_users) for user in received_user_id_ordering[3:]] + @override_config( + { + "user_directory": { + "enabled": True, + "search_all_users": True, + "exclude_remote_users": True, + } + } + ) + def test_exclude_remote_users(self) -> None: + """Tests that only local users are returned when + user_directory.exclude_remote_users is True. + """ + + # Create a room and few users to test the directory with + searching_user = self.register_user("searcher", "password") + searching_user_tok = self.login("searcher", "password") + + room_id = self.helper.create_room_as( + searching_user, + room_version=RoomVersions.V1.identifier, + tok=searching_user_tok, + ) + + # Create a few local users and join them to the room + local_user_1 = self.register_user("user_xxxxx", "password") + local_user_2 = self.register_user("user_bbbbb", "password") + local_user_3 = self.register_user("user_zzzzz", "password") + + self._add_user_to_room(room_id, RoomVersions.V1, local_user_1) + self._add_user_to_room(room_id, RoomVersions.V1, local_user_2) + self._add_user_to_room(room_id, RoomVersions.V1, local_user_3) + + # Create a few "remote" users and join them to the room + remote_user_1 = "@user_aaaaa:remote_server" + remote_user_2 = "@user_yyyyy:remote_server" + remote_user_3 = "@user_ccccc:remote_server" + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_1) + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_2) + self._add_user_to_room(room_id, RoomVersions.V1, remote_user_3) + + local_users = [local_user_1, local_user_2, local_user_3] + remote_users = [remote_user_1, remote_user_2, remote_user_3] + + # The local searching user searches for the term "user", which other users have + # in their user id + results = self.get_success( + self.handler.search_users(searching_user, "user", 20) + )["results"] + received_user_ids = [result["user_id"] for result in results] + + for user in local_users: + self.assertIn( + user, received_user_ids, f"Local user {user} not found in results" + ) + + for user in remote_users: + self.assertNotIn( + user, received_user_ids, f"Remote user {user} should not be in results" + ) + def _add_user_to_room( self, room_id: str, From 9f9eb563339079ee5ce082fcd63d0ab5d849b7ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 5 May 2025 10:08:50 +0100 Subject: [PATCH 292/691] Return specific error code when email / phone not supported (#17578) Implements https://github.com/matrix-org/matrix-spec-proposals/pull/4178 If this would need tests, could you give some idea of what tests would be needed and how best to add them? ### Pull Request Checklist * [ ] Pull request is based on the develop branch * [ ] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/17578.misc | 1 + synapse/api/errors.py | 1 + synapse/rest/client/account.py | 6 +++++- synapse/util/msisdn.py | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/17578.misc diff --git a/changelog.d/17578.misc b/changelog.d/17578.misc new file mode 100644 index 0000000000..7bf69576cd --- /dev/null +++ b/changelog.d/17578.misc @@ -0,0 +1 @@ +Return specific error code when adding an email address / phone number to account is not supported (MSC4178). diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5dd6e84289..edd2073384 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -70,6 +70,7 @@ class Codes(str, Enum): THREEPID_NOT_FOUND = "M_THREEPID_NOT_FOUND" THREEPID_DENIED = "M_THREEPID_DENIED" INVALID_USERNAME = "M_INVALID_USERNAME" + THREEPID_MEDIUM_NOT_SUPPORTED = "M_THREEPID_MEDIUM_NOT_SUPPORTED" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 59dbad3582..7d6c0afd9a 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -350,6 +350,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): raise SynapseError( 400, "Adding an email to your account is disabled on this server", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) body = parse_and_validate_json_object_from_request( @@ -456,6 +457,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): raise SynapseError( 400, "Adding phone numbers to user account is not supported by this homeserver", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) ret = await self.identity_handler.requestMsisdnToken( @@ -498,7 +500,9 @@ class AddThreepidEmailSubmitTokenServlet(RestServlet): "Adding emails have been disabled due to lack of an email config" ) raise SynapseError( - 400, "Adding an email to your account is disabled on this server" + 400, + "Adding an email to your account is disabled on this server", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, ) sid = parse_string(request, "sid", required=True) diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py index b6a784f0bc..dce8da5e18 100644 --- a/synapse/util/msisdn.py +++ b/synapse/util/msisdn.py @@ -21,7 +21,7 @@ import phonenumbers -from synapse.api.errors import SynapseError +from synapse.api.errors import Codes, SynapseError def phone_number_to_msisdn(country: str, number: str) -> str: @@ -45,7 +45,7 @@ def phone_number_to_msisdn(country: str, number: str) -> str: try: phoneNumber = phonenumbers.parse(number, country) except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") + raise SynapseError(400, "Unable to parse phone number", Codes.INVALID_PARAM) return phonenumbers.format_number(phoneNumber, phonenumbers.PhoneNumberFormat.E164)[ 1: ] From c9adbc6a1ce6039b1c04ae3298e463a3e3b25c38 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 5 May 2025 13:09:39 +0300 Subject: [PATCH 293/691] make tests tolerant to authlib 1.5.2 error messages (#18390) authlib 1.5.2 now single-quotes error messages in the claims, causing three tests to fail. Replace the comparison with a regex that accepts both single or double quotes. This succeeds the tests with both authlib 1.5.1 and 1.5.2. See https://github.com/NixOS/nixpkgs/pull/402797 for context. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18390.misc | 1 + tests/rest/client/test_login.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 changelog.d/18390.misc diff --git a/changelog.d/18390.misc b/changelog.d/18390.misc new file mode 100644 index 0000000000..e9a08dcfbf --- /dev/null +++ b/changelog.d/18390.misc @@ -0,0 +1 @@ +Fixed test failures when using authlib 1.5.2. diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index d7148917d0..c5c6604667 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -1262,18 +1262,18 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "iss": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "iss"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']iss[\"']$", ) # Not providing an issuer. channel = self.jwt_login({"sub": "kermit"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: missing_claim: Missing "iss" claim', + r"^JWT validation failed: missing_claim: Missing [\"']iss[\"'] claim$", ) def test_login_iss_no_config(self) -> None: @@ -1294,18 +1294,18 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "aud": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "aud"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']aud[\"']$", ) # Not providing an audience. channel = self.jwt_login({"sub": "kermit"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: missing_claim: Missing "aud" claim', + r"^JWT validation failed: missing_claim: Missing [\"']aud[\"'] claim$", ) def test_login_aud_no_config(self) -> None: @@ -1313,9 +1313,9 @@ class JWTTestCase(unittest.HomeserverTestCase): channel = self.jwt_login({"sub": "kermit", "aud": "invalid"}) self.assertEqual(channel.code, 403, msg=channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - self.assertEqual( + self.assertRegex( channel.json_body["error"], - 'JWT validation failed: invalid_claim: Invalid claim "aud"', + r"^JWT validation failed: invalid_claim: Invalid claim [\"']aud[\"']$", ) def test_login_default_sub(self) -> None: From d0873d549a8cce720a7842919126d78b4d9d030d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 6 May 2025 11:04:31 +0100 Subject: [PATCH 294/691] Ensure the url previewer also hashes and quarantines media (#18297) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18297.misc | 1 + synapse/media/media_repository.py | 1 - synapse/media/url_previewer.py | 17 ++++++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18297.misc diff --git a/changelog.d/18297.misc b/changelog.d/18297.misc new file mode 100644 index 0000000000..5032d48174 --- /dev/null +++ b/changelog.d/18297.misc @@ -0,0 +1 @@ +Apply file hashing and existing quarantines to media downloaded for URL previews. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 859b30e029..18c5a8ecec 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -378,7 +378,6 @@ class MediaRepository: media_length=content_length, user_id=auth_user, sha256=sha256, - # TODO: Better name? quarantined_by="system" if should_quarantine else None, ) diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 2e65a04789..8ef2b3f0c0 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -41,7 +41,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.http.client import SimpleHttpClient from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.media._base import FileInfo, get_filename_from_headers -from synapse.media.media_storage import MediaStorage +from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter from synapse.media.oembed import OEmbedProvider from synapse.media.preview_html import decode_body, parse_html_to_open_graph from synapse.metrics.background_process_metrics import run_as_background_process @@ -593,17 +593,26 @@ class UrlPreviewer: file_info = FileInfo(server_name=None, file_id=file_id, url_cache=True) async with self.media_storage.store_into_file(file_info) as (f, fname): + sha256writer = SHA256TransparentIOWriter(f) if url.startswith("data:"): if not allow_data_urls: raise SynapseError( 500, "Previewing of data: URLs is forbidden", Codes.UNKNOWN ) - download_result = await self._parse_data_url(url, f) + download_result = await self._parse_data_url(url, sha256writer.wrap()) else: - download_result = await self._download_url(url, f) + download_result = await self._download_url(url, sha256writer.wrap()) try: + sha256 = sha256writer.hexdigest() + should_quarantine = await self.store.get_is_hash_quarantined(sha256) + + if should_quarantine: + logger.warn( + "Media has been automatically quarantined as it matched existing quarantined media" + ) + time_now_ms = self.clock.time_msec() await self.store.store_local_media( @@ -614,6 +623,8 @@ class UrlPreviewer: media_length=download_result.length, user_id=user, url_cache=url, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, ) except Exception as e: From 9a62b2d47ae360a3c6d43710c3ab17d41563f359 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 6 May 2025 12:22:27 +0100 Subject: [PATCH 295/691] 1.129.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 71da6c29f5..f04c7ef026 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.129.0 (2025-05-06) + +No significant changes since 1.129.0rc2. + + + + # Synapse 1.129.0rc2 (2025-04-30) Synapse 1.129.0rc1 was never formally released due to regressions discovered during the release process. 1.129.0rc2 fixes those regressions by reverting the affected PRs. diff --git a/debian/changelog b/debian/changelog index c4bc3cc966..2c1cb20624 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.129.0) stable; urgency=medium + + * New Synapse release 1.129.0. + + -- Synapse Packaging team Tue, 06 May 2025 12:22:11 +0100 + matrix-synapse-py3 (1.129.0~rc2) stable; urgency=medium * New synapse release 1.129.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 3005f71a32..24ae0db05c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.129.0rc2" +version = "1.129.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From ae877aa101796a0cd57c3637a875140ddb25ed51 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 7 May 2025 15:07:58 +0000 Subject: [PATCH 296/691] Convert Sliding Sync tests to use higher-level `compute_interested_rooms` (#18399) Spawning from https://github.com/element-hq/synapse/pull/18375#discussion_r2071768635, This updates some sliding sync tests to use a higher level function in order to move test coverage to cover both fallback & new tables. Important when https://github.com/element-hq/synapse/pull/18375 is merged. In other words, adjust tests to target `compute_interested_room(...)` (relevant to both new and fallback path) instead of the lower level `get_room_membership_for_user_at_to_token(...)` that only applies to the fallback path. ### Dev notes ``` SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.handlers.test_sliding_sync.ComputeInterestedRoomsTestCase_new ``` ``` SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.rest.client.sliding_sync ``` ``` SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.handlers.test_sliding_sync.ComputeInterestedRoomsTestCase_new.test_display_name_changes_leave_after_token_range ``` ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/18399.misc | 1 + synapse/handlers/sliding_sync/room_lists.py | 122 +- synapse/storage/_base.py | 10 +- synapse/storage/databases/main/cache.py | 23 +- synapse/storage/databases/main/roommember.py | 135 +- synapse/storage/databases/main/stream.py | 2 + tests/handlers/test_sliding_sync.py | 1386 +++++++++++++----- 7 files changed, 1240 insertions(+), 439 deletions(-) create mode 100644 changelog.d/18399.misc diff --git a/changelog.d/18399.misc b/changelog.d/18399.misc new file mode 100644 index 0000000000..847dc9a2b1 --- /dev/null +++ b/changelog.d/18399.misc @@ -0,0 +1 @@ +Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Simplified Sliding Sync room list tests to cover both new and fallback logic paths. diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index a1730b7e05..7e3cf539df 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -244,14 +244,47 @@ class SlidingSyncRoomLists: # Note: this won't include rooms the user has left themselves. We add back # `newly_left` rooms below. This is more efficient than fetching all rooms and # then filtering out the old left rooms. - room_membership_for_user_map = await self.store.get_sliding_sync_rooms_for_user( - user_id + room_membership_for_user_map = ( + await self.store.get_sliding_sync_rooms_for_user_from_membership_snapshots( + user_id + ) ) + # To play nice with the rewind logic below, we need to go fetch the rooms the + # user has left themselves but only if it changed after the `to_token`. + # + # If a leave happens *after* the token range, we may have still been joined (or + # any non-self-leave which is relevant to sync) to the room before so we need to + # include it in the list of potentially relevant rooms and apply our rewind + # logic (outside of this function) to see if it's actually relevant. + # + # We do this separately from + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` as those results + # are cached and the `to_token` isn't very cache friendly (people are constantly + # requesting with new tokens) so we separate it out here. + self_leave_room_membership_for_user_map = ( + await self.store.get_sliding_sync_self_leave_rooms_after_to_token( + user_id, to_token + ) + ) + if self_leave_room_membership_for_user_map: + # FIXME: It would be nice to avoid this copy but since + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it + # can't return a mutable value like a `dict`. We make the copy to get a + # mutable dict that we can change. We try to only make a copy when necessary + # (if we actually need to change something) as in most cases, the logic + # doesn't need to run. + room_membership_for_user_map = dict(room_membership_for_user_map) + room_membership_for_user_map.update(self_leave_room_membership_for_user_map) # Remove invites from ignored users ignored_users = await self.store.ignored_users(user_id) if ignored_users: - # TODO: It would be nice to avoid these copies + # FIXME: It would be nice to avoid this copy but since + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it + # can't return a mutable value like a `dict`. We make the copy to get a + # mutable dict that we can change. We try to only make a copy when necessary + # (if we actually need to change something) as in most cases, the logic + # doesn't need to run. room_membership_for_user_map = dict(room_membership_for_user_map) # Make a copy so we don't run into an error: `dictionary changed size during # iteration`, when we remove items @@ -263,11 +296,23 @@ class SlidingSyncRoomLists: ): room_membership_for_user_map.pop(room_id, None) + ( + newly_joined_room_ids, + newly_left_room_map, + ) = await self._get_newly_joined_and_left_rooms( + user_id, from_token=from_token, to_token=to_token + ) + changes = await self._get_rewind_changes_to_current_membership_to_token( sync_config.user, room_membership_for_user_map, to_token=to_token ) if changes: - # TODO: It would be nice to avoid these copies + # FIXME: It would be nice to avoid this copy but since + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it + # can't return a mutable value like a `dict`. We make the copy to get a + # mutable dict that we can change. We try to only make a copy when necessary + # (if we actually need to change something) as in most cases, the logic + # doesn't need to run. room_membership_for_user_map = dict(room_membership_for_user_map) for room_id, change in changes.items(): if change is None: @@ -278,7 +323,7 @@ class SlidingSyncRoomLists: existing_room = room_membership_for_user_map.get(room_id) if existing_room is not None: # Update room membership events to the point in time of the `to_token` - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=change.sender, membership=change.membership, @@ -290,18 +335,18 @@ class SlidingSyncRoomLists: room_type=existing_room.room_type, is_encrypted=existing_room.is_encrypted, ) - - ( - newly_joined_room_ids, - newly_left_room_map, - ) = await self._get_newly_joined_and_left_rooms( - user_id, from_token=from_token, to_token=to_token - ) - dm_room_ids = await self._get_dm_rooms_for_user(user_id) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) # Add back `newly_left` rooms (rooms left in the from -> to token range). # - # We do this because `get_sliding_sync_rooms_for_user(...)` doesn't include + # We do this because `get_sliding_sync_rooms_for_user_from_membership_snapshots(...)` doesn't include # rooms that the user left themselves as it's more efficient to add them back # here than to fetch all rooms and then filter out the old left rooms. The user # only leaves a room once in a blue moon so this barely needs to run. @@ -310,7 +355,12 @@ class SlidingSyncRoomLists: newly_left_room_map.keys() - room_membership_for_user_map.keys() ) if missing_newly_left_rooms: - # TODO: It would be nice to avoid these copies + # FIXME: It would be nice to avoid this copy but since + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it + # can't return a mutable value like a `dict`. We make the copy to get a + # mutable dict that we can change. We try to only make a copy when necessary + # (if we actually need to change something) as in most cases, the logic + # doesn't need to run. room_membership_for_user_map = dict(room_membership_for_user_map) for room_id in missing_newly_left_rooms: newly_left_room_for_user = newly_left_room_map[room_id] @@ -327,14 +377,21 @@ class SlidingSyncRoomLists: # If the membership exists, it's just a normal user left the room on # their own if newly_left_room_for_user_sliding_sync is not None: - room_membership_for_user_map[room_id] = ( - newly_left_room_for_user_sliding_sync - ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=newly_left_room_for_user_sliding_sync, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = ( + newly_left_room_for_user_sliding_sync + ) + else: + room_membership_for_user_map.pop(room_id, None) change = changes.get(room_id) if change is not None: # Update room membership events to the point in time of the `to_token` - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=change.sender, membership=change.membership, @@ -346,6 +403,14 @@ class SlidingSyncRoomLists: room_type=newly_left_room_for_user_sliding_sync.room_type, is_encrypted=newly_left_room_for_user_sliding_sync.is_encrypted, ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) # If we are `newly_left` from the room but can't find any membership, # then we have been "state reset" out of the room @@ -367,7 +432,7 @@ class SlidingSyncRoomLists: newly_left_room_for_user.event_pos.to_room_stream_token(), ) - room_membership_for_user_map[room_id] = RoomsForUserSlidingSync( + room_for_user = RoomsForUserSlidingSync( room_id=room_id, sender=newly_left_room_for_user.sender, membership=newly_left_room_for_user.membership, @@ -378,6 +443,16 @@ class SlidingSyncRoomLists: room_type=room_type, is_encrypted=is_encrypted, ) + if filter_membership_for_sync( + user_id=user_id, + room_membership_for_user=room_for_user, + newly_left=room_id in newly_left_room_map, + ): + room_membership_for_user_map[room_id] = room_for_user + else: + room_membership_for_user_map.pop(room_id, None) + + dm_room_ids = await self._get_dm_rooms_for_user(user_id) if sync_config.lists: sync_room_map = room_membership_for_user_map @@ -493,7 +568,12 @@ class SlidingSyncRoomLists: if sync_config.room_subscriptions: with start_active_span("assemble_room_subscriptions"): - # TODO: It would be nice to avoid these copies + # FIXME: It would be nice to avoid this copy but since + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it + # can't return a mutable value like a `dict`. We make the copy to get a + # mutable dict that we can change. We try to only make a copy when necessary + # (if we actually need to change something) as in most cases, the logic + # doesn't need to run. room_membership_for_user_map = dict(room_membership_for_user_map) # Find which rooms are partially stated and may need to be filtered out diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7251e72e3a..b5fe7dd858 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -130,7 +130,7 @@ class SQLBaseStore(metaclass=ABCMeta): "_get_rooms_for_local_user_where_membership_is_inner", (user_id,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", (user_id,) + "get_sliding_sync_rooms_for_user_from_membership_snapshots", (user_id,) ) # Purge other caches based on room state. @@ -138,7 +138,9 @@ class SQLBaseStore(metaclass=ABCMeta): self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) def _invalidate_state_caches_all(self, room_id: str) -> None: """Invalidates caches that are based on the current state, but does @@ -168,7 +170,9 @@ class SQLBaseStore(metaclass=ABCMeta): self._attempt_to_invalidate_cache("get_room_summary", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) def _attempt_to_invalidate_cache( self, cache_name: str, key: Optional[Collection[Any]] diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index f364464c23..9418fb6dd7 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -307,7 +307,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): "get_rooms_for_user", (data.state_key,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", None + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None ) self._membership_stream_cache.entity_has_changed(data.state_key, token) # type: ignore[attr-defined] elif data.type == EventTypes.RoomEncryption: @@ -319,7 +319,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): if (data.type, data.state_key) in SLIDING_SYNC_RELEVANT_STATE_SET: self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", None + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None ) elif row.type == EventsStreamAllStateRow.TypeId: assert isinstance(data, EventsStreamAllStateRow) @@ -330,7 +330,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache("get_rooms_for_user", None) self._attempt_to_invalidate_cache("get_room_type", (data.room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (data.room_id,)) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) else: raise Exception("Unknown events stream row type %s" % (row.type,)) @@ -394,7 +396,8 @@ class CacheInvalidationWorkerStore(SQLBaseStore): "_get_rooms_for_local_user_where_membership_is_inner", (state_key,) ) self._attempt_to_invalidate_cache( - "get_sliding_sync_rooms_for_user", (state_key,) + "get_sliding_sync_rooms_for_user_from_membership_snapshots", + (state_key,), ) self._attempt_to_invalidate_cache( @@ -413,7 +416,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) if (etype, state_key) in SLIDING_SYNC_RELEVANT_STATE_SET: - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) if relates_to: self._attempt_to_invalidate_cache( @@ -470,7 +475,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "_get_rooms_for_local_user_where_membership_is_inner", None ) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) self._attempt_to_invalidate_cache("did_forget", None) self._attempt_to_invalidate_cache("get_forgotten_rooms_for_user", None) self._attempt_to_invalidate_cache("get_references_for_event", None) @@ -529,7 +536,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._attempt_to_invalidate_cache( "get_current_hosts_in_room_ordered", (room_id,) ) - self._attempt_to_invalidate_cache("get_sliding_sync_rooms_for_user", None) + self._attempt_to_invalidate_cache( + "get_sliding_sync_rooms_for_user_from_membership_snapshots", None + ) self._attempt_to_invalidate_cache("did_forget", None) self._attempt_to_invalidate_cache("get_forgotten_rooms_for_user", None) self._attempt_to_invalidate_cache("_get_membership_from_event_id", None) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index b8c78baa6c..2084776543 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -53,6 +53,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.storage.databases.main.stream import _filter_results_by_stream from synapse.storage.engines import Sqlite3Engine from synapse.storage.roommember import ( MemberSummary, @@ -65,6 +66,7 @@ from synapse.types import ( PersistedEventPosition, StateMap, StrCollection, + StreamToken, get_domain_from_id, ) from synapse.util.caches.descriptors import _CacheContext, cached, cachedList @@ -1389,7 +1391,9 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): txn, self.get_forgotten_rooms_for_user, (user_id,) ) self._invalidate_cache_and_stream( - txn, self.get_sliding_sync_rooms_for_user, (user_id,) + txn, + self.get_sliding_sync_rooms_for_user_from_membership_snapshots, + (user_id,), ) await self.db_pool.runInteraction("forget_membership", f) @@ -1421,25 +1425,30 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): ) @cached(iterable=True, max_entries=10000) - async def get_sliding_sync_rooms_for_user( - self, - user_id: str, + async def get_sliding_sync_rooms_for_user_from_membership_snapshots( + self, user_id: str ) -> Mapping[str, RoomsForUserSlidingSync]: - """Get all the rooms for a user to handle a sliding sync request. + """ + Get all the rooms for a user to handle a sliding sync request from the + `sliding_sync_membership_snapshots` table. These will be current memberships and + need to be rewound to the token range. Ignores forgotten rooms and rooms that the user has left themselves. + Args: + user_id: The user ID to get the rooms for. + Returns: Map from room ID to membership info """ - def get_sliding_sync_rooms_for_user_txn( + def _txn( txn: LoggingTransaction, ) -> Dict[str, RoomsForUserSlidingSync]: # XXX: If you use any new columns that can change (like from # `sliding_sync_joined_rooms` or `forgotten`), make sure to bust the - # `get_sliding_sync_rooms_for_user` cache in the appropriate places (and add - # tests). + # `get_sliding_sync_rooms_for_user_from_membership_snapshots` cache in the + # appropriate places (and add tests). sql = """ SELECT m.room_id, m.sender, m.membership, m.membership_event_id, r.room_version, @@ -1455,6 +1464,7 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): AND (m.membership != 'leave' OR m.user_id != m.sender) """ txn.execute(sql, (user_id,)) + return { row[0]: RoomsForUserSlidingSync( room_id=row[0], @@ -1475,8 +1485,113 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): } return await self.db_pool.runInteraction( - "get_sliding_sync_rooms_for_user", - get_sliding_sync_rooms_for_user_txn, + "get_sliding_sync_rooms_for_user_from_membership_snapshots", + _txn, + ) + + async def get_sliding_sync_self_leave_rooms_after_to_token( + self, + user_id: str, + to_token: StreamToken, + ) -> Dict[str, RoomsForUserSlidingSync]: + """ + Get all the self-leave rooms for a user after the `to_token` (outside the token + range) that are potentially relevant[1] and needed to handle a sliding sync + request. The results are from the `sliding_sync_membership_snapshots` table and + will be current memberships and need to be rewound to the token range. + + [1] If a leave happens after the token range, we may have still been joined (or + any non-self-leave which is relevant to sync) to the room before so we need to + include it in the list of potentially relevant rooms and apply + our rewind logic (outside of this function) to see if it's actually relevant. + + This is basically a sister-function to + `get_sliding_sync_rooms_for_user_from_membership_snapshots`. We could + alternatively incorporate this logic into + `get_sliding_sync_rooms_for_user_from_membership_snapshots` but those results + are cached and the `to_token` isn't very cache friendly (people are constantly + requesting with new tokens) so we separate it out here. + + Args: + user_id: The user ID to get the rooms for. + to_token: Any self-leave memberships after this position will be returned. + + Returns: + Map from room ID to membership info + """ + # TODO: Potential to check + # `self._membership_stream_cache.has_entity_changed(...)` as an early-return + # shortcut. + + def _txn( + txn: LoggingTransaction, + ) -> Dict[str, RoomsForUserSlidingSync]: + sql = """ + SELECT m.room_id, m.sender, m.membership, m.membership_event_id, + r.room_version, + m.event_instance_name, m.event_stream_ordering, + m.has_known_state, + m.room_type, + m.is_encrypted + FROM sliding_sync_membership_snapshots AS m + INNER JOIN rooms AS r USING (room_id) + WHERE user_id = ? + AND m.forgotten = 0 + AND m.membership = 'leave' + AND m.user_id = m.sender + AND (m.event_stream_ordering > ?) + """ + # If a leave happens after the token range, we may have still been joined + # (or any non-self-leave which is relevant to sync) to the room before so we + # need to include it in the list of potentially relevant rooms and apply our + # rewind logic (outside of this function). + # + # To handle tokens with a non-empty instance_map we fetch more + # results than necessary and then filter down + min_to_token_position = to_token.room_key.stream + txn.execute(sql, (user_id, min_to_token_position)) + + # Map from room_id to membership info + room_membership_for_user_map: Dict[str, RoomsForUserSlidingSync] = {} + for row in txn: + room_for_user = RoomsForUserSlidingSync( + room_id=row[0], + sender=row[1], + membership=row[2], + event_id=row[3], + room_version_id=row[4], + event_pos=PersistedEventPosition(row[5], row[6]), + has_known_state=bool(row[7]), + room_type=row[8], + is_encrypted=bool(row[9]), + ) + + # We filter out unknown room versions proactively. They shouldn't go + # down sync and their metadata may be in a broken state (causing + # errors). + if row[4] not in KNOWN_ROOM_VERSIONS: + continue + + # We only want to include the self-leave membership if it happened after + # the token range. + # + # Since the database pulls out more than necessary, we need to filter it + # down here. + if _filter_results_by_stream( + lower_token=None, + upper_token=to_token.room_key, + instance_name=room_for_user.event_pos.instance_name, + stream_ordering=room_for_user.event_pos.stream, + ): + continue + + room_membership_for_user_map[room_for_user.room_id] = room_for_user + + return room_membership_for_user_map + + return await self.db_pool.runInteraction( + "get_sliding_sync_self_leave_rooms_after_to_token", + _txn, ) async def get_sliding_sync_room_for_user( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 00e5208674..c52389b8a9 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -453,6 +453,8 @@ def _filter_results_by_stream( stream_ordering falls between the two tokens (taking a None token to mean unbounded). + The token range is defined by > `lower_token` and <= `upper_token`. + Used to filter results from fetching events in the DB against the given tokens. This is necessary to handle the case where the tokens include position maps, which we handle by fetching more than necessary from the DB diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 5b7e2937f8..cbacf21ae7 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -22,7 +22,7 @@ from typing import AbstractSet, Dict, Mapping, Optional, Set, Tuple from unittest.mock import patch import attr -from parameterized import parameterized +from parameterized import parameterized, parameterized_class from twisted.test.proto_helpers import MemoryReactor @@ -43,13 +43,15 @@ from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict, StateMap, StreamToken, UserID -from synapse.types.handlers.sliding_sync import SlidingSyncConfig +from synapse.types import JsonDict, StateMap, StreamToken, UserID, create_requester +from synapse.types.handlers.sliding_sync import PerConnectionState, SlidingSyncConfig from synapse.types.state import StateFilter from synapse.util import Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.test_utils.event_injection import create_event from tests.unittest import HomeserverTestCase, TestCase logger = logging.getLogger(__name__) @@ -572,9 +574,23 @@ class RoomSyncConfigTestCase(TestCase): self._assert_room_config_equal(combined_config, expected, "A into B") -class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class ComputeInterestedRoomsTestCase(SlidingSyncBase): """ - Tests Sliding Sync handler `get_room_membership_for_user_at_to_token()` to make sure it returns + Tests Sliding Sync handler `compute_interested_rooms()` to make sure it returns the correct list of rooms IDs. """ @@ -596,6 +612,11 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.storage_controllers = hs.get_storage_controllers() + persistence = self.hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence + + super().prepare(reactor, clock, hs) def test_no_rooms(self) -> None: """ @@ -606,15 +627,28 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): now_token = self.event_sources.get_current_token() - room_id_results, _, _ = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=now_token, to_token=now_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_get_newly_joined_room(self) -> None: """ @@ -633,22 +667,44 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id}) + self.assertIncludes( + room_id_results, + {room_id}, + exact=True, + ) # It should be pointing to the join event (latest membership event in the # from/to range) self.assertEqual( - room_id_results[room_id].event_id, + interested_rooms.room_membership_for_user_map[room_id].event_id, join_response["event_id"], ) - self.assertEqual(room_id_results[room_id].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id].membership, + Membership.JOIN, + ) # We should be considered `newly_joined` because we joined during the token # range self.assertTrue(room_id in newly_joined) @@ -668,22 +724,40 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id}) + self.assertIncludes(room_id_results, {room_id}, exact=True) # It should be pointing to the join event (latest membership event in the # from/to range) self.assertEqual( - room_id_results[room_id].event_id, + interested_rooms.room_membership_for_user_map[room_id].event_id, join_response["event_id"], ) - self.assertEqual(room_id_results[room_id].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id not in newly_joined) self.assertTrue(room_id not in newly_left) @@ -742,46 +816,71 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_token, to_token=after_room_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Ensure that the invited, ban, and knock rooms show up - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { invited_room_id, ban_room_id, knock_room_id, }, + exact=True, ) # It should be pointing to the the respective membership event (latest # membership event in the from/to range) self.assertEqual( - room_id_results[invited_room_id].event_id, + interested_rooms.room_membership_for_user_map[invited_room_id].event_id, invite_response["event_id"], ) - self.assertEqual(room_id_results[invited_room_id].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[invited_room_id].membership, + Membership.INVITE, + ) self.assertTrue(invited_room_id not in newly_joined) self.assertTrue(invited_room_id not in newly_left) self.assertEqual( - room_id_results[ban_room_id].event_id, + interested_rooms.room_membership_for_user_map[ban_room_id].event_id, ban_response["event_id"], ) - self.assertEqual(room_id_results[ban_room_id].membership, Membership.BAN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[ban_room_id].membership, + Membership.BAN, + ) self.assertTrue(ban_room_id not in newly_joined) self.assertTrue(ban_room_id not in newly_left) self.assertEqual( - room_id_results[knock_room_id].event_id, + interested_rooms.room_membership_for_user_map[knock_room_id].event_id, knock_room_membership_state_event.event_id, ) - self.assertEqual(room_id_results[knock_room_id].membership, Membership.KNOCK) + self.assertEqual( + interested_rooms.room_membership_for_user_map[knock_room_id].membership, + Membership.KNOCK, + ) self.assertTrue(knock_room_id not in newly_joined) self.assertTrue(knock_room_id not in newly_left) @@ -814,23 +913,43 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_kick_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_kick_token, to_token=after_kick_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # The kicked room should show up - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results, {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[kick_room_id].event_id, + interested_rooms.room_membership_for_user_map[kick_room_id].event_id, kick_response["event_id"], ) - self.assertEqual(room_id_results[kick_room_id].membership, Membership.LEAVE) - self.assertNotEqual(room_id_results[kick_room_id].sender, user1_id) + self.assertEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].membership, + Membership.LEAVE, + ) + self.assertNotEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].sender, user1_id + ) # We should *NOT* be `newly_joined` because we were not joined at the the time # of the `to_token`. self.assertTrue(kick_room_id not in newly_joined) @@ -907,16 +1026,29 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): ) self.assertEqual(channel.code, 200, channel.result) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room_forgets, to_token=before_room_forgets, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) # We shouldn't see the room because it was forgotten - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_newly_left_rooms(self) -> None: """ @@ -927,7 +1059,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave before we calculate the `from_token` room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - leave_response1 = self.helper.leave(room_id1, user1_id, tok=user1_tok) + _leave_response1 = self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() @@ -937,31 +1069,52 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room2_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room2_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) - - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response1["event_id"], + # `room_id1` should not show up because it was left before the token range. + # `room_id2` should show up because it is `newly_left` within the token range. + self.assertIncludes( + room_id_results, + {room_id2}, + exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + } + ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined` or `newly_left` because that happened before - # the from/to range - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) self.assertEqual( - room_id_results[room_id2].event_id, + interested_rooms.room_membership_for_user_map[room_id2].event_id, leave_response2["event_id"], ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id2].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are instead `newly_left` self.assertTrue(room_id2 not in newly_joined) self.assertTrue(room_id2 in newly_left) @@ -987,21 +1140,39 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok) self.helper.join(room_id2, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1027,20 +1198,35 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave the room after we already have our tokens leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We should still see the room because we were joined during the # from_token/to_token time period. - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1050,7 +1236,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1074,19 +1263,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave the room after we already have our tokens leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We should still see the room because we were joined before the `from_token` - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1096,7 +1300,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1138,19 +1345,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response2 = self.helper.join(kick_room_id, user1_id, tok=user1_tok) leave_response = self.helper.leave(kick_room_id, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_kick_token, to_token=after_kick_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # We shouldn't see the room because it was forgotten - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results, {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[kick_room_id].event_id, + interested_rooms.room_membership_for_user_map[kick_room_id].event_id, kick_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1162,8 +1384,13 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[kick_room_id].membership, Membership.LEAVE) - self.assertNotEqual(room_id_results[kick_room_id].sender, user1_id) + self.assertEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].membership, + Membership.LEAVE, + ) + self.assertNotEqual( + interested_rooms.room_membership_for_user_map[kick_room_id].sender, user1_id + ) # We should *NOT* be `newly_joined` because we were kicked self.assertTrue(kick_room_id not in newly_joined) self.assertTrue(kick_room_id not in newly_left) @@ -1194,19 +1421,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response2 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response2 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should still show up because it's newly_left during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, leave_response1["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1218,7 +1460,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are actually `newly_left` during # the token range self.assertTrue(room_id1 not in newly_joined) @@ -1249,19 +1494,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join the room after we already have our tokens join_response2 = self.helper.join(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should still show up because it's newly_left during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, leave_response1["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1272,7 +1532,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we are actually `newly_left` during # the token range self.assertTrue(room_id1 not in newly_joined) @@ -1301,48 +1564,54 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join and leave the room2 before the `to_token` self.helper.join(room_id2, user1_id, tok=user1_tok) - leave_response2 = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_response2 = self.helper.leave(room_id2, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() # Join the room2 after we already have our tokens self.helper.join(room_id2, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=None, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Only rooms we were joined to before the `to_token` should show up - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined`/`newly_left` because there is no # `from_token` to define a "live" range to compare against self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_response2["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because there is no - # `from_token` to define a "live" range to compare against - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - def test_from_token_ahead_of_to_token(self) -> None: """ Test when the provided `from_token` comes after the `to_token`. We should @@ -1365,7 +1634,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join and leave the room2 before `to_token` _join_room2_response1 = self.helper.join(room_id2, user1_id, tok=user1_tok) - leave_room2_response1 = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_room2_response1 = self.helper.leave(room_id2, user1_id, tok=user1_tok) # Note: These are purposely swapped. The `from_token` should come after # the `to_token` in this test @@ -1390,55 +1659,70 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Join the room4 after we already have our tokens self.helper.join(room_id4, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=from_token, to_token=to_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # In the "current" state snapshot, we're joined to all of the rooms but in the # from/to token range... self.assertIncludes( - room_id_results.keys(), + room_id_results, { # Included because we were joined before both tokens room_id1, - # Included because we had membership before the to_token - room_id2, + # Excluded because we left before the `from_token` and `to_token` + # room_id2, # Excluded because we joined after the `to_token` # room_id3, # Excluded because we joined after the `to_token` # room_id4, }, exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + "room_id3": room_id3, + "room_id4": room_id4, + } + ), ) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_room1_response1["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined`/`newly_left` because we joined `room1` # before either of the tokens self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_room2_response1["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - def test_leave_before_range_and_join_leave_after_to_token(self) -> None: """ Test old left rooms. But we're also testing that joining and leaving after the @@ -1455,7 +1739,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) # Join and leave the room before the from/to range self.helper.join(room_id1, user1_id, tok=user1_tok) - leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() @@ -1463,25 +1747,28 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): self.helper.join(room_id1, user1_id, tok=user1_tok) self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), {room_id1}) - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) + self.assertIncludes(room_id_results, set(), exact=True) def test_leave_before_range_and_join_after_to_token(self) -> None: """ @@ -1499,32 +1786,35 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) # Join and leave the room before the from/to range self.helper.join(room_id1, user1_id, tok=user1_tok) - leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) after_room1_token = self.event_sources.get_current_token() # Join the room after we already have our tokens self.helper.join(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) - self.assertEqual(room_id_results.keys(), {room_id1}) - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we joined and left - # `room1` before either of the tokens - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) + self.assertIncludes(room_id_results, set(), exact=True) def test_join_leave_multiple_times_during_range_and_after_to_token( self, @@ -1556,19 +1846,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response3 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response3 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because it was newly_left and joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1582,7 +1887,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) # We should *NOT* be `newly_left` because we joined during the token range and @@ -1618,19 +1926,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_response3 = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response3 = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1644,7 +1967,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1677,19 +2003,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): join_respsonse = self.helper.join(room_id1, user1_id, tok=user1_tok) leave_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were invited before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, invite_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1700,7 +2041,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.INVITE, + ) # We should *NOT* be `newly_joined` because we were only invited before the # token range self.assertTrue(room_id1 not in newly_joined) @@ -1751,19 +2095,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1778,7 +2137,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1816,19 +2178,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_change1_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_change1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1840,7 +2217,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -1888,19 +2268,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined before the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_before_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1915,18 +2310,22 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) self.assertTrue(room_id1 not in newly_left) - def test_display_name_changes_leave_after_token_range( + def test_newly_joined_display_name_changes_leave_after_token_range( self, ) -> None: """ Test that we point to the correct membership event within the from/to range even - if there are multiple `join` membership events in a row indicating - `displayname`/`avatar_url` updates and we leave after the `to_token`. + if we are `newly_joined` and there are multiple `join` membership events in a + row indicating `displayname`/`avatar_url` updates and we leave after the + `to_token`. See condition "1a)" comments in the `get_room_membership_for_user_at_to_token()` method. """ @@ -1941,6 +2340,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # leave and can still re-join. room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) join_response = self.helper.join(room_id1, user1_id, tok=user1_tok) + # Update the displayname during the token range displayname_change_during_token_range_response = self.helper.send_state( room_id1, @@ -1970,19 +2370,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave after the token self.helper.leave(room_id1, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -1997,11 +2412,118 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) + def test_display_name_changes_leave_after_token_range( + self, + ) -> None: + """ + Test that we point to the correct membership event within the from/to range even + if there are multiple `join` membership events in a row indicating + `displayname`/`avatar_url` updates and we leave after the `to_token`. + + See condition "1a)" comments in the `get_room_membership_for_user_at_to_token()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + _before_room1_token = self.event_sources.get_current_token() + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + join_response = self.helper.join(room_id1, user1_id, tok=user1_tok) + + after_join_token = self.event_sources.get_current_token() + + # Update the displayname during the token range + displayname_change_during_token_range_response = self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user1_id, + body={ + "membership": Membership.JOIN, + "displayname": "displayname during token range", + }, + tok=user1_tok, + ) + + after_display_name_change_token = self.event_sources.get_current_token() + + # Update the displayname after the token range + displayname_change_after_token_range_response = self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user1_id, + body={ + "membership": Membership.JOIN, + "displayname": "displayname after token range", + }, + tok=user1_tok, + ) + + # Leave after the token + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), + from_token=after_join_token, + to_token=after_display_name_change_token, + ) + ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms + + # Room should show up because we were joined during the from/to range + self.assertIncludes(room_id_results, {room_id1}, exact=True) + # It should be pointing to the latest membership event in the from/to range + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].event_id, + displayname_change_during_token_range_response["event_id"], + "Corresponding map to disambiguate the opaque event IDs: " + + str( + { + "join_response": join_response["event_id"], + "displayname_change_during_token_range_response": displayname_change_during_token_range_response[ + "event_id" + ], + "displayname_change_after_token_range_response": displayname_change_after_token_range_response[ + "event_id" + ], + } + ), + ) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) + # We only changed our display name during the token range so we shouldn't be + # considered `newly_joined` or `newly_left` + self.assertTrue(room_id1 not in newly_joined) + self.assertTrue(room_id1 not in newly_left) + def test_display_name_changes_join_after_token_range( self, ) -> None: @@ -2038,16 +2560,29 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): tok=user1_tok, ) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) # Room shouldn't show up because we joined after the from/to range - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results, set(), exact=True) def test_newly_joined_with_leave_join_in_token_range( self, @@ -2074,22 +2609,40 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_more_changes_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=after_room1_token, to_token=after_more_changes_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because we were joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_response2["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be considered `newly_joined` because there is some non-join event in # between our latest join event. self.assertTrue(room_id1 in newly_joined) @@ -2139,19 +2692,34 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): after_room1_token = self.event_sources.get_current_token() - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room1_token, to_token=after_room1_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room should show up because it was newly_left and joined during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1}) + self.assertIncludes(room_id_results, {room_id1}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, displayname_change_during_token_range_response2["event_id"], "Corresponding map to disambiguate the opaque event IDs: " + str( @@ -2166,7 +2734,10 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): } ), ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we first joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) @@ -2192,7 +2763,7 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Invited and left the room before the token self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) - leave_room1_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) + _leave_room1_response = self.helper.leave(room_id1, user1_id, tok=user1_tok) # Invited to room2 invite_room2_response = self.helper.invite( room_id2, src=user2_id, targ=user1_id, tok=user2_tok @@ -2215,45 +2786,52 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Leave room3 self.helper.leave(room_id3, user1_id, tok=user1_tok) - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_room3_token, to_token=after_room3_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { - # Left before the from/to range - room_id1, + # Excluded because we left before the from/to range + # room_id1, # Invited before the from/to range room_id2, # `newly_left` during the from/to range room_id3, }, + exact=True, ) - # Room1 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id1].event_id, - leave_room1_response["event_id"], - ) - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) - # We should *NOT* be `newly_joined`/`newly_left` because we were invited and left - # before the token range - self.assertTrue(room_id1 not in newly_joined) - self.assertTrue(room_id1 not in newly_left) - # Room2 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id2].event_id, + interested_rooms.room_membership_for_user_map[room_id2].event_id, invite_room2_response["event_id"], ) - self.assertEqual(room_id_results[room_id2].membership, Membership.INVITE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id2].membership, + Membership.INVITE, + ) # We should *NOT* be `newly_joined`/`newly_left` because we were invited before # the token range self.assertTrue(room_id2 not in newly_joined) @@ -2262,10 +2840,13 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): # Room3 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id3].event_id, + interested_rooms.room_membership_for_user_map[room_id3].event_id, leave_room3_response["event_id"], ) - self.assertEqual(room_id_results[room_id3].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id3].membership, + Membership.LEAVE, + ) # We should be `newly_left` because we were invited and left during # the token range self.assertTrue(room_id3 not in newly_joined) @@ -2282,7 +2863,16 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): user2_tok = self.login(user2_id, "pass") # The room where the state reset will happen - room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + room_id1 = self.helper.create_room_as( + user2_id, + is_public=True, + tok=user2_tok, + ) + # Create a dummy event for us to point back to for the state reset + dummy_event_response = self.helper.send(room_id1, "test", tok=user2_tok) + dummy_event_id = dummy_event_response["event_id"] + + # Join after the dummy event join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) # Join another room so we don't hit the short-circuit and return early if they @@ -2292,92 +2882,97 @@ class GetRoomMembershipForUserAtToTokenTestCase(HomeserverTestCase): before_reset_token = self.event_sources.get_current_token() - # Send another state event to make a position for the state reset to happen at - dummy_state_response = self.helper.send_state( - room_id1, - event_type="foobarbaz", - state_key="", - body={"foo": "bar"}, - tok=user2_tok, + # Trigger a state reset + join_rule_event, join_rule_context = self.get_success( + create_event( + self.hs, + prev_event_ids=[dummy_event_id], + type=EventTypes.JoinRules, + state_key="", + content={"join_rule": JoinRules.INVITE}, + sender=user2_id, + room_id=room_id1, + room_version=self.get_success(self.store.get_room_version_id(room_id1)), + ) ) - dummy_state_pos = self.get_success( - self.store.get_position_for_event(dummy_state_response["event_id"]) + _, join_rule_event_pos, _ = self.get_success( + self.persistence.persist_event(join_rule_event, join_rule_context) ) - # Mock a state reset removing the membership for user1 in the current state - self.get_success( - self.store.db_pool.simple_delete( - table="current_state_events", - keyvalues={ - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - }, - desc="state reset user in current_state_events", - ) - ) - self.get_success( - self.store.db_pool.simple_delete( - table="local_current_membership", - keyvalues={ - "room_id": room_id1, - "user_id": user1_id, - }, - desc="state reset user in local_current_membership", - ) - ) - self.get_success( - self.store.db_pool.simple_insert( - table="current_state_delta_stream", - values={ - "stream_id": dummy_state_pos.stream, - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - "event_id": None, - "prev_event_id": join_response1["event_id"], - "instance_name": dummy_state_pos.instance_name, - }, - desc="state reset user in current_state_delta_stream", - ) - ) - - # Manually bust the cache since we we're just manually messing with the database - # and not causing an actual state reset. - self.store._membership_stream_cache.entity_has_changed( - user1_id, dummy_state_pos.stream - ) + # Ensure that the state reset worked and only user2 is in the room now + users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) + self.assertIncludes(set(users_in_room), {user2_id}, exact=True) after_reset_token = self.event_sources.get_current_token() # The function under test - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_reset_token, to_token=after_reset_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms # Room1 should show up because it was `newly_left` via state reset during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results, {room_id1, room_id2}, exact=True) # It should be pointing to no event because we were removed from the room # without a corresponding leave event self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, None, + "Corresponding map to disambiguate the opaque event IDs: " + + str( + { + "join_response1": join_response1["event_id"], + } + ), ) # State reset caused us to leave the room and there is no corresponding leave event - self.assertEqual(room_id_results[room_id1].membership, Membership.LEAVE) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.LEAVE, + ) # We should *NOT* be `newly_joined` because we joined before the token range self.assertTrue(room_id1 not in newly_joined) # We should be `newly_left` because we were removed via state reset during the from/to range self.assertTrue(room_id1 in newly_left) -class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCase): +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class ComputeInterestedRoomsShardTestCase( + BaseMultiWorkerStreamTestCase, SlidingSyncBase +): """ - Tests Sliding Sync handler `get_room_membership_for_user_at_to_token()` to make sure it works with + Tests Sliding Sync handler `compute_interested_rooms()` to make sure it works with sharded event stream_writers enabled """ @@ -2475,7 +3070,7 @@ class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCa join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) join_response2 = self.helper.join(room_id2, user1_id, tok=user1_tok) # Leave room2 - leave_room2_response = self.helper.leave(room_id2, user1_id, tok=user1_tok) + _leave_room2_response = self.helper.leave(room_id2, user1_id, tok=user1_tok) join_response3 = self.helper.join(room_id3, user1_id, tok=user1_tok) # Leave room3 self.helper.leave(room_id3, user1_id, tok=user1_tok) @@ -2565,57 +3160,74 @@ class GetRoomMembershipForUserAtToTokenShardTestCase(BaseMultiWorkerStreamTestCa self.get_success(actx.__aexit__(None, None, None)) # The function under test - room_id_results, newly_joined, newly_left = self.get_success( - self.sliding_sync_handler.room_lists.get_room_membership_for_user_at_to_token( - UserID.from_string(user1_id), + interested_rooms = self.get_success( + self.sliding_sync_handler.room_lists.compute_interested_rooms( + SlidingSyncConfig( + user=UserID.from_string(user1_id), + requester=create_requester(user_id=user1_id), + lists={ + "foo-list": SlidingSyncConfig.SlidingSyncList( + ranges=[(0, 99)], + required_state=[], + timeline_limit=1, + ) + }, + conn_id=None, + ), + PerConnectionState(), from_token=before_stuck_activity_token, to_token=stuck_activity_token, ) ) + room_id_results = set(interested_rooms.lists["foo-list"].ops[0].room_ids) + newly_joined = interested_rooms.newly_joined_rooms + newly_left = interested_rooms.newly_left_rooms - self.assertEqual( - room_id_results.keys(), + self.assertIncludes( + room_id_results, { room_id1, - room_id2, + # Excluded because we left before the from/to range and the second join + # event happened while worker2 was stuck and technically occurs after + # the `stuck_activity_token`. + # room_id2, room_id3, }, + exact=True, + message="Corresponding map to disambiguate the opaque room IDs: " + + str( + { + "room_id1": room_id1, + "room_id2": room_id2, + "room_id3": room_id3, + } + ), ) # Room1 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id1].event_id, + interested_rooms.room_membership_for_user_map[room_id1].event_id, join_room1_response["event_id"], ) - self.assertEqual(room_id_results[room_id1].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id1].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id1 in newly_joined) self.assertTrue(room_id1 not in newly_left) - # Room2 - # It should be pointing to the latest membership event in the from/to range - self.assertEqual( - room_id_results[room_id2].event_id, - leave_room2_response["event_id"], - ) - self.assertEqual(room_id_results[room_id2].membership, Membership.LEAVE) - # room_id2 should *NOT* be considered `newly_left` because we left before the - # from/to range and the join event during the range happened while worker2 was - # stuck. This means that from the perspective of the master, where the - # `stuck_activity_token` is generated, the stream position for worker2 wasn't - # advanced to the join yet. Looking at the `instance_map`, the join technically - # comes after `stuck_activity_token`. - self.assertTrue(room_id2 not in newly_joined) - self.assertTrue(room_id2 not in newly_left) - # Room3 # It should be pointing to the latest membership event in the from/to range self.assertEqual( - room_id_results[room_id3].event_id, + interested_rooms.room_membership_for_user_map[room_id3].event_id, join_on_worker3_response["event_id"], ) - self.assertEqual(room_id_results[room_id3].membership, Membership.JOIN) + self.assertEqual( + interested_rooms.room_membership_for_user_map[room_id3].membership, + Membership.JOIN, + ) # We should be `newly_joined` because we joined during the token range self.assertTrue(room_id3 in newly_joined) self.assertTrue(room_id3 not in newly_left) @@ -2645,6 +3257,9 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.storage_controllers = hs.get_storage_controllers() + persistence = self.hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence def _get_sync_room_ids_for_user( self, @@ -2687,7 +3302,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): to_token=now_token, ) - self.assertEqual(room_id_results.keys(), set()) + self.assertIncludes(room_id_results.keys(), set(), exact=True) def test_basic_rooms(self) -> None: """ @@ -2753,7 +3368,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Ensure that the invited, ban, and knock rooms show up - self.assertEqual( + self.assertIncludes( room_id_results.keys(), { join_room_id, @@ -2761,6 +3376,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ban_room_id, knock_room_id, }, + exact=True, ) # It should be pointing to the the respective membership event (latest # membership event in the from/to range) @@ -2824,7 +3440,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Only the `newly_left` room should show up - self.assertEqual(room_id_results.keys(), {room_id2}) + self.assertIncludes(room_id_results.keys(), {room_id2}, exact=True) self.assertEqual( room_id_results[room_id2].event_id, _leave_response2["event_id"], @@ -2869,7 +3485,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # The kicked room should show up - self.assertEqual(room_id_results.keys(), {kick_room_id}) + self.assertIncludes(room_id_results.keys(), {kick_room_id}, exact=True) # It should be pointing to the latest membership event in the from/to range self.assertEqual( room_id_results[kick_room_id].event_id, @@ -2893,8 +3509,17 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): user2_tok = self.login(user2_id, "pass") # The room where the state reset will happen - room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) - join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) + room_id1 = self.helper.create_room_as( + user2_id, + is_public=True, + tok=user2_tok, + ) + # Create a dummy event for us to point back to for the state reset + dummy_event_response = self.helper.send(room_id1, "test", tok=user2_tok) + dummy_event_id = dummy_event_response["event_id"] + + # Join after the dummy event + self.helper.join(room_id1, user1_id, tok=user1_tok) # Join another room so we don't hit the short-circuit and return early if they # have no room membership @@ -2903,61 +3528,26 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): before_reset_token = self.event_sources.get_current_token() - # Send another state event to make a position for the state reset to happen at - dummy_state_response = self.helper.send_state( - room_id1, - event_type="foobarbaz", - state_key="", - body={"foo": "bar"}, - tok=user2_tok, + # Trigger a state reset + join_rule_event, join_rule_context = self.get_success( + create_event( + self.hs, + prev_event_ids=[dummy_event_id], + type=EventTypes.JoinRules, + state_key="", + content={"join_rule": JoinRules.INVITE}, + sender=user2_id, + room_id=room_id1, + room_version=self.get_success(self.store.get_room_version_id(room_id1)), + ) ) - dummy_state_pos = self.get_success( - self.store.get_position_for_event(dummy_state_response["event_id"]) + _, join_rule_event_pos, _ = self.get_success( + self.persistence.persist_event(join_rule_event, join_rule_context) ) - # Mock a state reset removing the membership for user1 in the current state - self.get_success( - self.store.db_pool.simple_delete( - table="current_state_events", - keyvalues={ - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - }, - desc="state reset user in current_state_events", - ) - ) - self.get_success( - self.store.db_pool.simple_delete( - table="local_current_membership", - keyvalues={ - "room_id": room_id1, - "user_id": user1_id, - }, - desc="state reset user in local_current_membership", - ) - ) - self.get_success( - self.store.db_pool.simple_insert( - table="current_state_delta_stream", - values={ - "stream_id": dummy_state_pos.stream, - "room_id": room_id1, - "type": EventTypes.Member, - "state_key": user1_id, - "event_id": None, - "prev_event_id": join_response1["event_id"], - "instance_name": dummy_state_pos.instance_name, - }, - desc="state reset user in current_state_delta_stream", - ) - ) - - # Manually bust the cache since we we're just manually messing with the database - # and not causing an actual state reset. - self.store._membership_stream_cache.entity_has_changed( - user1_id, dummy_state_pos.stream - ) + # Ensure that the state reset worked and only user2 is in the room now + users_in_room = self.get_success(self.store.get_users_in_room(room_id1)) + self.assertIncludes(set(users_in_room), {user2_id}, exact=True) after_reset_token = self.event_sources.get_current_token() @@ -2969,7 +3559,7 @@ class FilterRoomsRelevantForSyncTestCase(HomeserverTestCase): ) # Room1 should show up because it was `newly_left` via state reset during the from/to range - self.assertEqual(room_id_results.keys(), {room_id1, room_id2}) + self.assertIncludes(room_id_results.keys(), {room_id1, room_id2}, exact=True) # It should be pointing to no event because we were removed from the room # without a corresponding leave event self.assertEqual( From 7c633f1a58e22ea27a172efdc52d94bfdac8c728 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 8 May 2025 14:28:23 +0000 Subject: [PATCH 297/691] Pass leave from remote invite rejection down Sliding Sync (#18375) Fixes #17753 ### Dev notes The `sliding_sync_membership_snapshots` and `sliding_sync_joined_rooms` database tables were added in https://github.com/element-hq/synapse/pull/17512 ### Pull Request Checklist * [X] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Erik Johnston Co-authored-by: Olivier 'reivilibre Co-authored-by: Eric Eastwood --- changelog.d/18375.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 23 ++ synapse/handlers/sliding_sync/room_lists.py | 49 ++++- synapse/storage/databases/main/stream.py | 202 ++++++++++++++++++ .../92/03_ss_membership_snapshot_idx.sql | 16 ++ tests/handlers/test_sliding_sync.py | 12 ++ .../client/sliding_sync/test_sliding_sync.py | 58 +++++ 7 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18375.bugfix create mode 100644 synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql diff --git a/changelog.d/18375.bugfix b/changelog.d/18375.bugfix new file mode 100644 index 0000000000..faebe6f046 --- /dev/null +++ b/changelog.d/18375.bugfix @@ -0,0 +1 @@ +Pass leave from remote invite rejection down Sliding Sync. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 459d3c3e24..cb56eb53fc 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -271,6 +271,7 @@ class SlidingSyncHandler: from_token=from_token, to_token=to_token, newly_joined=room_id in interested_rooms.newly_joined_rooms, + newly_left=room_id in interested_rooms.newly_left_rooms, is_dm=room_id in interested_rooms.dm_room_ids, ) @@ -542,6 +543,7 @@ class SlidingSyncHandler: from_token: Optional[SlidingSyncStreamToken], to_token: StreamToken, newly_joined: bool, + newly_left: bool, is_dm: bool, ) -> SlidingSyncResult.RoomResult: """ @@ -559,6 +561,7 @@ class SlidingSyncHandler: from_token: The point in the stream to sync from. to_token: The point in the stream to sync up to. newly_joined: If the user has newly joined the room + newly_left: If the user has newly left the room is_dm: Whether the room is a DM room """ user = sync_config.user @@ -856,6 +859,26 @@ class SlidingSyncHandler: # TODO: Limit the number of state events we're about to send down # the room, if its too many we should change this to an # `initial=True`? + + # For the case of rejecting remote invites, the leave event won't be + # returned by `get_current_state_deltas_for_room`. This is due to the current + # state only being filled out for rooms the server is in, and so doesn't pick + # up out-of-band leaves (including locally rejected invites) as these events + # are outliers and not added to the `current_state_delta_stream`. + # + # We rely on being explicitly told that the room has been `newly_left` to + # ensure we extract the out-of-band leave. + if newly_left and room_membership_for_user_at_to_token.event_id is not None: + membership_changed = True + leave_event = await self.store.get_event( + room_membership_for_user_at_to_token.event_id + ) + state_key = leave_event.get_state_key() + if state_key is not None: + room_state_delta_id_map[(leave_event.type, state_key)] = ( + room_membership_for_user_at_to_token.event_id + ) + deltas = await self.get_current_state_deltas_for_room( room_id=room_id, room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 7e3cf539df..6d1ac91605 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -1120,7 +1120,7 @@ class SlidingSyncRoomLists: ( newly_joined_room_ids, newly_left_room_map, - ) = await self._get_newly_joined_and_left_rooms( + ) = await self._get_newly_joined_and_left_rooms_fallback( user_id, to_token=to_token, from_token=from_token ) @@ -1176,6 +1176,53 @@ class SlidingSyncRoomLists: "state reset" out of the room, and so that room would not be part of the "current memberships" of the user. + Returns: + A 2-tuple of newly joined room IDs and a map of newly_left room + IDs to the `RoomsForUserStateReset` entry. + + We're using `RoomsForUserStateReset` but that doesn't necessarily mean the + user was state reset of the rooms. It's just that the `event_id`/`sender` + are optional and we can't tell the difference between the server leaving the + room when the user was the last person participating in the room and left or + was state reset out of the room. To actually check for a state reset, you + need to check if a membership still exists in the room. + """ + + newly_joined_room_ids: Set[str] = set() + newly_left_room_map: Dict[str, RoomsForUserStateReset] = {} + + if not from_token: + return newly_joined_room_ids, newly_left_room_map + + changes = await self.store.get_sliding_sync_membership_changes( + user_id, + from_key=from_token.room_key, + to_key=to_token.room_key, + excluded_room_ids=set(self.rooms_to_exclude_globally), + ) + + for room_id, entry in changes.items(): + if entry.membership == Membership.JOIN: + newly_joined_room_ids.add(room_id) + elif entry.membership == Membership.LEAVE: + newly_left_room_map[room_id] = entry + + return newly_joined_room_ids, newly_left_room_map + + @trace + async def _get_newly_joined_and_left_rooms_fallback( + self, + user_id: str, + to_token: StreamToken, + from_token: Optional[StreamToken], + ) -> Tuple[AbstractSet[str], Mapping[str, RoomsForUserStateReset]]: + """Fetch the sets of rooms that the user newly joined or left in the + given token range. + + Note: there may be rooms in the newly left rooms where the user was + "state reset" out of the room, and so that room would not be part of the + "current memberships" of the user. + Returns: A 2-tuple of newly joined room IDs and a map of newly_left room IDs to the `RoomsForUserStateReset` entry. diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index c52389b8a9..3fda49f31f 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -80,6 +80,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine +from synapse.storage.roommember import RoomsForUserStateReset from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import PersistedEventPosition, RoomStreamToken, StrCollection from synapse.util.caches.descriptors import cached, cachedList @@ -993,6 +994,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): available in the `current_state_delta_stream` table. To actually check for a state reset, you need to check if a membership still exists in the room. """ + + assert from_key.topological is None + assert to_key.topological is None + # Start by ruling out cases where a DB query is not necessary. if from_key == to_key: return [] @@ -1138,6 +1143,203 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): if membership_change.room_id not in room_ids_to_exclude ] + @trace + async def get_sliding_sync_membership_changes( + self, + user_id: str, + from_key: RoomStreamToken, + to_key: RoomStreamToken, + excluded_room_ids: Optional[AbstractSet[str]] = None, + ) -> Dict[str, RoomsForUserStateReset]: + """ + Fetch membership events that result in a meaningful membership change for a + given user. + + A meaningful membership changes is one where the `membership` value actually + changes. This means memberships changes from `join` to `join` (like a display + name change) will be filtered out since they result in no meaningful change. + + Note: This function only works with "live" tokens with `stream_ordering` only. + + We're looking for membership changes in the token range (> `from_key` and <= + `to_key`). + + Args: + user_id: The user ID to fetch membership events for. + from_key: The point in the stream to sync from (fetching events > this point). + to_key: The token to fetch rooms up to (fetching events <= this point). + excluded_room_ids: Optional list of room IDs to exclude from the results. + + Returns: + All meaningful membership changes to the current state in the token range. + Events are sorted by `stream_ordering` ascending. + + `event_id`/`sender` can be `None` when the server leaves a room (meaning + everyone locally left) or a state reset which removed the person from the + room. We can't tell the difference between the two cases with what's + available in the `current_state_delta_stream` table. To actually check for a + state reset, you need to check if a membership still exists in the room. + """ + + assert from_key.topological is None + assert to_key.topological is None + + # Start by ruling out cases where a DB query is not necessary. + if from_key == to_key: + return {} + + if from_key: + has_changed = self._membership_stream_cache.has_entity_changed( + user_id, int(from_key.stream) + ) + if not has_changed: + return {} + + room_ids_to_exclude: AbstractSet[str] = set() + if excluded_room_ids is not None: + room_ids_to_exclude = excluded_room_ids + + def f(txn: LoggingTransaction) -> Dict[str, RoomsForUserStateReset]: + # To handle tokens with a non-empty instance_map we fetch more + # results than necessary and then filter down + min_from_id = from_key.stream + max_to_id = to_key.get_max_stream_pos() + + # This query looks at membership changes in + # `sliding_sync_membership_snapshots` which will not include users + # that were state reset out of rooms; so we need to look for that + # case in `current_state_delta_stream`. + sql = """ + SELECT + room_id, + membership_event_id, + event_instance_name, + event_stream_ordering, + membership, + sender, + prev_membership, + room_version + FROM + ( + SELECT + s.room_id, + s.membership_event_id, + s.event_instance_name, + s.event_stream_ordering, + s.membership, + s.sender, + m_prev.membership AS prev_membership + FROM sliding_sync_membership_snapshots as s + LEFT JOIN event_edges AS e ON e.event_id = s.membership_event_id + LEFT JOIN room_memberships AS m_prev ON m_prev.event_id = e.prev_event_id + WHERE s.user_id = ? + + UNION ALL + + SELECT + s.room_id, + e.event_id, + s.instance_name, + s.stream_id, + m.membership, + e.sender, + m_prev.membership AS prev_membership + FROM current_state_delta_stream AS s + LEFT JOIN events AS e ON e.event_id = s.event_id + LEFT JOIN room_memberships AS m ON m.event_id = s.event_id + LEFT JOIN room_memberships AS m_prev ON m_prev.event_id = s.prev_event_id + WHERE + s.type = ? + AND s.state_key = ? + ) AS c + INNER JOIN rooms USING (room_id) + WHERE event_stream_ordering > ? AND event_stream_ordering <= ? + ORDER BY event_stream_ordering ASC + """ + + txn.execute( + sql, + (user_id, EventTypes.Member, user_id, min_from_id, max_to_id), + ) + + membership_changes: Dict[str, RoomsForUserStateReset] = {} + for ( + room_id, + membership_event_id, + event_instance_name, + event_stream_ordering, + membership, + sender, + prev_membership, + room_version_id, + ) in txn: + assert room_id is not None + assert event_stream_ordering is not None + + if room_id in room_ids_to_exclude: + continue + + if _filter_results_by_stream( + from_key, + to_key, + event_instance_name, + event_stream_ordering, + ): + # When the server leaves a room, it will insert new rows into the + # `current_state_delta_stream` table with `event_id = null` for all + # current state. This means we might already have a row for the + # leave event and then another for the same leave where the + # `event_id=null` but the `prev_event_id` is pointing back at the + # earlier leave event. We don't want to report the leave, if we + # already have a leave event. + if ( + membership_event_id is None + and prev_membership == Membership.LEAVE + ): + continue + + if membership_event_id is None and room_id in membership_changes: + # SUSPICIOUS: if we join a room and get state reset out of it + # in the same queried window, + # won't this ignore the 'state reset out of it' part? + continue + + # When `s.event_id = null`, we won't be able to get respective + # `room_membership` but can assume the user has left the room + # because this only happens when the server leaves a room + # (meaning everyone locally left) or a state reset which removed + # the person from the room. + membership = ( + membership if membership is not None else Membership.LEAVE + ) + + if membership == prev_membership: + # If `membership` and `prev_membership` are the same then this + # is not a meaningful change so we can skip it. + # An example of this happening is when the user changes their display name. + continue + + membership_change = RoomsForUserStateReset( + room_id=room_id, + sender=sender, + membership=membership, + event_id=membership_event_id, + event_pos=PersistedEventPosition( + event_instance_name, event_stream_ordering + ), + room_version_id=room_version_id, + ) + + membership_changes[room_id] = membership_change + + return membership_changes + + membership_changes = await self.db_pool.runInteraction( + "get_sliding_sync_membership_changes", f + ) + + return membership_changes + @cancellable async def get_membership_changes_for_user( self, diff --git a/synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql b/synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql new file mode 100644 index 0000000000..c694203f95 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- So we can fetch all rooms for a given user sorted by stream order +DROP INDEX IF EXISTS sliding_sync_membership_snapshots_user_id; +CREATE INDEX IF NOT EXISTS sliding_sync_membership_snapshots_user_id ON sliding_sync_membership_snapshots(user_id, event_stream_ordering); diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index cbacf21ae7..7144c58217 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -594,6 +594,12 @@ class ComputeInterestedRoomsTestCase(SlidingSyncBase): the correct list of rooms IDs. """ + # FIXME: We should refactor these tests to run against `compute_interested_rooms(...)` + # instead of just `get_room_membership_for_user_at_to_token(...)` which is only used + # in the fallback path (`_compute_interested_rooms_fallback(...)`). These scenarios do + # well to stress that logic and we shouldn't remove them just because we're removing + # the fallback path (tracked by https://github.com/element-hq/synapse/issues/17623). + servlets = [ admin.register_servlets, knock.register_servlets, @@ -2976,6 +2982,12 @@ class ComputeInterestedRoomsShardTestCase( sharded event stream_writers enabled """ + # FIXME: We should refactor these tests to run against `compute_interested_rooms(...)` + # instead of just `get_room_membership_for_user_at_to_token(...)` which is only used + # in the fallback path (`_compute_interested_rooms_fallback(...)`). These scenarios do + # well to stress that logic and we shouldn't remove them just because we're removing + # the fallback path (tracked by https://github.com/element-hq/synapse/issues/17623). + servlets = [ admin.register_servlets_for_client_rest_resource, room.register_servlets, diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index f3cf2111ec..dcec5b4cf0 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -790,6 +790,64 @@ class SlidingSyncTestCase(SlidingSyncBase): exact=True, ) + def test_reject_remote_invite(self) -> None: + """Test that rejecting a remote invite comes down incremental sync""" + + user_id = self.register_user("user1", "pass") + user_tok = self.login(user_id, "pass") + + # Create a remote room invite (out-of-band membership) + room_id = "!room:remote.server" + self._create_remote_invite_room_for_user(user_id, None, room_id) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [(EventTypes.Member, StateValues.ME)], + "timeline_limit": 3, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user_tok) + # We should see the room (like normal) + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + + # Reject the remote room invite + self.helper.leave(room_id, user_id, tok=user_tok) + + # Sync again after rejecting the invite + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user_tok) + + # The fix to add the leave event to incremental sync when rejecting a remote + # invite relies on the new tables to work. + if self.use_new_tables: + # We should see the newly_left room + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id}, + exact=True, + ) + # We should see the leave state for the room so clients don't end up with stuck + # invites + self.assertIncludes( + { + ( + state["type"], + state["state_key"], + state["content"].get("membership"), + ) + for state in response_body["rooms"][room_id]["required_state"] + }, + {(EventTypes.Member, user_id, Membership.LEAVE)}, + exact=True, + ) + def test_ignored_user_invites_initial_sync(self) -> None: """ Make sure we ignore invites if they are from one of the `m.ignored_user_list` on From b5d94f654c32b0cd09ba727baddba93b0bf4f63f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 15:35:18 +0100 Subject: [PATCH 298/691] Bump sha2 from 0.10.8 to 0.10.9 (#18395) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 822eb2cdba..27a2e26be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,9 +480,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", From c6dfe70014c7f577a7fa749bfb8953bd08bc69d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 15:36:41 +0100 Subject: [PATCH 299/691] Bump txredisapi from 1.4.10 to 1.4.11 (#18392) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index abd97a785b..69d76936b0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2886,15 +2886,15 @@ windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)" [[package]] name = "txredisapi" -version = "1.4.10" +version = "1.4.11" description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] markers = "extra == \"all\" or extra == \"redis\"" files = [ - {file = "txredisapi-1.4.10-py3-none-any.whl", hash = "sha256:0a6ea77f27f8cf092f907654f08302a97b48fa35f24e0ad99dfb74115f018161"}, - {file = "txredisapi-1.4.10.tar.gz", hash = "sha256:7609a6af6ff4619a3189c0adfb86aeda789afba69eb59fc1e19ac0199e725395"}, + {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, + {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, ] [package.dependencies] From b7728a2df10de6cd09f5313ebca8a95e226c15fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 15:37:05 +0100 Subject: [PATCH 300/691] Bump packaging from 24.2 to 25.0 (#18393) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 69d76936b0..8ff3a377f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1561,14 +1561,14 @@ tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pyte [[package]] name = "packaging" -version = "24.2" +version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, - {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] [[package]] From 1920dfff40ad1078071e099a2afbfa31a5409e6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:36:54 +0100 Subject: [PATCH 301/691] Bump pydantic from 2.10.3 to 2.11.4 (#18394) --- poetry.lock | 229 ++++++++++++++++++++++++++++------------------------ 1 file changed, 122 insertions(+), 107 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8ff3a377f4..e06e08b7a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1795,20 +1795,21 @@ files = [ [[package]] name = "pydantic" -version = "2.10.3" +version = "2.11.4" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, - {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, + {file = "pydantic-2.11.4-py3-none-any.whl", hash = "sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb"}, + {file = "pydantic-2.11.4.tar.gz", hash = "sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.27.1" +pydantic-core = "2.33.2" typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1816,112 +1817,111 @@ timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows [[package]] name = "pydantic-core" -version = "2.27.1" +version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, - {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08"}, - {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c"}, - {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206"}, - {file = "pydantic_core-2.27.1-cp310-none-win32.whl", hash = "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c"}, - {file = "pydantic_core-2.27.1-cp310-none-win_amd64.whl", hash = "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17"}, - {file = "pydantic_core-2.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8"}, - {file = "pydantic_core-2.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025"}, - {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c"}, - {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc"}, - {file = "pydantic_core-2.27.1-cp311-none-win32.whl", hash = "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9"}, - {file = "pydantic_core-2.27.1-cp311-none-win_amd64.whl", hash = "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5"}, - {file = "pydantic_core-2.27.1-cp311-none-win_arm64.whl", hash = "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89"}, - {file = "pydantic_core-2.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f"}, - {file = "pydantic_core-2.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35"}, - {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb"}, - {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae"}, - {file = "pydantic_core-2.27.1-cp312-none-win32.whl", hash = "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c"}, - {file = "pydantic_core-2.27.1-cp312-none-win_amd64.whl", hash = "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16"}, - {file = "pydantic_core-2.27.1-cp312-none-win_arm64.whl", hash = "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e"}, - {file = "pydantic_core-2.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073"}, - {file = "pydantic_core-2.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51"}, - {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960"}, - {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23"}, - {file = "pydantic_core-2.27.1-cp313-none-win32.whl", hash = "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05"}, - {file = "pydantic_core-2.27.1-cp313-none-win_amd64.whl", hash = "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337"}, - {file = "pydantic_core-2.27.1-cp313-none-win_arm64.whl", hash = "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5"}, - {file = "pydantic_core-2.27.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62"}, - {file = "pydantic_core-2.27.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78"}, - {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a"}, - {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b"}, - {file = "pydantic_core-2.27.1-cp38-none-win32.whl", hash = "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618"}, - {file = "pydantic_core-2.27.1-cp38-none-win_amd64.whl", hash = "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4"}, - {file = "pydantic_core-2.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967"}, - {file = "pydantic_core-2.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e"}, - {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9"}, - {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131"}, - {file = "pydantic_core-2.27.1-cp39-none-win32.whl", hash = "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3"}, - {file = "pydantic_core-2.27.1-cp39-none-win_amd64.whl", hash = "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f"}, - {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2"}, - {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840"}, - {file = "pydantic_core-2.27.1.tar.gz", hash = "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235"}, + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, + {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2"}, + {file = "pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a"}, + {file = "pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22"}, + {file = "pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7"}, + {file = "pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef"}, + {file = "pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30"}, + {file = "pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab"}, + {file = "pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc"}, + {file = "pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6"}, + {file = "pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2"}, + {file = "pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f"}, + {file = "pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d"}, + {file = "pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e"}, + {file = "pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9"}, + {file = "pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5"}, + {file = "pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d"}, + {file = "pydantic_core-2.33.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3"}, + {file = "pydantic_core-2.33.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win32.whl", hash = "sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9"}, + {file = "pydantic_core-2.33.2-cp39-cp39-win_amd64.whl", hash = "sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c"}, + {file = "pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb"}, + {file = "pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039"}, + {file = "pydantic_core-2.33.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27"}, + {file = "pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc"}, ] [package.dependencies] @@ -3085,6 +3085,21 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "typing-inspection" +version = "0.4.0" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, + {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + [[package]] name = "unpaddedbase64" version = "2.1.0" From 3dade08e7cef99a83e3410365a14a21a2b24d545 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 09:34:23 +0100 Subject: [PATCH 302/691] Bump actions/setup-go from 5.4.0 to 5.5.0 (#18426) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/latest_deps.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/twisted_trunk.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index e7378ec0d3..366bb4cddb 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -200,7 +200,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb2e80a908..a7e35a0ece 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -669,7 +669,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 0176f17401..5638029b39 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -173,7 +173,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod From 40ce11ded0aa32158aee4d6526b8dd40c1c63a6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 09:46:03 +0100 Subject: [PATCH 303/691] Bump pillow from 11.1.0 to 11.2.1 (#18429) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 157 ++++++++++++++++++++++++++++------------------------ 1 file changed, 84 insertions(+), 73 deletions(-) diff --git a/poetry.lock b/poetry.lock index e06e08b7a7..1935df638a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1600,89 +1600,100 @@ files = [ [[package]] name = "pillow" -version = "11.1.0" +version = "11.2.1" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, - {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, - {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, - {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, - {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, - {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, - {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, - {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, - {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, - {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, - {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, - {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, - {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, - {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, - {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, - {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, - {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, - {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, - {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, - {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, - {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, - {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, - {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, - {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, - {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, - {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, - {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, - {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, - {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, - {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, - {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, - {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, - {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, - {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, - {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, - {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, - {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, - {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, - {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, - {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, - {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, - {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, - {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, - {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, - {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, - {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, - {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, - {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, - {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, - {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, - {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, - {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, - {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, - {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, - {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, - {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, - {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, - {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, - {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, - {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, - {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, - {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, - {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, - {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, - {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, + {file = "pillow-11.2.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:d57a75d53922fc20c165016a20d9c44f73305e67c351bbc60d1adaf662e74047"}, + {file = "pillow-11.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:127bf6ac4a5b58b3d32fc8289656f77f80567d65660bc46f72c0d77e6600cc95"}, + {file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ba4be812c7a40280629e55ae0b14a0aafa150dd6451297562e1764808bbe61"}, + {file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bd62331e5032bc396a93609982a9ab6b411c05078a52f5fe3cc59234a3abd1"}, + {file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:562d11134c97a62fe3af29581f083033179f7ff435f78392565a1ad2d1c2c45c"}, + {file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c97209e85b5be259994eb5b69ff50c5d20cca0f458ef9abd835e262d9d88b39d"}, + {file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0c3e6d0f59171dfa2e25d7116217543310908dfa2770aa64b8f87605f8cacc97"}, + {file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc1c3bc53befb6096b84165956e886b1729634a799e9d6329a0c512ab651e579"}, + {file = "pillow-11.2.1-cp310-cp310-win32.whl", hash = "sha256:312c77b7f07ab2139924d2639860e084ec2a13e72af54d4f08ac843a5fc9c79d"}, + {file = "pillow-11.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9bc7ae48b8057a611e5fe9f853baa88093b9a76303937449397899385da06fad"}, + {file = "pillow-11.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:2728567e249cdd939f6cc3d1f049595c66e4187f3c34078cbc0a7d21c47482d2"}, + {file = "pillow-11.2.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35ca289f712ccfc699508c4658a1d14652e8033e9b69839edf83cbdd0ba39e70"}, + {file = "pillow-11.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0409af9f829f87a2dfb7e259f78f317a5351f2045158be321fd135973fff7bf"}, + {file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e5c5edee874dce4f653dbe59db7c73a600119fbea8d31f53423586ee2aafd7"}, + {file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b93a07e76d13bff9444f1a029e0af2964e654bfc2e2c2d46bfd080df5ad5f3d8"}, + {file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e6def7eed9e7fa90fde255afaf08060dc4b343bbe524a8f69bdd2a2f0018f600"}, + {file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8f4f3724c068be008c08257207210c138d5f3731af6c155a81c2b09a9eb3a788"}, + {file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0a6709b47019dff32e678bc12c63008311b82b9327613f534e496dacaefb71e"}, + {file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f6b0c664ccb879109ee3ca702a9272d877f4fcd21e5eb63c26422fd6e415365e"}, + {file = "pillow-11.2.1-cp311-cp311-win32.whl", hash = "sha256:cc5d875d56e49f112b6def6813c4e3d3036d269c008bf8aef72cd08d20ca6df6"}, + {file = "pillow-11.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:0f5c7eda47bf8e3c8a283762cab94e496ba977a420868cb819159980b6709193"}, + {file = "pillow-11.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:4d375eb838755f2528ac8cbc926c3e31cc49ca4ad0cf79cff48b20e30634a4a7"}, + {file = "pillow-11.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:78afba22027b4accef10dbd5eed84425930ba41b3ea0a86fa8d20baaf19d807f"}, + {file = "pillow-11.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78092232a4ab376a35d68c4e6d5e00dfd73454bd12b230420025fbe178ee3b0b"}, + {file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a5f306095c6780c52e6bbb6109624b95c5b18e40aab1c3041da3e9e0cd3e2d"}, + {file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c7b29dbd4281923a2bfe562acb734cee96bbb129e96e6972d315ed9f232bef4"}, + {file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e645b020f3209a0181a418bffe7b4a93171eef6c4ef6cc20980b30bebf17b7d"}, + {file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2dbea1012ccb784a65349f57bbc93730b96e85b42e9bf7b01ef40443db720b4"}, + {file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3104c57bbd72948d75f6a9389e6727d2ab6333c3617f0a89d72d4940aa0443"}, + {file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:598174aef4589af795f66f9caab87ba4ff860ce08cd5bb447c6fc553ffee603c"}, + {file = "pillow-11.2.1-cp312-cp312-win32.whl", hash = "sha256:1d535df14716e7f8776b9e7fee118576d65572b4aad3ed639be9e4fa88a1cad3"}, + {file = "pillow-11.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:14e33b28bf17c7a38eede290f77db7c664e4eb01f7869e37fa98a5aa95978941"}, + {file = "pillow-11.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:21e1470ac9e5739ff880c211fc3af01e3ae505859392bf65458c224d0bf283eb"}, + {file = "pillow-11.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fdec757fea0b793056419bca3e9932eb2b0ceec90ef4813ea4c1e072c389eb28"}, + {file = "pillow-11.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0e130705d568e2f43a17bcbe74d90958e8a16263868a12c3e0d9c8162690830"}, + {file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bdb5e09068332578214cadd9c05e3d64d99e0e87591be22a324bdbc18925be0"}, + {file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d189ba1bebfbc0c0e529159631ec72bb9e9bc041f01ec6d3233d6d82eb823bc1"}, + {file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:191955c55d8a712fab8934a42bfefbf99dd0b5875078240943f913bb66d46d9f"}, + {file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad275964d52e2243430472fc5d2c2334b4fc3ff9c16cb0a19254e25efa03a155"}, + {file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:750f96efe0597382660d8b53e90dd1dd44568a8edb51cb7f9d5d918b80d4de14"}, + {file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fe15238d3798788d00716637b3d4e7bb6bde18b26e5d08335a96e88564a36b6b"}, + {file = "pillow-11.2.1-cp313-cp313-win32.whl", hash = "sha256:3fe735ced9a607fee4f481423a9c36701a39719252a9bb251679635f99d0f7d2"}, + {file = "pillow-11.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:74ee3d7ecb3f3c05459ba95eed5efa28d6092d751ce9bf20e3e253a4e497e691"}, + {file = "pillow-11.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:5119225c622403afb4b44bad4c1ca6c1f98eed79db8d3bc6e4e160fc6339d66c"}, + {file = "pillow-11.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8ce2e8411c7aaef53e6bb29fe98f28cd4fbd9a1d9be2eeea434331aac0536b22"}, + {file = "pillow-11.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ee66787e095127116d91dea2143db65c7bb1e232f617aa5957c0d9d2a3f23a7"}, + {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9622e3b6c1d8b551b6e6f21873bdcc55762b4b2126633014cea1803368a9aa16"}, + {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63b5dff3a68f371ea06025a1a6966c9a1e1ee452fc8020c2cd0ea41b83e9037b"}, + {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:31df6e2d3d8fc99f993fd253e97fae451a8db2e7207acf97859732273e108406"}, + {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:062b7a42d672c45a70fa1f8b43d1d38ff76b63421cbbe7f88146b39e8a558d91"}, + {file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4eb92eca2711ef8be42fd3f67533765d9fd043b8c80db204f16c8ea62ee1a751"}, + {file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f91ebf30830a48c825590aede79376cb40f110b387c17ee9bd59932c961044f9"}, + {file = "pillow-11.2.1-cp313-cp313t-win32.whl", hash = "sha256:e0b55f27f584ed623221cfe995c912c61606be8513bfa0e07d2c674b4516d9dd"}, + {file = "pillow-11.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:36d6b82164c39ce5482f649b437382c0fb2395eabc1e2b1702a6deb8ad647d6e"}, + {file = "pillow-11.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:225c832a13326e34f212d2072982bb1adb210e0cc0b153e688743018c94a2681"}, + {file = "pillow-11.2.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:7491cf8a79b8eb867d419648fff2f83cb0b3891c8b36da92cc7f1931d46108c8"}, + {file = "pillow-11.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b02d8f9cb83c52578a0b4beadba92e37d83a4ef11570a8688bbf43f4ca50909"}, + {file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:014ca0050c85003620526b0ac1ac53f56fc93af128f7546623cc8e31875ab928"}, + {file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3692b68c87096ac6308296d96354eddd25f98740c9d2ab54e1549d6c8aea9d79"}, + {file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:f781dcb0bc9929adc77bad571b8621ecb1e4cdef86e940fe2e5b5ee24fd33b35"}, + {file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:2b490402c96f907a166615e9a5afacf2519e28295f157ec3a2bb9bd57de638cb"}, + {file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dd6b20b93b3ccc9c1b597999209e4bc5cf2853f9ee66e3fc9a400a78733ffc9a"}, + {file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4b835d89c08a6c2ee7781b8dd0a30209a8012b5f09c0a665b65b0eb3560b6f36"}, + {file = "pillow-11.2.1-cp39-cp39-win32.whl", hash = "sha256:b10428b3416d4f9c61f94b494681280be7686bda15898a3a9e08eb66a6d92d67"}, + {file = "pillow-11.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:6ebce70c3f486acf7591a3d73431fa504a4e18a9b97ff27f5f47b7368e4b9dd1"}, + {file = "pillow-11.2.1-cp39-cp39-win_arm64.whl", hash = "sha256:c27476257b2fdcd7872d54cfd119b3a9ce4610fb85c8e32b70b42e3680a29a1e"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9b7b0d4fd2635f54ad82785d56bc0d94f147096493a79985d0ab57aedd563156"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:aa442755e31c64037aa7c1cb186e0b369f8416c567381852c63444dd666fb772"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0d3348c95b766f54b76116d53d4cb171b52992a1027e7ca50c81b43b9d9e363"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85d27ea4c889342f7e35f6d56e7e1cb345632ad592e8c51b693d7b7556043ce0"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bf2c33d6791c598142f00c9c4c7d47f6476731c31081331664eb26d6ab583e01"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e616e7154c37669fc1dfc14584f11e284e05d1c650e1c0f972f281c4ccc53193"}, + {file = "pillow-11.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:39ad2e0f424394e3aebc40168845fee52df1394a4673a6ee512d840d14ab3013"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:80f1df8dbe9572b4b7abdfa17eb5d78dd620b1d55d9e25f834efdbee872d3aed"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ea926cfbc3957090becbcbbb65ad177161a2ff2ad578b5a6ec9bb1e1cd78753c"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:738db0e0941ca0376804d4de6a782c005245264edaa253ffce24e5a15cbdc7bd"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db98ab6565c69082ec9b0d4e40dd9f6181dab0dd236d26f7a50b8b9bfbd5076"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:036e53f4170e270ddb8797d4c590e6dd14d28e15c7da375c18978045f7e6c37b"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:14f73f7c291279bd65fda51ee87affd7c1e097709f7fdd0188957a16c264601f"}, + {file = "pillow-11.2.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:208653868d5c9ecc2b327f9b9ef34e0e42a4cdd172c2988fd81d62d2bc9bc044"}, + {file = "pillow-11.2.1.tar.gz", hash = "sha256:a64dd61998416367b7ef979b73d3a85853ba9bec4c2925f74e588879a58716b6"}, ] [package.extras] -docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] +test-arrow = ["pyarrow"] tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] From 09b4109c2e3740d754f9256540bd96444fb82453 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 May 2025 10:44:11 +0100 Subject: [PATCH 304/691] 1.130.0rc1 --- CHANGES.md | 63 +++++++++++++++++++++++++++++++++++++++ changelog.d/17578.misc | 1 - changelog.d/18181.misc | 1 - changelog.d/18214.feature | 1 - changelog.d/18218.doc | 1 - changelog.d/18237.doc | 1 - changelog.d/18291.docker | 1 - changelog.d/18292.docker | 1 - changelog.d/18293.docker | 1 - changelog.d/18295.docker | 1 - changelog.d/18297.misc | 1 - changelog.d/18300.feature | 1 - changelog.d/18313.misc | 1 - changelog.d/18320.doc | 1 - changelog.d/18330.misc | 1 - changelog.d/18355.feature | 1 - changelog.d/18360.misc | 1 - changelog.d/18363.bugfix | 1 - changelog.d/18367.misc | 1 - changelog.d/18369.misc | 1 - changelog.d/18374.misc | 1 - changelog.d/18375.bugfix | 1 - changelog.d/18377.doc | 1 - changelog.d/18384.doc | 1 - changelog.d/18385.misc | 1 - changelog.d/18390.misc | 1 - changelog.d/18399.misc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- 29 files changed, 70 insertions(+), 27 deletions(-) delete mode 100644 changelog.d/17578.misc delete mode 100644 changelog.d/18181.misc delete mode 100644 changelog.d/18214.feature delete mode 100644 changelog.d/18218.doc delete mode 100644 changelog.d/18237.doc delete mode 100644 changelog.d/18291.docker delete mode 100644 changelog.d/18292.docker delete mode 100644 changelog.d/18293.docker delete mode 100644 changelog.d/18295.docker delete mode 100644 changelog.d/18297.misc delete mode 100644 changelog.d/18300.feature delete mode 100644 changelog.d/18313.misc delete mode 100644 changelog.d/18320.doc delete mode 100644 changelog.d/18330.misc delete mode 100644 changelog.d/18355.feature delete mode 100644 changelog.d/18360.misc delete mode 100644 changelog.d/18363.bugfix delete mode 100644 changelog.d/18367.misc delete mode 100644 changelog.d/18369.misc delete mode 100644 changelog.d/18374.misc delete mode 100644 changelog.d/18375.bugfix delete mode 100644 changelog.d/18377.doc delete mode 100644 changelog.d/18384.doc delete mode 100644 changelog.d/18385.misc delete mode 100644 changelog.d/18390.misc delete mode 100644 changelog.d/18399.misc diff --git a/CHANGES.md b/CHANGES.md index f04c7ef026..235d65c746 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,66 @@ +# Synapse 1.130.0rc1 (2025-05-13) + +### Features + +- Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. ([\#18214](https://github.com/element-hq/synapse/issues/18214)) +- Add config option `user_directory.exclude_remote_users` which, when enabled, excludes remote users from user directory search results. ([\#18300](https://github.com/element-hq/synapse/issues/18300)) +- Add support for handling `GET /devices/` on workers. ([\#18355](https://github.com/element-hq/synapse/issues/18355)) + +### Bugfixes + +- Fix longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. ([\#18363](https://github.com/element-hq/synapse/issues/18363)) +- Pass leave from remote invite rejection down Sliding Sync. ([\#18375](https://github.com/element-hq/synapse/issues/18375)) + +### Updates to the Docker image + +- In configure_workers_and_start.py, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. ([\#18291](https://github.com/element-hq/synapse/issues/18291)) +- Optimize the build of the workers image. ([\#18292](https://github.com/element-hq/synapse/issues/18292)) +- In start_for_complement.sh, replace some external program calls with shell builtins. ([\#18293](https://github.com/element-hq/synapse/issues/18293)) +- When generating container scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. ([\#18295](https://github.com/element-hq/synapse/issues/18295)) + +### Improved Documentation + +- Improve formatting of the README file. ([\#18218](https://github.com/element-hq/synapse/issues/18218)) +- Add documentation for configuring [Pocket ID](https://github.com/pocket-id/pocket-id) as an OIDC provider. ([\#18237](https://github.com/element-hq/synapse/issues/18237)) +- Fix typo in docs about the `push` config option. Contributed by @HarHarLinks. ([\#18320](https://github.com/element-hq/synapse/issues/18320)) +- Add `/_matrix/federation/v1/version` to list of federation endpoints that can be handled by workers. ([\#18377](https://github.com/element-hq/synapse/issues/18377)) +- Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. ([\#18384](https://github.com/element-hq/synapse/issues/18384)) + +### Internal Changes + +- Return specific error code when adding an email address / phone number to account is not supported (MSC4178). ([\#17578](https://github.com/element-hq/synapse/issues/17578)) +- Stop auto-provisionning missing users & devices when delegating auth to Matrix Authentication Service. Requires MAS 0.13.0 or later. ([\#18181](https://github.com/element-hq/synapse/issues/18181)) +- Apply file hashing and existing quarantines to media downloaded for URL previews. ([\#18297](https://github.com/element-hq/synapse/issues/18297)) +- Allow a few admin APIs used by matrix-authentication-service to run on workers. ([\#18313](https://github.com/element-hq/synapse/issues/18313)) +- Apply `should_drop_federated_event` to federation invites. ([\#18330](https://github.com/element-hq/synapse/issues/18330)) +- Allow `/rooms/` admin API to be run on workers. ([\#18360](https://github.com/element-hq/synapse/issues/18360)) +- Minor performance improvements to the notifier. ([\#18367](https://github.com/element-hq/synapse/issues/18367)) +- Slight performance increase when using the ratelimiter. ([\#18369](https://github.com/element-hq/synapse/issues/18369)) +- Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. ([\#18374](https://github.com/element-hq/synapse/issues/18374), [\#18385](https://github.com/element-hq/synapse/issues/18385)) +- Fixed test failures when using authlib 1.5.2. ([\#18390](https://github.com/element-hq/synapse/issues/18390)) +- Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Simplified Sliding Sync room list tests to cover both new and fallback logic paths. ([\#18399](https://github.com/element-hq/synapse/issues/18399)) + + + +### Updates to locked dependencies + +* Bump actions/add-to-project from 280af8ae1f83a494cfad2cb10f02f6d13529caa9 to 5b1a254a3546aef88e0a7724a77a623fa2e47c36. ([\#18365](https://github.com/element-hq/synapse/issues/18365)) +* Bump actions/download-artifact from 4.2.1 to 4.3.0. ([\#18364](https://github.com/element-hq/synapse/issues/18364)) +* Bump actions/setup-go from 5.4.0 to 5.5.0. ([\#18426](https://github.com/element-hq/synapse/issues/18426)) +* Bump anyhow from 1.0.97 to 1.0.98. ([\#18336](https://github.com/element-hq/synapse/issues/18336)) +* Bump packaging from 24.2 to 25.0. ([\#18393](https://github.com/element-hq/synapse/issues/18393)) +* Bump pillow from 11.1.0 to 11.2.1. ([\#18429](https://github.com/element-hq/synapse/issues/18429)) +* Bump pydantic from 2.10.3 to 2.11.4. ([\#18394](https://github.com/element-hq/synapse/issues/18394)) +* Bump pyo3-log from 0.12.2 to 0.12.3. ([\#18317](https://github.com/element-hq/synapse/issues/18317)) +* Bump pyopenssl from 24.3.0 to 25.0.0. ([\#18315](https://github.com/element-hq/synapse/issues/18315)) +* Bump sha2 from 0.10.8 to 0.10.9. ([\#18395](https://github.com/element-hq/synapse/issues/18395)) +* Bump sigstore/cosign-installer from 3.8.1 to 3.8.2. ([\#18366](https://github.com/element-hq/synapse/issues/18366)) +* Bump softprops/action-gh-release from 1 to 2. ([\#18264](https://github.com/element-hq/synapse/issues/18264)) +* Bump stefanzweifel/git-auto-commit-action from 5.1.0 to 5.2.0. ([\#18354](https://github.com/element-hq/synapse/issues/18354)) +* Bump txredisapi from 1.4.10 to 1.4.11. ([\#18392](https://github.com/element-hq/synapse/issues/18392)) +* Bump types-jsonschema from 4.23.0.20240813 to 4.23.0.20241208. ([\#18305](https://github.com/element-hq/synapse/issues/18305)) +* Bump types-psycopg2 from 2.9.21.20250121 to 2.9.21.20250318. ([\#18316](https://github.com/element-hq/synapse/issues/18316)) + # Synapse 1.129.0 (2025-05-06) No significant changes since 1.129.0rc2. diff --git a/changelog.d/17578.misc b/changelog.d/17578.misc deleted file mode 100644 index 7bf69576cd..0000000000 --- a/changelog.d/17578.misc +++ /dev/null @@ -1 +0,0 @@ -Return specific error code when adding an email address / phone number to account is not supported (MSC4178). diff --git a/changelog.d/18181.misc b/changelog.d/18181.misc deleted file mode 100644 index d9ba2f1dd1..0000000000 --- a/changelog.d/18181.misc +++ /dev/null @@ -1 +0,0 @@ -Stop auto-provisionning missing users & devices when delegating auth to Matrix Authentication Service. Requires MAS 0.13.0 or later. diff --git a/changelog.d/18214.feature b/changelog.d/18214.feature deleted file mode 100644 index 751cb7d383..0000000000 --- a/changelog.d/18214.feature +++ /dev/null @@ -1 +0,0 @@ -Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. \ No newline at end of file diff --git a/changelog.d/18218.doc b/changelog.d/18218.doc deleted file mode 100644 index f62da6a0b9..0000000000 --- a/changelog.d/18218.doc +++ /dev/null @@ -1 +0,0 @@ -Improve formatting of the README file. diff --git a/changelog.d/18237.doc b/changelog.d/18237.doc deleted file mode 100644 index 872f7cab7d..0000000000 --- a/changelog.d/18237.doc +++ /dev/null @@ -1 +0,0 @@ -Add documentation for configuring [Pocket ID](https://github.com/pocket-id/pocket-id) as an OIDC provider. \ No newline at end of file diff --git a/changelog.d/18291.docker b/changelog.d/18291.docker deleted file mode 100644 index b94c0e80e3..0000000000 --- a/changelog.d/18291.docker +++ /dev/null @@ -1 +0,0 @@ -In configure_workers_and_start.py, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. diff --git a/changelog.d/18292.docker b/changelog.d/18292.docker deleted file mode 100644 index cdb95b369b..0000000000 --- a/changelog.d/18292.docker +++ /dev/null @@ -1 +0,0 @@ -Optimize the build of the workers image. diff --git a/changelog.d/18293.docker b/changelog.d/18293.docker deleted file mode 100644 index df47a68bfe..0000000000 --- a/changelog.d/18293.docker +++ /dev/null @@ -1 +0,0 @@ -In start_for_complement.sh, replace some external program calls with shell builtins. diff --git a/changelog.d/18295.docker b/changelog.d/18295.docker deleted file mode 100644 index 239def1f54..0000000000 --- a/changelog.d/18295.docker +++ /dev/null @@ -1 +0,0 @@ -When generating container scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. diff --git a/changelog.d/18297.misc b/changelog.d/18297.misc deleted file mode 100644 index 5032d48174..0000000000 --- a/changelog.d/18297.misc +++ /dev/null @@ -1 +0,0 @@ -Apply file hashing and existing quarantines to media downloaded for URL previews. diff --git a/changelog.d/18300.feature b/changelog.d/18300.feature deleted file mode 100644 index 92bea77556..0000000000 --- a/changelog.d/18300.feature +++ /dev/null @@ -1 +0,0 @@ -Add config option `user_directory.exclude_remote_users` which, when enabled, excludes remote users from user directory search results. \ No newline at end of file diff --git a/changelog.d/18313.misc b/changelog.d/18313.misc deleted file mode 100644 index febf3ac06e..0000000000 --- a/changelog.d/18313.misc +++ /dev/null @@ -1 +0,0 @@ -Allow a few admin APIs used by matrix-authentication-service to run on workers. diff --git a/changelog.d/18320.doc b/changelog.d/18320.doc deleted file mode 100644 index d84c279940..0000000000 --- a/changelog.d/18320.doc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in docs about the `push` config option. Contributed by @HarHarLinks. diff --git a/changelog.d/18330.misc b/changelog.d/18330.misc deleted file mode 100644 index dcf341fa34..0000000000 --- a/changelog.d/18330.misc +++ /dev/null @@ -1 +0,0 @@ -Apply `should_drop_federated_event` to federation invites. diff --git a/changelog.d/18355.feature b/changelog.d/18355.feature deleted file mode 100644 index 4813f0a291..0000000000 --- a/changelog.d/18355.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for handling `GET /devices/` on workers. diff --git a/changelog.d/18360.misc b/changelog.d/18360.misc deleted file mode 100644 index e5bf4f536f..0000000000 --- a/changelog.d/18360.misc +++ /dev/null @@ -1 +0,0 @@ -Allow `/rooms/` admin API to be run on workers. diff --git a/changelog.d/18363.bugfix b/changelog.d/18363.bugfix deleted file mode 100644 index bfa336d52f..0000000000 --- a/changelog.d/18363.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. diff --git a/changelog.d/18367.misc b/changelog.d/18367.misc deleted file mode 100644 index 2e8b897fa6..0000000000 --- a/changelog.d/18367.misc +++ /dev/null @@ -1 +0,0 @@ -Minor performance improvements to the notifier. diff --git a/changelog.d/18369.misc b/changelog.d/18369.misc deleted file mode 100644 index f4c0e5f006..0000000000 --- a/changelog.d/18369.misc +++ /dev/null @@ -1 +0,0 @@ -Slight performance increase when using the ratelimiter. diff --git a/changelog.d/18374.misc b/changelog.d/18374.misc deleted file mode 100644 index a8efca68d0..0000000000 --- a/changelog.d/18374.misc +++ /dev/null @@ -1 +0,0 @@ -Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. \ No newline at end of file diff --git a/changelog.d/18375.bugfix b/changelog.d/18375.bugfix deleted file mode 100644 index faebe6f046..0000000000 --- a/changelog.d/18375.bugfix +++ /dev/null @@ -1 +0,0 @@ -Pass leave from remote invite rejection down Sliding Sync. diff --git a/changelog.d/18377.doc b/changelog.d/18377.doc deleted file mode 100644 index ceb2b64e5d..0000000000 --- a/changelog.d/18377.doc +++ /dev/null @@ -1 +0,0 @@ -Add `/_matrix/federation/v1/version` to list of federation endpoints that can be handled by workers. diff --git a/changelog.d/18384.doc b/changelog.d/18384.doc deleted file mode 100644 index ebcd029639..0000000000 --- a/changelog.d/18384.doc +++ /dev/null @@ -1 +0,0 @@ -Add an Admin API endpoint `GET /_synapse/admin/v1/scheduled_tasks` to fetch scheduled tasks. diff --git a/changelog.d/18385.misc b/changelog.d/18385.misc deleted file mode 100644 index a8efca68d0..0000000000 --- a/changelog.d/18385.misc +++ /dev/null @@ -1 +0,0 @@ -Don't validate the `at_hash` (access token hash) field in OIDC ID Tokens if we don't end up actually using the OIDC Access Token. \ No newline at end of file diff --git a/changelog.d/18390.misc b/changelog.d/18390.misc deleted file mode 100644 index e9a08dcfbf..0000000000 --- a/changelog.d/18390.misc +++ /dev/null @@ -1 +0,0 @@ -Fixed test failures when using authlib 1.5.2. diff --git a/changelog.d/18399.misc b/changelog.d/18399.misc deleted file mode 100644 index 847dc9a2b1..0000000000 --- a/changelog.d/18399.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Simplified Sliding Sync room list tests to cover both new and fallback logic paths. diff --git a/debian/changelog b/debian/changelog index 2c1cb20624..e3eb894851 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.130.0~rc1) stable; urgency=medium + + * New Synapse release 1.130.0rc1. + + -- Synapse Packaging team Tue, 13 May 2025 10:44:04 +0100 + matrix-synapse-py3 (1.129.0) stable; urgency=medium * New Synapse release 1.129.0. diff --git a/pyproject.toml b/pyproject.toml index 24ae0db05c..5f80d28344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.129.0" +version = "1.130.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 99c15f4630a7c9983c1b134505eaab703c138ea9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 May 2025 10:54:23 +0100 Subject: [PATCH 305/691] Fix up changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 235d65c746..a0a9d2f064 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,14 +8,14 @@ ### Bugfixes -- Fix longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. ([\#18363](https://github.com/element-hq/synapse/issues/18363)) +- Fix a longstanding bug where Synapse would immediately retry a failing push endpoint when a new event is received, ignoring any backoff timers. ([\#18363](https://github.com/element-hq/synapse/issues/18363)) - Pass leave from remote invite rejection down Sliding Sync. ([\#18375](https://github.com/element-hq/synapse/issues/18375)) ### Updates to the Docker image -- In configure_workers_and_start.py, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. ([\#18291](https://github.com/element-hq/synapse/issues/18291)) +- In `configure_workers_and_start.py`, use the same absolute path of Python in the interpreter shebang, and invoke child Python processes with `sys.executable`. ([\#18291](https://github.com/element-hq/synapse/issues/18291)) - Optimize the build of the workers image. ([\#18292](https://github.com/element-hq/synapse/issues/18292)) -- In start_for_complement.sh, replace some external program calls with shell builtins. ([\#18293](https://github.com/element-hq/synapse/issues/18293)) +- In `start_for_complement.sh`, replace some external program calls with shell builtins. ([\#18293](https://github.com/element-hq/synapse/issues/18293)) - When generating container scripts from templates, don't add a leading newline so that their shebangs may be handled correctly. ([\#18295](https://github.com/element-hq/synapse/issues/18295)) ### Improved Documentation @@ -28,7 +28,7 @@ ### Internal Changes -- Return specific error code when adding an email address / phone number to account is not supported (MSC4178). ([\#17578](https://github.com/element-hq/synapse/issues/17578)) +- Return specific error code when adding an email address / phone number to account is not supported ([MSC4178](https://github.com/matrix-org/matrix-spec-proposals/pull/4178)). ([\#17578](https://github.com/element-hq/synapse/issues/17578)) - Stop auto-provisionning missing users & devices when delegating auth to Matrix Authentication Service. Requires MAS 0.13.0 or later. ([\#18181](https://github.com/element-hq/synapse/issues/18181)) - Apply file hashing and existing quarantines to media downloaded for URL previews. ([\#18297](https://github.com/element-hq/synapse/issues/18297)) - Allow a few admin APIs used by matrix-authentication-service to run on workers. ([\#18313](https://github.com/element-hq/synapse/issues/18313)) From c626d54cea3a99200c162a2578550e56242e8213 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 15:12:22 +0100 Subject: [PATCH 306/691] Bump mypy-zope from 1.0.9 to 1.0.11 (#18428) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1935df638a..b6e32f43b8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1511,18 +1511,18 @@ files = [ [[package]] name = "mypy-zope" -version = "1.0.9" +version = "1.0.11" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" groups = ["dev"] files = [ - {file = "mypy_zope-1.0.9-py3-none-any.whl", hash = "sha256:6666c1556891a3cb186137519dbd7a58cb30fb72b2504798cad47b35391921ba"}, - {file = "mypy_zope-1.0.9.tar.gz", hash = "sha256:37d6985dfb05a4c27b35cff47577fd5bad878db4893ddedf54d165f7389a1cdb"}, + {file = "mypy_zope-1.0.11-py3-none-any.whl", hash = "sha256:4395d716b43ab89916edf6d0b5761655b4d4a43b2692fce806bbd733829977ee"}, + {file = "mypy_zope-1.0.11.tar.gz", hash = "sha256:1c95e49e9dcdf070a0858f067dac55e8e4e47519fdc15dfdab9b7eee273a0e01"}, ] [package.dependencies] -mypy = ">=1.0.0,<1.14.0" +mypy = ">=1.0.0,<1.16.0" "zope.interface" = "*" "zope.schema" = "*" From ba2f1be891a4dbc2fe55af968dd72a146a8c9068 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 15:12:34 +0100 Subject: [PATCH 307/691] Bump types-requests from 2.32.0.20241016 to 2.32.0.20250328 (#18427) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index b6e32f43b8..7d7868d37f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3059,14 +3059,14 @@ files = [ [[package]] name = "types-requests" -version = "2.32.0.20241016" +version = "2.32.0.20250328" description = "Typing stubs for requests" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, - {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, + {file = "types_requests-2.32.0.20250328-py3-none-any.whl", hash = "sha256:72ff80f84b15eb3aa7a8e2625fffb6a93f2ad5a0c20215fc1dcfa61117bcb2a2"}, + {file = "types_requests-2.32.0.20250328.tar.gz", hash = "sha256:c9e67228ea103bd811c96984fac36ed2ae8da87a36a633964a21f199d60baf32"}, ] [package.dependencies] From 480d4faa38401f37b0b5608356ee1959aa5829c8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 13 May 2025 15:14:00 +0100 Subject: [PATCH 308/691] Remove newline from final bullet point of PR template (#18419) --- .github/PULL_REQUEST_TEMPLATE.md | 3 +-- changelog.d/18419.misc | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18419.misc diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 07d4f6dfce..f8e60815fa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,5 +9,4 @@ - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. -* [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct - (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) +* [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) diff --git a/changelog.d/18419.misc b/changelog.d/18419.misc new file mode 100644 index 0000000000..0ff36e27b8 --- /dev/null +++ b/changelog.d/18419.misc @@ -0,0 +1 @@ +Update the PR review template to remove an erroneous line break from the final bullet point. \ No newline at end of file From 2db54c88ff54a5377d96088c23ac1f4dfef8faf3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 13 May 2025 15:19:30 +0100 Subject: [PATCH 309/691] Explicitly enable pypy for cibuildwheel (#18417) --- changelog.d/18417.misc | 1 + poetry.lock | 49 +++++++++++++++++++++--------------------- pyproject.toml | 3 +++ 3 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 changelog.d/18417.misc diff --git a/changelog.d/18417.misc b/changelog.d/18417.misc new file mode 100644 index 0000000000..5f650a202a --- /dev/null +++ b/changelog.d/18417.misc @@ -0,0 +1 @@ +Explicitly enable PyPy builds in `cibuildwheel`s config to avoid it being disabled on a future upgrade to `cibuildwheel` v3. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 7d7868d37f..7190d0f788 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"}, {file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"}, @@ -451,7 +451,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -494,7 +494,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -544,7 +544,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -890,7 +890,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1028,7 +1028,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1044,7 +1044,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -1330,7 +1330,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1551,7 +1551,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1720,7 +1720,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1728,7 +1728,6 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1741,7 +1740,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1757,7 +1756,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1980,7 +1979,7 @@ description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"user-search\"" +markers = "extra == \"user-search\" or extra == \"all\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -2029,7 +2028,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2089,7 +2088,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2114,7 +2113,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2142,7 +2141,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2506,7 +2505,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2690,7 +2689,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2706,7 +2705,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2768,7 +2767,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, @@ -2902,7 +2901,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3245,7 +3244,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, diff --git a/pyproject.toml b/pyproject.toml index 5f80d28344..914a5804aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -385,6 +385,9 @@ build-backend = "poetry.core.masonry.api" # - PyPy on Aarch64 and musllinux on aarch64: too slow to build. # c.f. https://github.com/matrix-org/synapse/pull/14259 skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" +# Enable non-default builds. +# "pypy" used to be included by default up until cibuildwheel 3. +enable = "pypy" # We need a rust compiler. # From 6e910e2b2c5cef393473dcc6bf957a8671a1186e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 13 May 2025 10:22:15 -0500 Subject: [PATCH 310/691] Fix a couple type annotations in the `RootConfig`/`Config` (#18409) Fix a couple type annotations in the `RootConfig`/`Config`. Discovered while cribbing this code for another project. It's really sucks that `mypy` type checking doesn't catch this. I assume this is because we also have a `synapse/config/_base.pyi` that overrides all of this. Still unclear to me why the `Iterable[str]` vs `StrSequence` issue wasn't caught as that's what `ConfigError` expects. --- changelog.d/18409.misc | 1 + synapse/config/_base.py | 6 +++--- synapse/config/_base.pyi | 4 ++-- synapse/config/experimental.py | 6 +++--- synapse/config/key.py | 6 ++++-- synapse/config/workers.py | 2 +- tests/config/test_api.py | 3 ++- tests/config/test_appservice.py | 7 ++++--- tests/config/test_cache.py | 3 ++- tests/config/test_database.py | 5 ++++- tests/config/test_room_directory.py | 5 +++-- tests/config/test_server.py | 10 +++++----- tests/events/test_auto_accept_invites.py | 5 +++-- 13 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 changelog.d/18409.misc diff --git a/changelog.d/18409.misc b/changelog.d/18409.misc new file mode 100644 index 0000000000..bbb9bdbb1b --- /dev/null +++ b/changelog.d/18409.misc @@ -0,0 +1 @@ +Fix a couple type annotations in the `RootConfig`/`Config`. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 132ba26af9..d367d45fea 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -170,7 +170,7 @@ class Config: section: ClassVar[str] - def __init__(self, root_config: "RootConfig" = None): + def __init__(self, root_config: "RootConfig"): self.root = root_config # Get the path to the default Synapse template directory @@ -445,7 +445,7 @@ class RootConfig: return res @classmethod - def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: any) -> None: + def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: Any) -> None: """ Invoke a static function on config objects this RootConfig is configured to use. @@ -1047,7 +1047,7 @@ class RoutableShardedWorkerHandlingConfig(ShardedWorkerHandlingConfig): return self._get_instance(key) -def read_file(file_path: Any, config_path: Iterable[str]) -> str: +def read_file(file_path: Any, config_path: StrSequence) -> str: """Check the given file exists, and read it into a string If it does not, emit an error indicating the problem diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 55b0e2cbf4..9c4ec8f713 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -179,7 +179,7 @@ class RootConfig: class Config: root: RootConfig default_template_dir: str - def __init__(self, root_config: Optional[RootConfig] = ...) -> None: ... + def __init__(self, root_config: RootConfig = ...) -> None: ... @staticmethod def parse_size(value: Union[str, int]) -> int: ... @staticmethod @@ -212,4 +212,4 @@ class ShardedWorkerHandlingConfig: class RoutableShardedWorkerHandlingConfig(ShardedWorkerHandlingConfig): def get_instance(self, key: str) -> str: ... # noqa: F811 -def read_file(file_path: Any, config_path: Iterable[str]) -> str: ... +def read_file(file_path: Any, config_path: StrSequence) -> str: ... diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 0a963b121a..1226eaa58a 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -21,7 +21,7 @@ import enum from functools import cache -from typing import TYPE_CHECKING, Any, Iterable, Optional +from typing import TYPE_CHECKING, Any, Optional import attr import attr.validators @@ -29,7 +29,7 @@ import attr.validators from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config import ConfigError from synapse.config._base import Config, RootConfig, read_file -from synapse.types import JsonDict +from synapse.types import JsonDict, StrSequence # Determine whether authlib is installed. try: @@ -45,7 +45,7 @@ if TYPE_CHECKING: @cache -def read_secret_from_file_once(file_path: Any, config_path: Iterable[str]) -> str: +def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: """Returns the memoized secret read from file.""" return read_file(file_path, config_path).strip() diff --git a/synapse/config/key.py b/synapse/config/key.py index 337f98dbc1..29c558448b 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -191,7 +191,7 @@ class KeyConfig(Config): if macaroon_secret_key: raise ConfigError(CONFLICTING_MACAROON_SECRET_KEY_OPTS_ERROR) macaroon_secret_key = read_file( - macaroon_secret_key_path, "macaroon_secret_key_path" + macaroon_secret_key_path, ("macaroon_secret_key_path",) ).strip() if not macaroon_secret_key: macaroon_secret_key = self.root.registration.registration_shared_secret @@ -216,7 +216,9 @@ class KeyConfig(Config): if form_secret_path: if form_secret: raise ConfigError(CONFLICTING_FORM_SECRET_OPTS_ERROR) - self.form_secret = read_file(form_secret_path, "form_secret_path").strip() + self.form_secret = read_file( + form_secret_path, ("form_secret_path",) + ).strip() else: self.form_secret = form_secret diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 5af50ee952..2dfeb47c2e 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -263,7 +263,7 @@ class WorkerConfig(Config): if worker_replication_secret: raise ConfigError(CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR) self.worker_replication_secret = read_file( - worker_replication_secret_path, "worker_replication_secret_path" + worker_replication_secret_path, ("worker_replication_secret_path",) ).strip() else: self.worker_replication_secret = worker_replication_secret diff --git a/tests/config/test_api.py b/tests/config/test_api.py index 6773c9a277..e6cc3e21ed 100644 --- a/tests/config/test_api.py +++ b/tests/config/test_api.py @@ -3,6 +3,7 @@ from unittest import TestCase as StdlibTestCase import yaml from synapse.config import ConfigError +from synapse.config._base import RootConfig from synapse.config.api import ApiConfig from synapse.types.state import StateFilter @@ -19,7 +20,7 @@ DEFAULT_PREJOIN_STATE_PAIRS = { class TestRoomPrejoinState(StdlibTestCase): def read_config(self, source: str) -> ApiConfig: - config = ApiConfig() + config = ApiConfig(RootConfig()) config.read_config(yaml.safe_load(source)) return config diff --git a/tests/config/test_appservice.py b/tests/config/test_appservice.py index e3021b59d8..2572681224 100644 --- a/tests/config/test_appservice.py +++ b/tests/config/test_appservice.py @@ -19,6 +19,7 @@ # # +from synapse.config._base import RootConfig from synapse.config.appservice import AppServiceConfig, ConfigError from tests.unittest import TestCase @@ -36,12 +37,12 @@ class AppServiceConfigTest(TestCase): ["foo", "bar", False], ]: with self.assertRaises(ConfigError): - AppServiceConfig().read_config( + AppServiceConfig(RootConfig()).read_config( {"app_service_config_files": invalid_value} ) def test_valid_app_service_config_files(self) -> None: - AppServiceConfig().read_config({"app_service_config_files": []}) - AppServiceConfig().read_config( + AppServiceConfig(RootConfig()).read_config({"app_service_config_files": []}) + AppServiceConfig(RootConfig()).read_config( {"app_service_config_files": ["/not/a/real/path", "/not/a/real/path/2"]} ) diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index 631263b5ca..aead73e059 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -19,6 +19,7 @@ # # +from synapse.config._base import RootConfig from synapse.config.cache import CacheConfig, add_resizable_cache from synapse.types import JsonDict from synapse.util.caches.lrucache import LruCache @@ -29,7 +30,7 @@ from tests.unittest import TestCase class CacheConfigTests(TestCase): def setUp(self) -> None: # Reset caches before each test since there's global state involved. - self.config = CacheConfig() + self.config = CacheConfig(RootConfig()) self.config.reset() def tearDown(self) -> None: diff --git a/tests/config/test_database.py b/tests/config/test_database.py index b46519f84a..3fa5fff2b2 100644 --- a/tests/config/test_database.py +++ b/tests/config/test_database.py @@ -20,6 +20,7 @@ import yaml +from synapse.config._base import RootConfig from synapse.config.database import DatabaseConfig from tests import unittest @@ -28,7 +29,9 @@ from tests import unittest class DatabaseConfigTestCase(unittest.TestCase): def test_database_configured_correctly(self) -> None: conf = yaml.safe_load( - DatabaseConfig().generate_config_section(data_dir_path="/data_dir_path") + DatabaseConfig(RootConfig()).generate_config_section( + data_dir_path="/data_dir_path" + ) ) expected_database_conf = { diff --git a/tests/config/test_room_directory.py b/tests/config/test_room_directory.py index e25f7787f4..5208381279 100644 --- a/tests/config/test_room_directory.py +++ b/tests/config/test_room_directory.py @@ -24,6 +24,7 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin import synapse.rest.client.login import synapse.rest.client.room +from synapse.config._base import RootConfig from synapse.config.room_directory import RoomDirectoryConfig from synapse.server import HomeServer from synapse.util import Clock @@ -63,7 +64,7 @@ class RoomDirectoryConfigTestCase(unittest.HomeserverTestCase): """ ) - rd_config = RoomDirectoryConfig() + rd_config = RoomDirectoryConfig(RootConfig()) rd_config.read_config(config) self.assertFalse( @@ -123,7 +124,7 @@ class RoomDirectoryConfigTestCase(unittest.HomeserverTestCase): """ ) - rd_config = RoomDirectoryConfig() + rd_config = RoomDirectoryConfig(RootConfig()) rd_config.read_config(config) self.assertFalse( diff --git a/tests/config/test_server.py b/tests/config/test_server.py index 74073cfdc5..5eb2540439 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -20,7 +20,7 @@ import yaml -from synapse.config._base import ConfigError +from synapse.config._base import ConfigError, RootConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved from tests import unittest @@ -40,7 +40,7 @@ class ServerConfigTestCase(unittest.TestCase): def test_unsecure_listener_no_listeners_open_private_ports_false(self) -> None: conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "che.org", False, None ) ) @@ -60,7 +60,7 @@ class ServerConfigTestCase(unittest.TestCase): def test_unsecure_listener_no_listeners_open_private_ports_true(self) -> None: conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "che.org", True, None ) ) @@ -94,7 +94,7 @@ class ServerConfigTestCase(unittest.TestCase): ] conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners ) ) @@ -128,7 +128,7 @@ class ServerConfigTestCase(unittest.TestCase): expected_listeners[1]["bind_addresses"] = ["::1", "127.0.0.1"] conf = yaml.safe_load( - ServerConfig().generate_config_section( + ServerConfig(RootConfig()).generate_config_section( "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners ) ) diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index d4e87b1b7f..d2100e9903 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -31,6 +31,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError +from synapse.config._base import RootConfig from synapse.config.auto_accept_invites import AutoAcceptInvitesConfig from synapse.events.auto_accept_invites import InviteAutoAccepter from synapse.federation.federation_base import event_from_pdu_json @@ -690,7 +691,7 @@ class InviteAutoAccepterInternalTestCase(TestCase): "only_from_local_users": True, } } - parsed_config = AutoAcceptInvitesConfig() + parsed_config = AutoAcceptInvitesConfig(RootConfig()) parsed_config.read_config(config) self.assertTrue(parsed_config.enabled) @@ -830,7 +831,7 @@ def create_module( if config_override is None: config_override = {} - config = AutoAcceptInvitesConfig() + config = AutoAcceptInvitesConfig(RootConfig()) config.read_config(config_override) return InviteAutoAccepter(config, module_api) From a3bbd7eeabee7c6b229e95e0e04af5b430ea32db Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 13 May 2025 10:40:49 -0500 Subject: [PATCH 311/691] Explain why we `flush_buffer()` for Python `print(...)` output (#18420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spawning from using this code elsewhere and not knowing why it's there. Based on this article and @reivilibre's experience mentioning `PYTHONUNBUFFERED=1`, > #### programming languages where the default “print” statement buffers > > Also, here are a few programming language where the default print statement will buffer output when writing to a pipe, and some ways to disable buffering if you want: > > - Python (disable with `python -u`, or `PYTHONUNBUFFERED=1`, or `sys.stdout.reconfigure(line_buffering=False)`, or `print(x, flush=True)`) > > _-- https://jvns.ca/blog/2024/11/29/why-pipes-get-stuck-buffering/#programming-languages-where-the-default-print-statement-buffers_ --- changelog.d/18420.misc | 1 + docker/configure_workers_and_start.py | 5 +++++ docker/start.py | 5 +++++ 3 files changed, 11 insertions(+) create mode 100644 changelog.d/18420.misc diff --git a/changelog.d/18420.misc b/changelog.d/18420.misc new file mode 100644 index 0000000000..d52175af91 --- /dev/null +++ b/changelog.d/18420.misc @@ -0,0 +1 @@ +Explain why we `flush_buffer()` for Python `print(...)` output. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index df34d51f77..102a88fad1 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -352,6 +352,11 @@ def error(txt: str) -> NoReturn: def flush_buffers() -> None: + """ + Python's `print()` buffers output by default, typically waiting until ~8KB + accumulates. This method can be used to flush the buffers so we can see the output + of any print statements so far. + """ sys.stdout.flush() sys.stderr.flush() diff --git a/docker/start.py b/docker/start.py index 818a5355ca..0be9976a0c 100755 --- a/docker/start.py +++ b/docker/start.py @@ -22,6 +22,11 @@ def error(txt: str) -> NoReturn: def flush_buffers() -> None: + """ + Python's `print()` buffers output by default, typically waiting until ~8KB + accumulates. This method can be used to flush the buffers so we can see the output + of any print statements so far. + """ sys.stdout.flush() sys.stderr.flush() From 194b923a6e625af6ca90bbbdc1f8a85a9215797e Mon Sep 17 00:00:00 2001 From: Kim Brose <2803622+HarHarLinks@users.noreply.github.com> Date: Wed, 14 May 2025 10:36:54 +0000 Subject: [PATCH 312/691] Fix room_list_publication_rules docs for v1.126.0 (#18286) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18286.doc | 1 + .../configuration/config_documentation.md | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 changelog.d/18286.doc diff --git a/changelog.d/18286.doc b/changelog.d/18286.doc new file mode 100644 index 0000000000..37728351c5 --- /dev/null +++ b/changelog.d/18286.doc @@ -0,0 +1 @@ +Update `room_list_publication_rules` docs to consider defaults that changed in v1.126.0. Contributed by @HarHarLinks. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 5351bef83a..2228c18a6c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4331,23 +4331,11 @@ room list by default_ Example configuration: ```yaml -# No rule list specified. Anyone may publish any room to the public list. +# No rule list specified. No one may publish any room to the public list, except server admins. # This is the default behaviour. room_list_publication_rules: ``` -```yaml -# A list of one rule which allows everything. -# This has the same effect as the previous example. -room_list_publication_rules: - - "action": "allow" -``` - -```yaml -# An empty list of rules. No-one may publish to the room list. -room_list_publication_rules: [] -``` - ```yaml # A list of one rule which denies everything. # This has the same effect as the previous example. @@ -4355,6 +4343,19 @@ room_list_publication_rules: - "action": "deny" ``` +```yaml +# An empty list of rules. +# This has the same effect as the previous example. +room_list_publication_rules: [] +``` + +```yaml +# A list of one rule which allows everything. +# This was the default behaviour pre v1.126.0. +room_list_publication_rules: + - "action": "allow" +``` + ```yaml # Prevent a specific user from publishing rooms. # Allow other users to publish anything. From 44ae5362fd952dbb209f4b52ee9c96641163f032 Mon Sep 17 00:00:00 2001 From: _ Date: Thu, 15 May 2025 04:31:52 -0700 Subject: [PATCH 313/691] Add option to allow registrations that begin with '_' (#18262) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18262.feature | 1 + .../configuration/config_documentation.md | 14 +++++++++++ synapse/config/registration.py | 4 ++++ synapse/handlers/register.py | 5 +++- tests/handlers/test_register.py | 23 +++++++++++++++++++ 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18262.feature diff --git a/changelog.d/18262.feature b/changelog.d/18262.feature new file mode 100644 index 0000000000..c8249faa76 --- /dev/null +++ b/changelog.d/18262.feature @@ -0,0 +1 @@ +Add option to allow registrations that begin with `_`. Contributed by `_` (@hex5f). diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 2228c18a6c..e688bc5cd8 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2887,6 +2887,20 @@ Example configuration: inhibit_user_in_use_error: true ``` --- +### `allow_underscore_prefixed_registration` + +Whether users are allowed to register with a underscore-prefixed localpart. +By default, AppServices use prefixes like `_example` to namespace their +associated ghost users. If turned on, this may result in clashes or confusion. +Useful when provisioning users from an external identity provider. + +Defaults to false. + +Example configuration: +```yaml +allow_underscore_prefixed_registration: false +``` +--- ## User session management --- ### `session_lifetime` diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 3cf7031656..8adf21079e 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -162,6 +162,10 @@ class RegistrationConfig(Config): "disable_msisdn_registration", False ) + self.allow_underscore_prefixed_localpart = config.get( + "allow_underscore_prefixed_localpart", False + ) + session_lifetime = config.get("session_lifetime") if session_lifetime is not None: session_lifetime = self.parse_duration(session_lifetime) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index ecfea175c7..3e86349981 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -159,7 +159,10 @@ class RegistrationHandler: if not localpart: raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) - if localpart[0] == "_": + if ( + localpart[0] == "_" + and not self.hs.config.registration.allow_underscore_prefixed_localpart + ): raise SynapseError( 400, "User ID may not begin with _", Codes.INVALID_USERNAME ) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 92487692db..dda389c08b 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -588,6 +588,29 @@ class RegistrationTestCase(unittest.HomeserverTestCase): d = self.store.is_support_user(user_id) self.assertFalse(self.get_success(d)) + def test_underscore_localpart_rejected_by_default(self) -> None: + for invalid_user_id in ("_", "_prefixed"): + with self.subTest(invalid_user_id=invalid_user_id): + self.get_failure( + self.handler.register_user(localpart=invalid_user_id), + SynapseError, + ) + + @override_config( + { + "allow_underscore_prefixed_localpart": True, + } + ) + def test_underscore_localpart_allowed_if_configured(self) -> None: + for valid_user_id in ("_", "_prefixed"): + with self.subTest(valid_user_id=valid_user_id): + user_id = self.get_success( + self.handler.register_user( + localpart=valid_user_id, + ), + ) + self.assertEqual(user_id, f"@{valid_user_id}:test") + def test_invalid_user_id(self) -> None: invalid_user_id = "^abcd" self.get_failure( From f5ed52c1e24b5649d7d81dd9690bb606e387961b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 May 2025 12:43:24 +0100 Subject: [PATCH 314/691] Move index creation to background update (#18439) Follow on from #18375. This prevents blocking startup on creating the index, which can take a while --------- Co-authored-by: Devon Hudson --- changelog.d/18439.bugfix | 1 + synapse/storage/databases/main/sliding_sync.py | 8 ++++++++ ...snapshot_idx.sql => 04_ss_membership_snapshot_idx.sql} | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18439.bugfix rename synapse/storage/schema/main/delta/92/{03_ss_membership_snapshot_idx.sql => 04_ss_membership_snapshot_idx.sql} (73%) diff --git a/changelog.d/18439.bugfix b/changelog.d/18439.bugfix new file mode 100644 index 0000000000..5ee9bda474 --- /dev/null +++ b/changelog.d/18439.bugfix @@ -0,0 +1 @@ +Fix startup being blocked on creating a new index. Introduced in v1.130.0rc1. diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index a287fd2a3f..6a62b11d1e 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -68,6 +68,14 @@ class SlidingSyncStore(SQLBaseStore): columns=("membership_event_id",), ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_membership_snapshots_user_id_stream_ordering", + index_name="sliding_sync_membership_snapshots_user_id_stream_ordering", + table="sliding_sync_membership_snapshots", + columns=("user_id", "event_stream_ordering"), + replaces_index="sliding_sync_membership_snapshots_user_id", + ) + async def get_latest_bump_stamp_for_room( self, room_id: str, diff --git a/synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql b/synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql similarity index 73% rename from synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql rename to synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql index c694203f95..6f5b7cb06e 100644 --- a/synapse/storage/schema/main/delta/92/03_ss_membership_snapshot_idx.sql +++ b/synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql @@ -12,5 +12,5 @@ -- . -- So we can fetch all rooms for a given user sorted by stream order -DROP INDEX IF EXISTS sliding_sync_membership_snapshots_user_id; -CREATE INDEX IF NOT EXISTS sliding_sync_membership_snapshots_user_id ON sliding_sync_membership_snapshots(user_id, event_stream_ordering); +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9204, 'sliding_sync_membership_snapshots_user_id_stream_ordering', '{}'); From 0afdc0fc7ffe2cb7a2fa6d47f22b685cbacc7223 Mon Sep 17 00:00:00 2001 From: Stanislav Kazantsev Date: Thu, 15 May 2025 23:18:17 +0600 Subject: [PATCH 315/691] remove room without listeners from Notifier.room_to_user_streams (#18380) Co-authored-by: Andrew Morgan --- changelog.d/18380.misc | 1 + synapse/notifier.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/18380.misc diff --git a/changelog.d/18380.misc b/changelog.d/18380.misc new file mode 100644 index 0000000000..5d6017be26 --- /dev/null +++ b/changelog.d/18380.misc @@ -0,0 +1 @@ +Fix a memory leak in `_NotifierUserStream`. \ No newline at end of file diff --git a/synapse/notifier.py b/synapse/notifier.py index 1914d0c914..6190432b87 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -158,6 +158,9 @@ class _NotifierUserStream: lst = notifier.room_to_user_streams.get(room, set()) lst.discard(self) + if not lst: + notifier.room_to_user_streams.pop(room, None) + notifier.user_to_user_stream.pop(self.user_id) def count_listeners(self) -> int: From 74e2f028bbcaeb2a572d03e66334f3c671bffae2 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 19 May 2025 01:48:46 -0700 Subject: [PATCH 316/691] Fix admin redaction endpoint not redacting encrypted messages (#18434) --- changelog.d/18434.bugfix | 1 + synapse/handlers/admin.py | 2 +- tests/rest/admin/test_user.py | 55 ++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18434.bugfix diff --git a/changelog.d/18434.bugfix b/changelog.d/18434.bugfix new file mode 100644 index 0000000000..dd094c83e8 --- /dev/null +++ b/changelog.d/18434.bugfix @@ -0,0 +1 @@ +Fix admin redaction endpoint not redacting encrypted messages. \ No newline at end of file diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index f3e7790d43..971a74244f 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -445,7 +445,7 @@ class AdminHandler: user_id, room, limit, - ["m.room.member", "m.room.message"], + ["m.room.member", "m.room.message", "m.room.encrypted"], ) if not event_ids: # nothing to redact in this room diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index a35a250975..874c29c935 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -36,7 +36,13 @@ from twisted.test.proto_helpers import MemoryReactor from twisted.web.resource import Resource import synapse.rest.admin -from synapse.api.constants import ApprovalNoticeMedium, EventTypes, LoginType, UserTypes +from synapse.api.constants import ( + ApprovalNoticeMedium, + EventContentFields, + EventTypes, + LoginType, + UserTypes, +) from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError from synapse.api.room_versions import RoomVersions from synapse.media.filepath import MediaFilePaths @@ -5467,6 +5473,53 @@ class UserRedactionTestCase(unittest.HomeserverTestCase): # we originally sent 5 messages so 5 should be redacted self.assertEqual(len(original_message_ids), 0) + def test_redact_redacts_encrypted_messages(self) -> None: + """ + Test that user's encrypted messages are redacted + """ + encrypted_room = self.helper.create_room_as( + self.admin, tok=self.admin_tok, room_version="7" + ) + self.helper.send_state( + encrypted_room, + EventTypes.RoomEncryption, + {EventContentFields.ENCRYPTION_ALGORITHM: "m.megolm.v1.aes-sha2"}, + tok=self.admin_tok, + ) + # join room send some messages + originals = [] + join = self.helper.join(encrypted_room, self.bad_user, tok=self.bad_user_tok) + originals.append(join["event_id"]) + for _ in range(15): + res = self.helper.send_event( + encrypted_room, "m.room.encrypted", {}, tok=self.bad_user_tok + ) + originals.append(res["event_id"]) + + # redact user's events + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": []}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + matched = [] + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{encrypted_room}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + for event in channel.json_body["chunk"]: + for event_id in originals: + if event["type"] == "m.room.redaction" and event["redacts"] == event_id: + matched.append(event_id) + self.assertEqual(len(matched), len(originals)) + class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase): servlets = [ From 078cefd014806a67249ddb59b5976c7e93227f37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 09:51:08 +0100 Subject: [PATCH 317/691] Bump actions/setup-python from 5.5.0 to 5.6.0 (#18398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.5.0 to 5.6.0.
Release notes

Sourced from actions/setup-python's releases.

v5.6.0

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v5...v5.6.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=5.5.0&new-version=5.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/tests.yml | 12 ++++++------ 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 616ef0f9cf..1f4f79598a 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.4.17' - name: Setup python - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 05ae608d06..930c71a8b4 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 366bb4cddb..ee0dac3beb 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -86,7 +86,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - run: pip install .[all,test] diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 31b9147e98..1668ad81d2 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index e03c9d2bd5..572d73e6ad 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' - id: set-distros @@ -74,7 +74,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' @@ -132,7 +132,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -177,7 +177,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.10' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a7e35a0ece..848240f68e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" @@ -112,7 +112,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -192,7 +192,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -279,7 +279,7 @@ jobs: if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -327,7 +327,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - id: get-matrix @@ -414,7 +414,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.9' From 7d4c3b64e34571f3ace10fa7e33d07853bf16d67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 09:51:52 +0100 Subject: [PATCH 318/691] Bump docker/build-push-action from 6.15.0 to 6.16.0 (#18397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.15.0 to 6.16.0.
Release notes

Sourced from docker/build-push-action's releases.

v6.16.0

Full Changelog: https://github.com/docker/build-push-action/compare/v6.15.0...v6.16.0

Commits
  • 14487ce Merge pull request #1343 from crazy-max/fix-no-default-attest
  • 0ec9126 Merge pull request #1366 from crazy-max/pr-assign-author
  • b749522 pr-assign-author workflow
  • c566248 Merge pull request #1363 from crazy-max/fix-codecov
  • 13275dd ci: fix missing source for codecov
  • 67dc78b Merge pull request #1361 from mschoettle/patch-1
  • 0760504 docs: add validating build configuration example
  • 1c198f4 chore: update generated content
  • 288d9e2 handle no default attestations env var
  • 88844b9 Merge pull request #1353 from crazy-max/summary-secret-keys
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/build-push-action&package-manager=github_actions&previous-version=6.15.0&new-version=6.16.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c617753c7a..009089db3a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -72,7 +72,7 @@ jobs: - name: Build and push all platforms id: build-and-push - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0 + uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 with: push: true labels: | From fa4a00a2da753a52dde582c0f56e3ea6567bd53b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 19 May 2025 11:52:05 +0100 Subject: [PATCH 319/691] Check for `CREATE/DROP INDEX` in schema deltas (#18440) As these should be background updates. --- changelog.d/18440.misc | 1 + scripts-dev/check_schema_delta.py | 131 +++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 31 deletions(-) create mode 100644 changelog.d/18440.misc diff --git a/changelog.d/18440.misc b/changelog.d/18440.misc new file mode 100644 index 0000000000..6aaa6dde5c --- /dev/null +++ b/changelog.d/18440.misc @@ -0,0 +1 @@ +Add lint to ensure we don't add a `CREATE/DROP INDEX` in a schema delta. diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index 467be96fdf..454784c3ae 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # Check that no schema deltas have been added to the wrong version. +# +# Also checks that schema deltas do not try and create or drop indices. import re from typing import Any, Dict, List @@ -9,6 +11,13 @@ import click import git SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") +INDEX_CREATION_REGEX = re.compile(r"CREATE .*INDEX .*ON ([a-z_]+)", flags=re.IGNORECASE) +INDEX_DELETION_REGEX = re.compile(r"DROP .*INDEX ([a-z_]+)", flags=re.IGNORECASE) +TABLE_CREATION_REGEX = re.compile(r"CREATE .*TABLE ([a-z_]+)", flags=re.IGNORECASE) + +# The base branch we want to check against. We use the main development branch +# on the assumption that is what we are developing against. +DEVELOP_BRANCH = "develop" @click.command() @@ -20,6 +29,9 @@ SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") help="Always output ANSI colours", ) def main(force_colors: bool) -> None: + # Return code. Set to non-zero when we encounter an error + return_code = 0 + click.secho( "+++ Checking schema deltas are in the right folder", fg="green", @@ -30,17 +42,17 @@ def main(force_colors: bool) -> None: click.secho("Updating repo...") repo = git.Repo() - repo.remote().fetch() + repo.remote().fetch(refspec=DEVELOP_BRANCH) click.secho("Getting current schema version...") - r = repo.git.show("origin/develop:synapse/storage/schema/__init__.py") + r = repo.git.show(f"origin/{DEVELOP_BRANCH}:synapse/storage/schema/__init__.py") locals: Dict[str, Any] = {} exec(r, locals) current_schema_version = locals["SCHEMA_VERSION"] - diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None) + diffs: List[git.Diff] = repo.remote().refs[DEVELOP_BRANCH].commit.diff(None) # Get the schema version of the local file to check against current schema on develop with open("synapse/storage/schema/__init__.py") as file: @@ -53,7 +65,7 @@ def main(force_colors: bool) -> None: # local schema version must be +/-1 the current schema version on develop if abs(local_schema_version - current_schema_version) != 1: click.secho( - "The proposed schema version has diverged more than one version from develop, please fix!", + f"The proposed schema version has diverged more than one version from {DEVELOP_BRANCH}, please fix!", fg="red", bold=True, color=force_colors, @@ -67,21 +79,28 @@ def main(force_colors: bool) -> None: click.secho(f"Current schema version: {current_schema_version}") seen_deltas = False - bad_files = [] + bad_delta_files = [] + changed_delta_files = [] for diff in diffs: - if not diff.new_file or diff.b_path is None: + if diff.b_path is None: + # We don't lint deleted files. continue match = SCHEMA_FILE_REGEX.match(diff.b_path) if not match: continue + changed_delta_files.append(diff.b_path) + + if not diff.new_file: + continue + seen_deltas = True _, delta_version, _ = match.groups() if delta_version != str(current_schema_version): - bad_files.append(diff.b_path) + bad_delta_files.append(diff.b_path) if not seen_deltas: click.secho( @@ -92,41 +111,91 @@ def main(force_colors: bool) -> None: ) return - if not bad_files: + if bad_delta_files: + bad_delta_files.sort() + + click.secho( + "Found deltas in the wrong folder!", + fg="red", + bold=True, + color=force_colors, + ) + + for f in bad_delta_files: + click.secho( + f"\t{f}", + fg="red", + bold=True, + color=force_colors, + ) + + click.secho() + click.secho( + f"Please move these files to delta/{current_schema_version}/", + fg="red", + bold=True, + color=force_colors, + ) + + else: click.secho( f"All deltas are in the correct folder: {current_schema_version}!", fg="green", bold=True, color=force_colors, ) - return - bad_files.sort() + # Make sure we process them in order. This sort works because deltas are numbered + # and delta files are also numbered in order. + changed_delta_files.sort() - click.secho( - "Found deltas in the wrong folder!", - fg="red", - bold=True, - color=force_colors, - ) + # Now check that we're not trying to create or drop indices. If we want to + # do that they should be in background updates. The exception is when we + # create indices on tables we've just created. + created_tables = set() + for delta_file in changed_delta_files: + with open(delta_file) as fd: + delta_lines = fd.readlines() - for f in bad_files: - click.secho( - f"\t{f}", - fg="red", - bold=True, - color=force_colors, - ) + for line in delta_lines: + # Strip SQL comments + line = line.split("--", maxsplit=1)[0] - click.secho() - click.secho( - f"Please move these files to delta/{current_schema_version}/", - fg="red", - bold=True, - color=force_colors, - ) + # Check and track any tables we create + match = TABLE_CREATION_REGEX.search(line) + if match: + table_name = match.group(1) + created_tables.add(table_name) - click.get_current_context().exit(1) + # Check for dropping indices, these are always banned + match = INDEX_DELETION_REGEX.search(line) + if match: + clause = match.group() + + click.secho( + f"Found delta with index deletion: '{clause}' in {delta_file}\nThese should be in background updates.", + fg="red", + bold=True, + color=force_colors, + ) + return_code = 1 + + # Check for index creation, which is only allowed for tables we've + # created. + match = INDEX_CREATION_REGEX.search(line) + if match: + clause = match.group() + table_name = match.group(1) + if table_name not in created_tables: + click.secho( + f"Found delta with index creation: '{clause}' in {delta_file}\nThese should be in background updates.", + fg="red", + bold=True, + color=force_colors, + ) + return_code = 1 + + click.get_current_context().exit(return_code) if __name__ == "__main__": From b3b24c69fcbdb67de04b0388aa104d43780ba88f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 13:04:15 +0100 Subject: [PATCH 320/691] Bump pyo3-log from 0.12.3 to 0.12.4 (#18453) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27a2e26be5..13156e67b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079e412e909af5d6be7c04a7f29f6a2837a080410e1c529c9dee2c367383db4" +checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264" dependencies = [ "arc-swap", "log", From cd1a3ac584d9a353e24e42354ae71028654f7f61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 13:06:11 +0100 Subject: [PATCH 321/691] Bump authlib from 1.5.1 to 1.5.2 (#18452) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 55 +++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7190d0f788..cf3ca18611 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -34,15 +34,15 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a [[package]] name = "authlib" -version = "1.5.1" +version = "1.5.2" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"}, - {file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"}, + {file = "authlib-1.5.2-py2.py3-none-any.whl", hash = "sha256:8804dd4402ac5e4a0435ac49e0b6e19e395357cfa632a3f624dcb4f6df13b4b1"}, + {file = "authlib-1.5.2.tar.gz", hash = "sha256:fe85ec7e50c5f86f1e2603518bb3b4f632985eb4a355e52256530790e326c512"}, ] [package.dependencies] @@ -451,7 +451,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -494,7 +494,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -544,7 +544,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -890,7 +890,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1028,7 +1028,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1044,7 +1044,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -1330,7 +1330,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1551,7 +1551,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1720,7 +1720,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1728,6 +1728,7 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1740,7 +1741,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1756,7 +1757,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1979,7 +1980,7 @@ description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"user-search\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"user-search\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -2028,7 +2029,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2088,7 +2089,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2113,7 +2114,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2141,7 +2142,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2505,7 +2506,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2689,7 +2690,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2705,7 +2706,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2767,7 +2768,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, @@ -2901,7 +2902,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3244,7 +3245,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From afeb0e01c552216d0d987cd504aab440b07bdb10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 13:06:45 +0100 Subject: [PATCH 322/691] Bump pyopenssl from 25.0.0 to 25.1.0 (#18450) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index cf3ca18611..54ddad3bdd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2064,18 +2064,18 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" -version = "25.0.0" +version = "25.1.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "pyOpenSSL-25.0.0-py3-none-any.whl", hash = "sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90"}, - {file = "pyopenssl-25.0.0.tar.gz", hash = "sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16"}, + {file = "pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab"}, + {file = "pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b"}, ] [package.dependencies] -cryptography = ">=41.0.5,<45" +cryptography = ">=41.0.5,<46" typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] From 17e6b32966670550c5fb4f232b390dd25ec77759 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 13:07:24 +0100 Subject: [PATCH 323/691] Bump docker/build-push-action from 6.16.0 to 6.17.0 (#18449) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 009089db3a..feeadf170d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -72,7 +72,7 @@ jobs: - name: Build and push all platforms id: build-and-push - uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 + uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0 with: push: true labels: | From 67920c0aca6bb23f76390fa4827ce2e6e1889547 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 19 May 2025 13:36:30 +0100 Subject: [PATCH 324/691] Fix up the topological ordering for events above `MAX_DEPTH` (#18447) Synapse previously did not correctly cap the max depth of an event to the max canonical json int. This can cause ordering issues for any events that were sent locally at the time. This background update goes and correctly caps the topological ordering to the new `MAX_DEPTH`. c.f. GHSA-v56r-hwv5-mxg6 --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18447.bugfix | 1 + .../databases/main/events_bg_updates.py | 82 ++++++++- .../main/delta/92/05_fixup_max_depth_cap.sql | 17 ++ synapse/types/storage/__init__.py | 2 + tests/storage/test_events_bg_updates.py | 157 ++++++++++++++++++ 5 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18447.bugfix create mode 100644 synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql create mode 100644 tests/storage/test_events_bg_updates.py diff --git a/changelog.d/18447.bugfix b/changelog.d/18447.bugfix new file mode 100644 index 0000000000..578be1ffe9 --- /dev/null +++ b/changelog.d/18447.bugfix @@ -0,0 +1 @@ +Fix the ordering of local messages in rooms that were affected by [GHSA-v56r-hwv5-mxg6](https://github.com/advisories/GHSA-v56r-hwv5-mxg6). diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 4b0bdd79c6..5c83a9f779 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -24,7 +24,12 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast import attr -from synapse.api.constants import EventContentFields, Membership, RelationTypes +from synapse.api.constants import ( + MAX_DEPTH, + EventContentFields, + Membership, + RelationTypes, +) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase, make_event_from_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause @@ -311,6 +316,10 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS self._sliding_sync_membership_snapshots_fix_forgotten_column_bg_update, ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, self.fixup_max_depth_cap_bg_update + ) + # We want this to run on the main database at startup before we start processing # events. # @@ -2547,6 +2556,77 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS return num_rows + async def fixup_max_depth_cap_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: + """Fixes the topological ordering for events that have a depth greater + than MAX_DEPTH. This should fix /messages ordering oddities.""" + + room_id_bound = progress.get("room_id", "") + + def redo_max_depth_bg_update_txn(txn: LoggingTransaction) -> Tuple[bool, int]: + txn.execute( + """ + SELECT room_id, room_version FROM rooms + WHERE room_id > ? + ORDER BY room_id + LIMIT ? + """, + (room_id_bound, batch_size), + ) + + # Find the next room ID to process, with a relevant room version. + room_ids: List[str] = [] + max_room_id: Optional[str] = None + for room_id, room_version_str in txn: + max_room_id = room_id + + # We only want to process rooms with a known room version that + # has strict canonical json validation enabled. + room_version = KNOWN_ROOM_VERSIONS.get(room_version_str) + if room_version and room_version.strict_canonicaljson: + room_ids.append(room_id) + + if max_room_id is None: + # The query did not return any rooms, so we are done. + return True, 0 + + # Update the progress to the last room ID we pulled from the DB, + # this ensures we always make progress. + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, + progress={"room_id": max_room_id}, + ) + + if not room_ids: + # There were no rooms in this batch that required the fix. + return False, 0 + + clause, list_args = make_in_list_sql_clause( + self.database_engine, "room_id", room_ids + ) + sql = f""" + UPDATE events SET topological_ordering = ? + WHERE topological_ordering > ? AND {clause} + """ + args = [MAX_DEPTH, MAX_DEPTH] + args.extend(list_args) + txn.execute(sql, args) + + return False, len(room_ids) + + done, num_rooms = await self.db_pool.runInteraction( + "redo_max_depth_bg_update", redo_max_depth_bg_update_txn + ) + + if done: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP + ) + + return num_rooms + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql b/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql new file mode 100644 index 0000000000..c1ebf8b58b --- /dev/null +++ b/synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql @@ -0,0 +1,17 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Background update that fixes any events with a topological ordering above the +-- MAX_DEPTH value. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9205, 'fixup_max_depth_cap', '{}'); diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index e03ff7ffc8..378a15e038 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -52,3 +52,5 @@ class _BackgroundUpdates: MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = ( "mark_unreferenced_state_groups_for_deletion_bg_update" ) + + FIXUP_MAX_DEPTH_CAP = "fixup_max_depth_cap" diff --git a/tests/storage/test_events_bg_updates.py b/tests/storage/test_events_bg_updates.py new file mode 100644 index 0000000000..ecdf413e3b --- /dev/null +++ b/tests/storage/test_events_bg_updates.py @@ -0,0 +1,157 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from typing import Dict + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import MAX_DEPTH +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.server import HomeServer +from synapse.util import Clock + +from tests.unittest import HomeserverTestCase + + +class TestFixupMaxDepthCapBgUpdate(HomeserverTestCase): + """Test the background update that caps topological_ordering at MAX_DEPTH.""" + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = self.hs.get_datastores().main + self.db_pool = self.store.db_pool + + self.room_id = "!testroom:example.com" + + # Reinsert the background update as it was already run at the start of + # the test. + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "fixup_max_depth_cap", + "progress_json": "{}", + }, + ) + ) + + def create_room(self, room_version: RoomVersion) -> Dict[str, int]: + """Create a room with a known room version and insert events. + + Returns the set of event IDs that exceed MAX_DEPTH and + their depth. + """ + + # Create a room with a specific room version + self.get_success( + self.db_pool.simple_insert( + table="rooms", + values={ + "room_id": self.room_id, + "room_version": room_version.identifier, + }, + ) + ) + + # Insert events with some depths exceeding MAX_DEPTH + event_id_to_depth: Dict[str, int] = {} + for depth in range(MAX_DEPTH - 5, MAX_DEPTH + 5): + event_id = f"$event{depth}:example.com" + event_id_to_depth[event_id] = depth + + self.get_success( + self.db_pool.simple_insert( + table="events", + values={ + "event_id": event_id, + "room_id": self.room_id, + "topological_ordering": depth, + "depth": depth, + "type": "m.test", + "sender": "@user:test", + "processed": True, + "outlier": False, + }, + ) + ) + + return event_id_to_depth + + def test_fixup_max_depth_cap_bg_update(self) -> None: + """Test that the background update correctly caps topological_ordering + at MAX_DEPTH.""" + + event_id_to_depth = self.create_room(RoomVersions.V6) + + # Run the background update + progress = {"room_id": ""} + batch_size = 10 + num_rooms = self.get_success( + self.store.fixup_max_depth_cap_bg_update(progress, batch_size) + ) + + # Verify the number of rooms processed + self.assertEqual(num_rooms, 1) + + # Verify that the topological_ordering of events has been capped at + # MAX_DEPTH + rows = self.get_success( + self.db_pool.simple_select_list( + table="events", + keyvalues={"room_id": self.room_id}, + retcols=["event_id", "topological_ordering"], + ) + ) + + for event_id, topological_ordering in rows: + if event_id_to_depth[event_id] >= MAX_DEPTH: + # Events with a depth greater than or equal to MAX_DEPTH should + # be capped at MAX_DEPTH. + self.assertEqual(topological_ordering, MAX_DEPTH) + else: + # Events with a depth less than MAX_DEPTH should remain + # unchanged. + self.assertEqual(topological_ordering, event_id_to_depth[event_id]) + + def test_fixup_max_depth_cap_bg_update_old_room_version(self) -> None: + """Test that the background update does not cap topological_ordering for + rooms with old room versions.""" + + event_id_to_depth = self.create_room(RoomVersions.V5) + + # Run the background update + progress = {"room_id": ""} + batch_size = 10 + num_rooms = self.get_success( + self.store.fixup_max_depth_cap_bg_update(progress, batch_size) + ) + + # Verify the number of rooms processed + self.assertEqual(num_rooms, 0) + + # Verify that the topological_ordering of events has been capped at + # MAX_DEPTH + rows = self.get_success( + self.db_pool.simple_select_list( + table="events", + keyvalues={"room_id": self.room_id}, + retcols=["event_id", "topological_ordering"], + ) + ) + + # Assert that the topological_ordering of events has not been changed + # from their depth. + self.assertDictEqual(event_id_to_depth, dict(rows)) From 1f4ae2f9eb94808f651b683b4650092015ec39e1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 19 May 2025 17:50:02 +0100 Subject: [PATCH 325/691] Allow only requiring a field be present in an SSO response, rather than specifying a required value (#18454) --- changelog.d/18454.misc | 1 + .../configuration/config_documentation.md | 10 ++- synapse/config/sso.py | 7 +- tests/handlers/test_oidc.py | 77 ++++++++++++++++++- 4 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 changelog.d/18454.misc diff --git a/changelog.d/18454.misc b/changelog.d/18454.misc new file mode 100644 index 0000000000..892fbd1d94 --- /dev/null +++ b/changelog.d/18454.misc @@ -0,0 +1 @@ +Allow checking only for the existence of a field in an SSO provider's response, rather than requiring the value(s) to check. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e688bc5cd8..3927b9ca14 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3782,17 +3782,23 @@ match particular values in the OIDC userinfo. The requirements can be listed und ```yaml attribute_requirements: - attribute: family_name - value: "Stephensson" + one_of: ["Stephensson", "Smith"] - attribute: groups value: "admin" + # If `value` or `one_of` are not specified, the attribute only needs + # to exist, regardless of value. + - attribute: picture ``` + +`attribute` is a required field, while `value` and `one_of` are optional. + All of the listed attributes must match for the login to be permitted. Additional attributes can be added to userinfo by expanding the `scopes` section of the OIDC config to retrieve additional information from the OIDC provider. If the OIDC claim is a list, then the attribute must match any value in the list. Otherwise, it must exactly match the value of the claim. Using the example -above, the `family_name` claim MUST be "Stephensson", but the `groups` +above, the `family_name` claim MUST be either "Stephensson" or "Smith", but the `groups` claim MUST contain "admin". Example configuration: diff --git a/synapse/config/sso.py b/synapse/config/sso.py index 97b85e47ea..cf27a7ee13 100644 --- a/synapse/config/sso.py +++ b/synapse/config/sso.py @@ -43,8 +43,7 @@ class SsoAttributeRequirement: """Object describing a single requirement for SSO attributes.""" attribute: str - # If neither value nor one_of is given, the attribute must simply exist. This is - # only true for CAS configs which use a different JSON schema than the one below. + # If neither `value` nor `one_of` is given, the attribute must simply exist. value: Optional[str] = None one_of: Optional[List[str]] = None @@ -56,10 +55,6 @@ class SsoAttributeRequirement: "one_of": {"type": "array", "items": {"type": "string"}}, }, "required": ["attribute"], - "oneOf": [ - {"required": ["value"]}, - {"required": ["one_of"]}, - ], } diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index e5f31d57ca..ff8e3c5cb6 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -1453,7 +1453,7 @@ class OidcHandlerTestCase(HomeserverTestCase): } } ) - def test_attribute_requirements_one_of(self) -> None: + def test_attribute_requirements_one_of_succeeds(self) -> None: """Test that auth succeeds if userinfo attribute has multiple values and CONTAINS required value""" # userinfo with "test": ["bar"] attribute should succeed. userinfo = { @@ -1475,6 +1475,81 @@ class OidcHandlerTestCase(HomeserverTestCase): auth_provider_session_id=None, ) + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [ + {"attribute": "test", "one_of": ["foo", "bar"]} + ], + } + } + ) + def test_attribute_requirements_one_of_fails(self) -> None: + """Test that auth fails if userinfo attribute has multiple values yet + DOES NOT CONTAIN a required value + """ + # userinfo with "test": ["something else"] attribute should fail. + userinfo = { + "sub": "tester", + "username": "tester", + "test": ["something else"], + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + self.complete_sso_login.assert_not_called() + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [{"attribute": "test"}], + } + } + ) + def test_attribute_requirements_does_not_exist(self) -> None: + """OIDC login fails if the required attribute does not exist in the OIDC userinfo response.""" + # userinfo lacking "test" attribute should fail. + userinfo = { + "sub": "tester", + "username": "tester", + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + self.complete_sso_login.assert_not_called() + + @override_config( + { + "oidc_config": { + **DEFAULT_CONFIG, + "attribute_requirements": [{"attribute": "test"}], + } + } + ) + def test_attribute_requirements_exist(self) -> None: + """OIDC login succeeds if the required attribute exist (regardless of value) + in the OIDC userinfo response. + """ + # userinfo with "test" attribute and random value should succeed. + userinfo = { + "sub": "tester", + "username": "tester", + "test": random_string(5), # value does not matter + } + request, _ = self.start_authorization(userinfo) + self.get_success(self.handler.handle_oidc_callback(request)) + + # check that the auth handler got called as expected + self.complete_sso_login.assert_called_once_with( + "@tester:test", + self.provider.idp_id, + request, + ANY, + None, + new_user=True, + auth_provider_session_id=None, + ) + @override_config( { "oidc_config": { From 303c5c4daa6986a91ab4632bd4df0448199b1813 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 May 2025 12:03:10 +0100 Subject: [PATCH 326/691] Bump setuptools from 72.1.0 to 78.1.1 (#18461) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 54ddad3bdd..3c53dfb376 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2584,20 +2584,24 @@ tests = ["coverage[toml] (>=5.0.2)", "pytest"] [[package]] name = "setuptools" -version = "72.1.0" +version = "78.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-78.1.1-py3-none-any.whl", hash = "sha256:c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561"}, + {file = "setuptools-78.1.1.tar.gz", hash = "sha256:fcc17fd9cd898242f6b4adfaca46137a9edef687f43e6f78469692a5e70d851d"}, ] [package.extras] -core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (<0.4) ; platform_system == \"Windows\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.3.2) ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "setuptools-rust" From a6cb3533db77ebeb6b7ed86fb3d3dd86c046f4a4 Mon Sep 17 00:00:00 2001 From: Strac Consulting Engineers Pty Ltd Date: Tue, 20 May 2025 23:31:05 +1000 Subject: [PATCH 327/691] Update postgres.md (#18445) --- changelog.d/18445.doc | 1 + docs/postgres.md | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/18445.doc diff --git a/changelog.d/18445.doc b/changelog.d/18445.doc new file mode 100644 index 0000000000..1e05a791b2 --- /dev/null +++ b/changelog.d/18445.doc @@ -0,0 +1 @@ +Add advice for upgrading between major PostgreSQL versions to the database documentation. diff --git a/docs/postgres.md b/docs/postgres.md index 51670667e8..d51f54c722 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -100,6 +100,14 @@ database: keepalives_count: 3 ``` +## Postgresql major version upgrades + +Postgres uses separate directories for database locations between major versions (typically `/var/lib/postgresql//main`). + +Therefore, it is recommended to stop Synapse and other services (MAS, etc) before upgrading Postgres major versions. + +It is also strongly recommended to [back up](./usage/administration/backups.md#database) your database beforehand to ensure no data loss arising from a failed upgrade. + ## Backups Don't forget to [back up](./usage/administration/backups.md#database) your database! From 9d43bec3268d9a454fe992f25edfc013a50fb9cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 May 2025 15:23:30 +0100 Subject: [PATCH 328/691] Bump ruff from 0.7.3 to 0.11.10 (#18451) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Morgan Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18451.misc | 1 + poetry.lock | 40 +++++++++---------- pyproject.toml | 2 +- synapse/_scripts/synapse_port_db.py | 2 +- synapse/_scripts/synctl.py | 6 +-- synapse/app/generic_worker.py | 3 +- synapse/app/homeserver.py | 3 +- synapse/config/tls.py | 3 +- synapse/event_auth.py | 3 +- synapse/handlers/e2e_keys.py | 12 ++---- synapse/handlers/federation.py | 6 +-- synapse/handlers/message.py | 22 +++++----- synapse/handlers/sso.py | 6 +-- synapse/http/matrixfederationclient.py | 6 +-- synapse/http/proxyagent.py | 12 +++--- synapse/http/servlet.py | 6 +-- synapse/module_api/__init__.py | 6 +-- synapse/replication/http/_base.py | 6 +-- synapse/replication/tcp/streams/events.py | 6 +-- synapse/rest/admin/__init__.py | 3 +- synapse/rest/client/receipts.py | 4 +- synapse/rest/client/rendezvous.py | 6 +-- synapse/rest/client/transactions.py | 6 +-- synapse/storage/background_updates.py | 12 +++--- synapse/storage/controllers/persist_events.py | 3 +- synapse/storage/databases/main/client_ips.py | 12 +++--- synapse/storage/databases/main/deviceinbox.py | 6 +-- synapse/storage/databases/main/devices.py | 2 +- synapse/storage/databases/main/events.py | 27 ++++++------- .../storage/databases/main/events_worker.py | 6 +-- .../databases/main/monthly_active_users.py | 24 +++++------ .../storage/databases/main/purge_events.py | 3 +- .../storage/databases/main/state_deltas.py | 6 +-- synapse/storage/databases/main/tags.py | 5 +-- .../storage/databases/main/user_directory.py | 6 +-- synapse/storage/databases/state/bg_updates.py | 3 +- synapse/storage/schema/main/delta/25/fts.py | 3 +- synapse/storage/schema/main/delta/27/ts.py | 3 +- .../schema/main/delta/31/search_update.py | 3 +- .../schema/main/delta/33/event_fields.py | 3 +- synapse/types/__init__.py | 3 +- synapse/types/state.py | 2 +- synapse/util/iterutils.py | 4 +- .../test_federation_out_of_band_membership.py | 18 ++++----- tests/handlers/test_user_directory.py | 4 +- tests/http/test_matrixfederationclient.py | 8 +--- tests/media/test_media_storage.py | 4 +- tests/replication/tcp/streams/test_events.py | 2 +- tests/rest/admin/test_room.py | 2 +- tests/rest/admin/test_user.py | 4 +- .../sliding_sync/test_rooms_timeline.py | 6 +-- tests/rest/client/test_media.py | 2 +- tests/rest/client/utils.py | 6 +-- tests/rest/media/test_url_preview.py | 2 +- tests/server.py | 6 +-- tests/storage/test_base.py | 2 +- tests/storage/test_devices.py | 6 +-- tests/storage/test_event_federation.py | 2 +- tests/test_state.py | 2 +- tests/test_utils/logging_setup.py | 2 +- 60 files changed, 178 insertions(+), 206 deletions(-) create mode 100644 changelog.d/18451.misc diff --git a/changelog.d/18451.misc b/changelog.d/18451.misc new file mode 100644 index 0000000000..593e83eb7f --- /dev/null +++ b/changelog.d/18451.misc @@ -0,0 +1 @@ +Bump ruff from 0.7.3 to 0.11.10. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 3c53dfb376..ada0646215 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2440,30 +2440,30 @@ files = [ [[package]] name = "ruff" -version = "0.7.3" +version = "0.11.10" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.7.3-py3-none-linux_armv6l.whl", hash = "sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344"}, - {file = "ruff-0.7.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0"}, - {file = "ruff-0.7.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:37d0b619546103274e7f62643d14e1adcbccb242efda4e4bdb9544d7764782e9"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59f0c3ee4d1a6787614e7135b72e21024875266101142a09a61439cb6e38a5"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44eb93c2499a169d49fafd07bc62ac89b1bc800b197e50ff4633aed212569299"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d0242ce53f3a576c35ee32d907475a8d569944c0407f91d207c8af5be5dae4e"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6b6224af8b5e09772c2ecb8dc9f3f344c1aa48201c7f07e7315367f6dd90ac29"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c50f95a82b94421c964fae4c27c0242890a20fe67d203d127e84fbb8013855f5"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f3eff9961b5d2644bcf1616c606e93baa2d6b349e8aa8b035f654df252c8c67"}, - {file = "ruff-0.7.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8963cab06d130c4df2fd52c84e9f10d297826d2e8169ae0c798b6221be1d1d2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:61b46049d6edc0e4317fb14b33bd693245281a3007288b68a3f5b74a22a0746d"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:10ebce7696afe4644e8c1a23b3cf8c0f2193a310c18387c06e583ae9ef284de2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3f36d56326b3aef8eeee150b700e519880d1aab92f471eefdef656fd57492aa2"}, - {file = "ruff-0.7.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5d024301109a0007b78d57ab0ba190087b43dce852e552734ebf0b0b85e4fb16"}, - {file = "ruff-0.7.3-py3-none-win32.whl", hash = "sha256:4ba81a5f0c5478aa61674c5a2194de8b02652f17addf8dfc40c8937e6e7d79fc"}, - {file = "ruff-0.7.3-py3-none-win_amd64.whl", hash = "sha256:588a9ff2fecf01025ed065fe28809cd5a53b43505f48b69a1ac7707b1b7e4088"}, - {file = "ruff-0.7.3-py3-none-win_arm64.whl", hash = "sha256:1713e2c5545863cdbfe2cbce21f69ffaf37b813bfd1fb3b90dc9a6f1963f5a8c"}, - {file = "ruff-0.7.3.tar.gz", hash = "sha256:e1d1ba2e40b6e71a61b063354d04be669ab0d39c352461f3d789cac68b54a313"}, + {file = "ruff-0.11.10-py3-none-linux_armv6l.whl", hash = "sha256:859a7bfa7bc8888abbea31ef8a2b411714e6a80f0d173c2a82f9041ed6b50f58"}, + {file = "ruff-0.11.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:968220a57e09ea5e4fd48ed1c646419961a0570727c7e069842edd018ee8afed"}, + {file = "ruff-0.11.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1067245bad978e7aa7b22f67113ecc6eb241dca0d9b696144256c3a879663bca"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4854fd09c7aed5b1590e996a81aeff0c9ff51378b084eb5a0b9cd9518e6cff2"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b4564e9f99168c0f9195a0fd5fa5928004b33b377137f978055e40008a082c5"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b6a9cc5b62c03cc1fea0044ed8576379dbaf751d5503d718c973d5418483641"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:607ecbb6f03e44c9e0a93aedacb17b4eb4f3563d00e8b474298a201622677947"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b3a522fa389402cd2137df9ddefe848f727250535c70dafa840badffb56b7a4"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f071b0deed7e9245d5820dac235cbdd4ef99d7b12ff04c330a241ad3534319f"}, + {file = "ruff-0.11.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a60e3a0a617eafba1f2e4186d827759d65348fa53708ca547e384db28406a0b"}, + {file = "ruff-0.11.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da8ec977eaa4b7bf75470fb575bea2cb41a0e07c7ea9d5a0a97d13dbca697bf2"}, + {file = "ruff-0.11.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ddf8967e08227d1bd95cc0851ef80d2ad9c7c0c5aab1eba31db49cf0a7b99523"}, + {file = "ruff-0.11.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5a94acf798a82db188f6f36575d80609072b032105d114b0f98661e1679c9125"}, + {file = "ruff-0.11.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3afead355f1d16d95630df28d4ba17fb2cb9c8dfac8d21ced14984121f639bad"}, + {file = "ruff-0.11.10-py3-none-win32.whl", hash = "sha256:dc061a98d32a97211af7e7f3fa1d4ca2fcf919fb96c28f39551f35fc55bdbc19"}, + {file = "ruff-0.11.10-py3-none-win_amd64.whl", hash = "sha256:5cc725fbb4d25b0f185cb42df07ab6b76c4489b4bfb740a175f3a59c70e8a224"}, + {file = "ruff-0.11.10-py3-none-win_arm64.whl", hash = "sha256:ef69637b35fb8b210743926778d0e45e1bffa850a7c61e428c6b971549b5f5d1"}, + {file = "ruff-0.11.10.tar.gz", hash = "sha256:d522fb204b4959909ecac47da02830daec102eeb100fb50ea9554818d47a5fa6"}, ] [[package]] @@ -3394,4 +3394,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d" +content-hash = "522f5bacf5610646876452e0e397038dd5c959692d2ab76214431bff78562d01" diff --git a/pyproject.toml b/pyproject.toml index 914a5804aa..6ce05805a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -320,7 +320,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.7.3" +ruff = "0.11.10" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 438b2ff8a0..573c70696e 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -1065,7 +1065,7 @@ class Porter: def get_sent_table_size(txn: LoggingTransaction) -> int: txn.execute( - "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,) + "SELECT count(*) FROM sent_transactions WHERE ts >= ?", (yesterday,) ) result = txn.fetchone() assert result is not None diff --git a/synapse/_scripts/synctl.py b/synapse/_scripts/synctl.py index 688df9485c..2e2aa27a17 100755 --- a/synapse/_scripts/synctl.py +++ b/synapse/_scripts/synctl.py @@ -292,9 +292,9 @@ def main() -> None: for key in worker_config: if key == "worker_app": # But we allow worker_app continue - assert not key.startswith( - "worker_" - ), "Main process cannot use worker_* config" + assert not key.startswith("worker_"), ( + "Main process cannot use worker_* config" + ) else: worker_pidfile = worker_config["worker_pid_file"] worker_cache_factor = worker_config.get("synctl_cache_factor") diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index f495d5b7e4..75c65ccc0d 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -287,8 +287,7 @@ class GenericWorkerServer(HomeServer): elif listener.type == "metrics": if not self.config.metrics.enable_metrics: logger.warning( - "Metrics listener configured, but " - "enable_metrics is not True!" + "Metrics listener configured, but enable_metrics is not True!" ) else: if isinstance(listener, TCPListenerConfig): diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6da2194cf7..e027b5eaea 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -289,8 +289,7 @@ class SynapseHomeServer(HomeServer): elif listener.type == "metrics": if not self.config.metrics.enable_metrics: logger.warning( - "Metrics listener configured, but " - "enable_metrics is not True!" + "Metrics listener configured, but enable_metrics is not True!" ) else: if isinstance(listener, TCPListenerConfig): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 51dc15eb61..a48d81fdc3 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -108,8 +108,7 @@ class TlsConfig(Config): # Raise an error if this option has been specified without any # corresponding certificates. raise ConfigError( - "federation_custom_ca_list specified without " - "any certificate files" + "federation_custom_ca_list specified without any certificate files" ) certs = [] diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 5ecf493f98..5999c264dc 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -986,8 +986,7 @@ def _check_power_levels( if old_level == user_level: raise AuthError( 403, - "You don't have permission to remove ops level equal " - "to your own", + "You don't have permission to remove ops level equal to your own", ) # Check if the old and new levels are greater than the user level diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 540995e062..f2b2e30bf4 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -1163,7 +1163,7 @@ class E2eKeysHandler: devices = devices[user_id] except SynapseError as e: failure = _exception_to_failure(e) - failures[user_id] = {device: failure for device in signatures.keys()} + failures[user_id] = dict.fromkeys(signatures.keys(), failure) return signature_list, failures for device_id, device in signatures.items(): @@ -1303,7 +1303,7 @@ class E2eKeysHandler: except SynapseError as e: failure = _exception_to_failure(e) for user, devicemap in signatures.items(): - failures[user] = {device_id: failure for device_id in devicemap.keys()} + failures[user] = dict.fromkeys(devicemap.keys(), failure) return signature_list, failures for target_user, devicemap in signatures.items(): @@ -1344,9 +1344,7 @@ class E2eKeysHandler: # other devices were signed -- mark those as failures logger.debug("upload signature: too many devices specified") failure = _exception_to_failure(NotFoundError("Unknown device")) - failures[target_user] = { - device: failure for device in other_devices - } + failures[target_user] = dict.fromkeys(other_devices, failure) if user_signing_key_id in master_key.get("signatures", {}).get( user_id, {} @@ -1367,9 +1365,7 @@ class E2eKeysHandler: except SynapseError as e: failure = _exception_to_failure(e) if device_id is None: - failures[target_user] = { - device_id: failure for device_id in devicemap.keys() - } + failures[target_user] = dict.fromkeys(devicemap.keys(), failure) else: failures.setdefault(target_user, {})[device_id] = failure diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 17dd4af13e..b1640e3246 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1312,9 +1312,9 @@ class FederationHandler: if state_key is not None: # the event was not rejected (get_event raises a NotFoundError for rejected # events) so the state at the event should include the event itself. - assert ( - state_map.get((event.type, state_key)) == event.event_id - ), "State at event did not include event itself" + assert state_map.get((event.type, state_key)) == event.event_id, ( + "State at event did not include event itself" + ) # ... but we need the state *before* that event if "replaces_state" in event.unsigned: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 52c61cfa54..ff6eb5a514 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -143,9 +143,9 @@ class MessageHandler: elif membership == Membership.LEAVE: key = (event_type, state_key) # If the membership is not JOIN, then the event ID should exist. - assert ( - membership_event_id is not None - ), "check_user_in_room_or_world_readable returned invalid data" + assert membership_event_id is not None, ( + "check_user_in_room_or_world_readable returned invalid data" + ) room_state = await self._state_storage_controller.get_state_for_events( [membership_event_id], StateFilter.from_types([key]) ) @@ -242,9 +242,9 @@ class MessageHandler: room_state = await self.store.get_events(state_ids.values()) elif membership == Membership.LEAVE: # If the membership is not JOIN, then the event ID should exist. - assert ( - membership_event_id is not None - ), "check_user_in_room_or_world_readable returned invalid data" + assert membership_event_id is not None, ( + "check_user_in_room_or_world_readable returned invalid data" + ) room_state_events = ( await self._state_storage_controller.get_state_for_events( [membership_event_id], state_filter=state_filter @@ -1266,12 +1266,14 @@ class EventCreationHandler: # Allow an event to have empty list of prev_event_ids # only if it has auth_event_ids. or auth_event_ids - ), "Attempting to create a non-m.room.create event with no prev_events or auth_event_ids" + ), ( + "Attempting to create a non-m.room.create event with no prev_events or auth_event_ids" + ) else: # we now ought to have some prev_events (unless it's a create event). - assert ( - builder.type == EventTypes.Create or prev_event_ids - ), "Attempting to create a non-m.room.create event with no prev_events" + assert builder.type == EventTypes.Create or prev_event_ids, ( + "Attempting to create a non-m.room.create event with no prev_events" + ) if for_batch: assert prev_event_ids is not None diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 9c0d665461..07827cf95b 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -1192,9 +1192,9 @@ class SsoHandler: """ # It is expected that this is the main process. - assert isinstance( - self._device_handler, DeviceHandler - ), "revoking SSO sessions can only be called on the main process" + assert isinstance(self._device_handler, DeviceHandler), ( + "revoking SSO sessions can only be called on the main process" + ) # Invalidate any running user-mapping sessions to_delete = [] diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index f6d2536957..88bf98045c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -425,9 +425,9 @@ class MatrixFederationHttpClient: ) else: proxy_authorization_secret = hs.config.worker.worker_replication_secret - assert ( - proxy_authorization_secret is not None - ), "`worker_replication_secret` must be set when using `outbound_federation_restricted_to` (used to authenticate requests across workers)" + assert proxy_authorization_secret is not None, ( + "`worker_replication_secret` must be set when using `outbound_federation_restricted_to` (used to authenticate requests across workers)" + ) federation_proxy_credentials = BearerProxyCredentials( proxy_authorization_secret.encode("ascii") ) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index fd16ee42dd..6817199035 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -173,9 +173,9 @@ class ProxyAgent(_AgentBase): self._federation_proxy_endpoint: Optional[IStreamClientEndpoint] = None self._federation_proxy_credentials: Optional[ProxyCredentials] = None if federation_proxy_locations: - assert ( - federation_proxy_credentials is not None - ), "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + assert federation_proxy_credentials is not None, ( + "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + ) endpoints: List[IStreamClientEndpoint] = [] for federation_proxy_location in federation_proxy_locations: @@ -302,9 +302,9 @@ class ProxyAgent(_AgentBase): parsed_uri.scheme == b"matrix-federation" and self._federation_proxy_endpoint ): - assert ( - self._federation_proxy_credentials is not None - ), "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + assert self._federation_proxy_credentials is not None, ( + "`federation_proxy_credentials` are required when using `federation_proxy_locations`" + ) # Set a Proxy-Authorization header if headers is None: diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index ed6ab08336..47d8bd5eaf 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -582,9 +582,9 @@ def parse_enum( is not one of those allowed values. """ # Assert the enum values are strings. - assert all( - isinstance(e.value, str) for e in E - ), "parse_enum only works with string values" + assert all(isinstance(e.value, str) for e in E), ( + "parse_enum only works with string values" + ) str_value = parse_string( request, name, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index bf9532e891..7834da759c 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -894,9 +894,9 @@ class ModuleApi: Raises: synapse.api.errors.AuthError: the access token is invalid """ - assert isinstance( - self._device_handler, DeviceHandler - ), "invalidate_access_token can only be called on the main process" + assert isinstance(self._device_handler, DeviceHandler), ( + "invalidate_access_token can only be called on the main process" + ) # see if the access token corresponds to a device user_info = yield defer.ensureDeferred( diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 9aa8d90bfe..0002538680 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -128,9 +128,9 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): # We reserve `instance_name` as a parameter to sending requests, so we # assert here that sub classes don't try and use the name. - assert ( - "instance_name" not in self.PATH_ARGS - ), "`instance_name` is a reserved parameter name" + assert "instance_name" not in self.PATH_ARGS, ( + "`instance_name` is a reserved parameter name" + ) assert ( "instance_name" not in signature(self.__class__._serialize_payload).parameters diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index ea0803dfc2..05b55fb033 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -200,9 +200,9 @@ class EventsStream(_StreamFromIdGen): # we rely on get_all_new_forward_event_rows strictly honouring the limit, so # that we know it is safe to just take upper_limit = event_rows[-1][0]. - assert ( - len(event_rows) <= target_row_count - ), "get_all_new_forward_event_rows did not honour row limit" + assert len(event_rows) <= target_row_count, ( + "get_all_new_forward_event_rows did not honour row limit" + ) # if we hit the limit on event_updates, there's no point in going beyond the # last stream_id in the batch for the other sources. diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index b1335fed66..e55cdc0470 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -207,8 +207,7 @@ class PurgeHistoryRestServlet(RestServlet): (stream, topo, _event_id) = r token = "t%d-%d" % (topo, stream) logger.info( - "[purge] purging up to token %s (received_ts %i => " - "stream_ordering %i)", + "[purge] purging up to token %s (received_ts %i => stream_ordering %i)", token, ts, stream_ordering, diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index 89203dc45a..4bf93f485c 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -39,9 +39,7 @@ logger = logging.getLogger(__name__) class ReceiptRestServlet(RestServlet): PATTERNS = client_patterns( - "/rooms/(?P[^/]*)" - "/receipt/(?P[^/]*)" - "/(?P[^/]*)$" + "/rooms/(?P[^/]*)/receipt/(?P[^/]*)/(?P[^/]*)$" ) CATEGORY = "Receipts requests" diff --git a/synapse/rest/client/rendezvous.py b/synapse/rest/client/rendezvous.py index 02f166b4ea..a1808847f0 100644 --- a/synapse/rest/client/rendezvous.py +++ b/synapse/rest/client/rendezvous.py @@ -44,9 +44,9 @@ class MSC4108DelegationRendezvousServlet(RestServlet): redirection_target: Optional[str] = ( hs.config.experimental.msc4108_delegation_endpoint ) - assert ( - redirection_target is not None - ), "Servlet is only registered if there is a delegation target" + assert redirection_target is not None, ( + "Servlet is only registered if there is a delegation target" + ) self.endpoint = redirection_target.encode("utf-8") async def on_POST(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index f791904168..1a57996aec 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -94,9 +94,9 @@ class HttpTransactionCache: # (appservice and guest users), but does not cover access tokens minted # by the admin API. Use the access token ID instead. else: - assert ( - requester.access_token_id is not None - ), "Requester must have an access_token_id" + assert requester.access_token_id is not None, ( + "Requester must have an access_token_id" + ) return (path, "user_admin", requester.access_token_id) def fetch_or_execute_request( diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index a02b4cc9ce..d170bbddaa 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -739,9 +739,9 @@ class BackgroundUpdater: c.execute(sql) async def updater(progress: JsonDict, batch_size: int) -> int: - assert isinstance( - self.db_pool.engine, engines.PostgresEngine - ), "validate constraint background update registered for non-Postres database" + assert isinstance(self.db_pool.engine, engines.PostgresEngine), ( + "validate constraint background update registered for non-Postres database" + ) logger.info("Validating constraint %s to %s", constraint_name, table) await self.db_pool.runWithConnection(runner) @@ -900,9 +900,9 @@ class BackgroundUpdater: on the table. Used to iterate over the table. """ - assert isinstance( - self.db_pool.engine, engines.PostgresEngine - ), "validate constraint background update registered for non-Postres database" + assert isinstance(self.db_pool.engine, engines.PostgresEngine), ( + "validate constraint background update registered for non-Postres database" + ) async def updater(progress: JsonDict, batch_size: int) -> int: return await self.validate_constraint_and_delete_in_background( diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 7963905479..f5131fe291 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -870,8 +870,7 @@ class EventsPersistenceStorageController: # This should only happen for outlier events. if not ev.internal_metadata.is_outlier(): raise Exception( - "Context for new event %s has no state " - "group" % (ev.event_id,) + "Context for new event %s has no state group" % (ev.event_id,) ) continue if ctx.state_group_deltas: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index e8c322ab5c..69008804bd 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -650,9 +650,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke @wrap_as_background_process("update_client_ips") async def _update_client_ips_batch(self) -> None: - assert ( - self._update_on_this_worker - ), "This worker is not designated to update client IPs" + assert self._update_on_this_worker, ( + "This worker is not designated to update client IPs" + ) # If the DB pool has already terminated, don't try updating if not self.db_pool.is_running(): @@ -671,9 +671,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke txn: LoggingTransaction, to_update: Mapping[Tuple[str, str, str], Tuple[str, Optional[str], int]], ) -> None: - assert ( - self._update_on_this_worker - ), "This worker is not designated to update client IPs" + assert self._update_on_this_worker, ( + "This worker is not designated to update client IPs" + ) # Keys and values for the `user_ips` upsert. user_ips_keys = [] diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 0612b82b9b..d47833655d 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -200,9 +200,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): to_stream_id=to_stream_id, ) - assert ( - last_processed_stream_id == to_stream_id - ), "Expected _get_device_messages to process all to-device messages up to `to_stream_id`" + assert last_processed_stream_id == to_stream_id, ( + "Expected _get_device_messages to process all to-device messages up to `to_stream_id`" + ) return user_id_device_id_to_messages diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 3f0b2f5d84..6191f22cd6 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1092,7 +1092,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ), ) - results: Dict[str, Optional[str]] = {user_id: None for user_id in user_ids} + results: Dict[str, Optional[str]] = dict.fromkeys(user_ids) results.update(rows) return results diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 26fbc1a483..b7cc0433e7 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -246,9 +246,9 @@ class PersistEventsStore: self.is_mine_id = hs.is_mine_id # This should only exist on instances that are configured to write - assert ( - hs.get_instance_name() in hs.config.worker.writers.events - ), "Can only instantiate EventsStore on master" + assert hs.get_instance_name() in hs.config.worker.writers.events, ( + "Can only instantiate EventsStore on master" + ) # Since we have been configured to write, we ought to have id generators, # rather than id trackers. @@ -465,9 +465,9 @@ class PersistEventsStore: missing_membership_event_ids ) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_membership_event_ids - ), missing_membership_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_membership_event_ids, ( + missing_membership_event_ids.difference(remaining_events.keys()) + ) membership_event_map.update(remaining_events) for ( @@ -534,9 +534,9 @@ class PersistEventsStore: missing_state_event_ids ) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_state_event_ids - ), missing_state_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_state_event_ids, ( + missing_state_event_ids.difference(remaining_events.keys()) + ) for event in remaining_events.values(): current_state_map[(event.type, event.state_key)] = event @@ -644,9 +644,9 @@ class PersistEventsStore: if missing_event_ids: remaining_events = await self.store.get_events(missing_event_ids) # There shouldn't be any missing events - assert ( - remaining_events.keys() == missing_event_ids - ), missing_event_ids.difference(remaining_events.keys()) + assert remaining_events.keys() == missing_event_ids, ( + missing_event_ids.difference(remaining_events.keys()) + ) for event in remaining_events.values(): current_state_map[(event.type, event.state_key)] = event @@ -3448,8 +3448,7 @@ class PersistEventsStore: # Delete all these events that we've already fetched and now know that their # prev events are the new backwards extremeties. query = ( - "DELETE FROM event_backward_extremities" - " WHERE event_id = ? AND room_id = ?" + "DELETE FROM event_backward_extremities WHERE event_id = ? AND room_id = ?" ) backward_extremity_tuples_to_remove = [ (ev.event_id, ev.room_id) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 82b2ad4408..3db4460f57 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -824,9 +824,9 @@ class EventsWorkerStore(SQLBaseStore): if missing_events_ids: - async def get_missing_events_from_cache_or_db() -> ( - Dict[str, EventCacheEntry] - ): + async def get_missing_events_from_cache_or_db() -> Dict[ + str, EventCacheEntry + ]: """Fetches the events in `missing_event_ids` from the database. Also creates entries in `self._current_event_fetches` to allow diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 8e948c5e8d..659ee13d71 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -304,9 +304,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn: threepids: List of threepid dicts to reserve """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # XXX what is this function trying to achieve? It upserts into # monthly_active_users for each *registered* reserved mau user, but why? @@ -340,9 +340,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): Args: user_id: user to add/update """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # Support user never to be included in MAU stats. Note I can't easily call this # from upsert_monthly_active_user_txn because then I need a _txn form of @@ -379,9 +379,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn: user_id: user to add/update """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) # Am consciously deciding to lock the table on the basis that is ought # never be a big table and alternative approaches (batching multiple @@ -409,9 +409,9 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): Args: user_id: the user_id to query """ - assert ( - self._update_on_this_worker - ), "This worker is not designated to update MAUs" + assert self._update_on_this_worker, ( + "This worker is not designated to update MAUs" + ) if self._limit_usage_by_mau or self._mau_stats_only: # Trial users and guests should not be included as part of MAU group diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index ebdeb8fbd7..a11f522f03 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -199,8 +199,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore): # Update backward extremeties txn.execute_batch( - "INSERT INTO event_backward_extremities (room_id, event_id)" - " VALUES (?, ?)", + "INSERT INTO event_backward_extremities (room_id, event_id) VALUES (?, ?)", [(room_id, event_id) for (event_id,) in new_backwards_extrems], ) diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index b90f667da8..00f87cc3a1 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -98,9 +98,9 @@ class StateDeltasStore(SQLBaseStore): prev_stream_id = int(prev_stream_id) # check we're not going backwards - assert ( - prev_stream_id <= max_stream_id - ), f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}" + assert prev_stream_id <= max_stream_id, ( + f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}" + ) if not self._curr_state_delta_stream_cache.has_any_entity_changed( prev_stream_id diff --git a/synapse/storage/databases/main/tags.py b/synapse/storage/databases/main/tags.py index 44f395f315..97b190bccc 100644 --- a/synapse/storage/databases/main/tags.py +++ b/synapse/storage/databases/main/tags.py @@ -274,10 +274,7 @@ class TagsWorkerStore(AccountDataWorkerStore): assert isinstance(self._account_data_id_gen, AbstractStreamIdGenerator) def remove_tag_txn(txn: LoggingTransaction, next_id: int) -> None: - sql = ( - "DELETE FROM room_tags " - " WHERE user_id = ? AND room_id = ? AND tag = ?" - ) + sql = "DELETE FROM room_tags WHERE user_id = ? AND room_id = ? AND tag = ?" txn.execute(sql, (user_id, room_id, tag)) self._update_revision_txn(txn, user_id, room_id, next_id) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 391f0dd638..2b867cdb6e 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -582,9 +582,9 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): retry_counter: number of failures in refreshing the profile so far. Used for exponential backoff calculations. """ - assert not self.hs.is_mine_id( - user_id - ), "Can't mark a local user as a stale remote user." + assert not self.hs.is_mine_id(user_id), ( + "Can't mark a local user as a stale remote user." + ) server_name = UserID.from_string(user_id).domain diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 95fd0ae73a..5b594fe8dd 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -396,8 +396,7 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): return True, count txn.execute( - "SELECT state_group FROM state_group_edges" - " WHERE state_group = ?", + "SELECT state_group FROM state_group_edges WHERE state_group = ?", (state_group,), ) diff --git a/synapse/storage/schema/main/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py index b050cc16a7..c01c1325cb 100644 --- a/synapse/storage/schema/main/delta/25/fts.py +++ b/synapse/storage/schema/main/delta/25/fts.py @@ -75,8 +75,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_search", progress_json)) diff --git a/synapse/storage/schema/main/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py index d7f360b6e6..e6e73e1b77 100644 --- a/synapse/storage/schema/main/delta/27/ts.py +++ b/synapse/storage/schema/main/delta/27/ts.py @@ -55,8 +55,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_origin_server_ts", progress_json)) diff --git a/synapse/storage/schema/main/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py index 0e65c9a841..46355122bb 100644 --- a/synapse/storage/schema/main/delta/31/search_update.py +++ b/synapse/storage/schema/main/delta/31/search_update.py @@ -59,8 +59,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_search_order", progress_json)) diff --git a/synapse/storage/schema/main/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py index 9c02aeda88..53d215337e 100644 --- a/synapse/storage/schema/main/delta/33/event_fields.py +++ b/synapse/storage/schema/main/delta/33/event_fields.py @@ -55,8 +55,7 @@ def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> progress_json = json.dumps(progress) sql = ( - "INSERT into background_updates (update_name, progress_json)" - " VALUES (?, ?)" + "INSERT into background_updates (update_name, progress_json) VALUES (?, ?)" ) cur.execute(sql, ("event_fields_sender_url", progress_json)) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index e9cdd19868..5549f3c9f8 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -889,8 +889,7 @@ class MultiWriterStreamToken(AbstractMultiWriterStreamToken): def __str__(self) -> str: instances = ", ".join(f"{k}: {v}" for k, v in sorted(self.instance_map.items())) return ( - f"MultiWriterStreamToken(stream: {self.stream}, " - f"instances: {{{instances}}})" + f"MultiWriterStreamToken(stream: {self.stream}, instances: {{{instances}}})" ) diff --git a/synapse/types/state.py b/synapse/types/state.py index e641215f18..6420e050a5 100644 --- a/synapse/types/state.py +++ b/synapse/types/state.py @@ -462,7 +462,7 @@ class StateFilter: new_types.update({state_type: set() for state_type in minus_wildcards}) # insert the plus wildcards - new_types.update({state_type: None for state_type in plus_wildcards}) + new_types.update(dict.fromkeys(plus_wildcards)) # insert the specific state keys for state_type, state_key in plus_state_keys: diff --git a/synapse/util/iterutils.py b/synapse/util/iterutils.py index ff6adeb716..0a6a30aab2 100644 --- a/synapse/util/iterutils.py +++ b/synapse/util/iterutils.py @@ -114,7 +114,7 @@ def sorted_topologically( # This is implemented by Kahn's algorithm. - degree_map = {node: 0 for node in nodes} + degree_map = dict.fromkeys(nodes, 0) reverse_graph: Dict[T, Set[T]] = {} for node, edges in graph.items(): @@ -164,7 +164,7 @@ def sorted_topologically_batched( persisted. """ - degree_map = {node: 0 for node in nodes} + degree_map = dict.fromkeys(nodes, 0) reverse_graph: Dict[T, Set[T]] = {} for node, edges in graph.items(): diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index a4a266cf06..f77b8fe300 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -65,20 +65,20 @@ def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase] if isinstance(required_state, list): for state_event_dict in required_state: # Yell because we're in a test and this is unexpected - assert isinstance( - state_event_dict, dict - ), "`required_state` should be a list of event dicts" + assert isinstance(state_event_dict, dict), ( + "`required_state` should be a list of event dicts" + ) event_type = state_event_dict["type"] event_state_key = state_event_dict["state_key"] # Yell because we're in a test and this is unexpected - assert isinstance( - event_type, str - ), "Each event in `required_state` should have a string `type`" - assert isinstance( - event_state_key, str - ), "Each event in `required_state` should have a string `state_key`" + assert isinstance(event_type, str), ( + "Each event in `required_state` should have a string `type`" + ) + assert isinstance(event_state_key, str), ( + "Each event in `required_state` should have a string `state_key`" + ) state_map[(event_type, event_state_key)] = make_event_from_dict( state_event_dict diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index a9e9d7d7ea..b12ffc3665 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -1178,10 +1178,10 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): for use_numeric in [False, True]: if use_numeric: prefix1 = f"{i}" - prefix2 = f"{i+1}" + prefix2 = f"{i + 1}" else: prefix1 = f"a{i}" - prefix2 = f"a{i+1}" + prefix2 = f"a{i + 1}" local_user_1 = self.register_user(f"user{char}{prefix1}", "password") local_user_2 = self.register_user(f"user{char}{prefix2}", "password") diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py index e34df54e13..d5ebf10eac 100644 --- a/tests/http/test_matrixfederationclient.py +++ b/tests/http/test_matrixfederationclient.py @@ -436,8 +436,7 @@ class FederationClientTests(HomeserverTestCase): # Send it the HTTP response client.dataReceived( - b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" - b"Server: Fake\r\n\r\n" + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nServer: Fake\r\n\r\n" ) # Push by enough to time it out @@ -691,10 +690,7 @@ class FederationClientTests(HomeserverTestCase): # Send it a huge HTTP response protocol.dataReceived( - b"HTTP/1.1 200 OK\r\n" - b"Server: Fake\r\n" - b"Content-Type: application/json\r\n" - b"\r\n" + b"HTTP/1.1 200 OK\r\nServer: Fake\r\nContent-Type: application/json\r\n\r\n" ) self.pump() diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 35e16a99ba..31dc32d67e 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -250,9 +250,7 @@ small_cmyk_jpeg = TestImage( ) small_lossless_webp = TestImage( - unhexlify( - b"524946461a000000574542505650384c0d0000002f0000001007" b"1011118888fe0700" - ), + unhexlify(b"524946461a000000574542505650384c0d0000002f00000010071011118888fe0700"), b"image/webp", b".webp", ) diff --git a/tests/replication/tcp/streams/test_events.py b/tests/replication/tcp/streams/test_events.py index fdc74efb5a..2a0189a4e1 100644 --- a/tests/replication/tcp/streams/test_events.py +++ b/tests/replication/tcp/streams/test_events.py @@ -324,7 +324,7 @@ class EventsStreamTestCase(BaseStreamTestCase): pls = self.helper.get_state( self.room_id, EventTypes.PowerLevels, tok=self.user_tok ) - pls["users"].update({u: 50 for u in user_ids}) + pls["users"].update(dict.fromkeys(user_ids, 50)) self.helper.send_state( self.room_id, EventTypes.PowerLevels, diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 1d44106bd7..165d175ab2 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -1312,7 +1312,7 @@ class RoomTestCase(unittest.HomeserverTestCase): # Check that response json body contains a "rooms" key self.assertTrue( "rooms" in channel.json_body, - msg="Response body does not " "contain a 'rooms' key", + msg="Response body does not contain a 'rooms' key", ) # Check that 3 rooms were returned diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 874c29c935..f09f66da00 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -3901,9 +3901,7 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): image_data1 = SMALL_PNG # Resolution: 1×1, MIME type: image/gif, Extension: gif, Size: 35 B image_data2 = unhexlify( - b"47494638376101000100800100000000" - b"ffffff2c00000000010001000002024c" - b"01003b" + b"47494638376101000100800100000000ffffff2c00000000010001000002024c01003b" ) # Resolution: 1×1, MIME type: image/bmp, Extension: bmp, Size: 54 B image_data3 = unhexlify( diff --git a/tests/rest/client/sliding_sync/test_rooms_timeline.py b/tests/rest/client/sliding_sync/test_rooms_timeline.py index 2293994793..535420209b 100644 --- a/tests/rest/client/sliding_sync/test_rooms_timeline.py +++ b/tests/rest/client/sliding_sync/test_rooms_timeline.py @@ -309,8 +309,8 @@ class SlidingSyncRoomsTimelineTestCase(SlidingSyncBase): self.assertEqual( response_body["rooms"][room_id1]["limited"], False, - f'Our `timeline_limit` was {sync_body["lists"]["foo-list"]["timeline_limit"]} ' - + f'and {len(response_body["rooms"][room_id1]["timeline"])} events were returned in the timeline. ' + f"Our `timeline_limit` was {sync_body['lists']['foo-list']['timeline_limit']} " + + f"and {len(response_body['rooms'][room_id1]['timeline'])} events were returned in the timeline. " + str(response_body["rooms"][room_id1]), ) # Check to make sure the latest events are returned @@ -387,7 +387,7 @@ class SlidingSyncRoomsTimelineTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["limited"], True, f"Our `timeline_limit` was {timeline_limit} " - + f'and {len(response_body["rooms"][room_id1]["timeline"])} events were returned in the timeline. ' + + f"and {len(response_body['rooms'][room_id1]['timeline'])} events were returned in the timeline. " + str(response_body["rooms"][room_id1]), ) # Check to make sure that the "live" and historical events are returned diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 1ea2a5c884..9ad8ecf1cd 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1006,7 +1006,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): data = base64.b64encode(SMALL_PNG) end_content = ( - b"" b'' b"" + b'' ) % (data,) channel = self.make_request( diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index 53f1782d59..280486da08 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -716,9 +716,9 @@ class RestHelper: "/login", content={"type": "m.login.token", "token": login_token}, ) - assert ( - channel.code == expected_status - ), f"unexpected status in response: {channel.code}" + assert channel.code == expected_status, ( + f"unexpected status in response: {channel.code}" + ) return channel.json_body def auth_via_oidc( diff --git a/tests/rest/media/test_url_preview.py b/tests/rest/media/test_url_preview.py index 103d7662d9..2a7bee19f9 100644 --- a/tests/rest/media/test_url_preview.py +++ b/tests/rest/media/test_url_preview.py @@ -878,7 +878,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): data = base64.b64encode(SMALL_PNG) end_content = ( - b"" b'' b"" + b'' ) % (data,) channel = self.make_request( diff --git a/tests/server.py b/tests/server.py index 84ed9f68eb..f01708b77f 100644 --- a/tests/server.py +++ b/tests/server.py @@ -225,9 +225,9 @@ class FakeChannel: new_headers.addRawHeader(k, v) headers = new_headers - assert isinstance( - headers, Headers - ), f"headers are of the wrong type: {headers!r}" + assert isinstance(headers, Headers), ( + f"headers are of the wrong type: {headers!r}" + ) self.result["headers"] = headers diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 9420d03841..11313fc933 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -349,7 +349,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): ) self.mock_txn.execute.assert_called_once_with( - "UPDATE tablename SET colC = ?, colD = ? WHERE" " colA = ? AND colB = ?", + "UPDATE tablename SET colC = ?, colD = ? WHERE colA = ? AND colB = ?", [3, 4, 1, 2], ) diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index ba01b038ab..74edca7523 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -211,9 +211,9 @@ class DeviceStoreTestCase(HomeserverTestCase): even if that means leaving an earlier batch one EDU short of the limit. """ - assert self.hs.is_mine_id( - "@user_id:test" - ), "Test not valid: this MXID should be considered local" + assert self.hs.is_mine_id("@user_id:test"), ( + "Test not valid: this MXID should be considered local" + ) self.get_success( self.store.set_e2e_cross_signing_key( diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 088f0d24f9..0500c68e9d 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -114,7 +114,7 @@ def get_all_topologically_sorted_orders( # This is implemented by Kahn's algorithm, and forking execution each time # we have a choice over which node to consider next. - degree_map = {node: 0 for node in nodes} + degree_map = dict.fromkeys(nodes, 0) reverse_graph: Dict[T, Set[T]] = {} for node, edges in graph.items(): diff --git a/tests/test_state.py b/tests/test_state.py index dce56fe78a..adb72b0730 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -149,7 +149,7 @@ class _DummyStore: async def get_partial_state_events( self, event_ids: Collection[str] ) -> Dict[str, bool]: - return {e: False for e in event_ids} + return dict.fromkeys(event_ids, False) async def get_state_group_delta( self, name: str diff --git a/tests/test_utils/logging_setup.py b/tests/test_utils/logging_setup.py index dd40c338d6..d58222a9f6 100644 --- a/tests/test_utils/logging_setup.py +++ b/tests/test_utils/logging_setup.py @@ -48,7 +48,7 @@ def setup_logging() -> None: # We exclude `%(asctime)s` from this format because the Twisted logger adds its own # timestamp - log_format = "%(name)s - %(lineno)d - " "%(levelname)s - %(request)s - %(message)s" + log_format = "%(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s" handler = ToTwistedHandler() formatter = logging.Formatter(log_format) From a36f3a6d875ce92e3cf6f3659f99ad71f8a0c069 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 20 May 2025 08:35:23 -0600 Subject: [PATCH 329/691] 1.130.0 --- CHANGES.md | 10 ++++++++++ changelog.d/18439.bugfix | 1 - changelog.d/18447.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/18439.bugfix delete mode 100644 changelog.d/18447.bugfix diff --git a/CHANGES.md b/CHANGES.md index a0a9d2f064..6837ad6bef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +# Synapse 1.130.0 (2025-05-20) + +### Bugfixes + +- Fix startup being blocked on creating a new index. Introduced in v1.130.0rc1. ([\#18439](https://github.com/element-hq/synapse/issues/18439)) +- Fix the ordering of local messages in rooms that were affected by [GHSA-v56r-hwv5-mxg6](https://github.com/advisories/GHSA-v56r-hwv5-mxg6). ([\#18447](https://github.com/element-hq/synapse/issues/18447)) + + + + # Synapse 1.130.0rc1 (2025-05-13) ### Features diff --git a/changelog.d/18439.bugfix b/changelog.d/18439.bugfix deleted file mode 100644 index 5ee9bda474..0000000000 --- a/changelog.d/18439.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix startup being blocked on creating a new index. Introduced in v1.130.0rc1. diff --git a/changelog.d/18447.bugfix b/changelog.d/18447.bugfix deleted file mode 100644 index 578be1ffe9..0000000000 --- a/changelog.d/18447.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the ordering of local messages in rooms that were affected by [GHSA-v56r-hwv5-mxg6](https://github.com/advisories/GHSA-v56r-hwv5-mxg6). diff --git a/debian/changelog b/debian/changelog index e3eb894851..56776a7d86 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.130.0) stable; urgency=medium + + * New Synapse release 1.130.0. + + -- Synapse Packaging team Tue, 20 May 2025 08:34:13 -0600 + matrix-synapse-py3 (1.130.0~rc1) stable; urgency=medium * New Synapse release 1.130.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 5f80d28344..7bc9fd4130 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.130.0rc1" +version = "1.130.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From f92c6455efbecaba1ddb1595e597aec0d7e4fb42 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 20 May 2025 08:46:37 -0600 Subject: [PATCH 330/691] Tweak changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 6837ad6bef..d29027bbfb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ### Bugfixes -- Fix startup being blocked on creating a new index. Introduced in v1.130.0rc1. ([\#18439](https://github.com/element-hq/synapse/issues/18439)) +- Fix startup being blocked on creating a new index that was introduced in v1.130.0rc1. ([\#18439](https://github.com/element-hq/synapse/issues/18439)) - Fix the ordering of local messages in rooms that were affected by [GHSA-v56r-hwv5-mxg6](https://github.com/advisories/GHSA-v56r-hwv5-mxg6). ([\#18447](https://github.com/element-hq/synapse/issues/18447)) From 4b1d9d5d0e3df7a3151c07f9d42b02dad13a27bf Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 20 May 2025 16:26:45 +0100 Subject: [PATCH 331/691] Add a unit test for the phone home stats (#18463) --- changelog.d/18463.misc | 1 + .../reporting_homeserver_usage_statistics.md | 6 +- synapse/app/phone_stats_home.py | 33 ++- tests/metrics/test_phone_home_stats.py | 263 ++++++++++++++++++ 4 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 changelog.d/18463.misc create mode 100644 tests/metrics/test_phone_home_stats.py diff --git a/changelog.d/18463.misc b/changelog.d/18463.misc new file mode 100644 index 0000000000..1264758d7c --- /dev/null +++ b/changelog.d/18463.misc @@ -0,0 +1 @@ +Add unit tests for homeserver usage statistics. \ No newline at end of file diff --git a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md index 4c0dbb5acd..a8a717e2a2 100644 --- a/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md +++ b/docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md @@ -30,7 +30,7 @@ The following statistics are sent to the configured reporting endpoint: | `python_version` | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`. | | `total_users` | int | The number of registered users on the homeserver. | | `total_nonbridged_users` | int | The number of users, excluding those created by an Application Service. | -| `daily_user_type_native` | int | The number of native users created in the last 24 hours. | +| `daily_user_type_native` | int | The number of native, non-guest users created in the last 24 hours. | | `daily_user_type_guest` | int | The number of guest users created in the last 24 hours. | | `daily_user_type_bridged` | int | The number of users created by Application Services in the last 24 hours. | | `total_room_count` | int | The total number of rooms present on the homeserver. | @@ -50,8 +50,8 @@ The following statistics are sent to the configured reporting endpoint: | `cache_factor` | int | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching. | | `event_cache_size` | int | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching. | | `database_engine` | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3. | -| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | -| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | +| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system. | +| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc. | [^1]: Native matrix users and guests are always counted. If the diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index f602bbbeea..bb450a235c 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -34,6 +34,22 @@ if TYPE_CHECKING: logger = logging.getLogger("synapse.app.homeserver") +ONE_MINUTE_SECONDS = 60 +ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS + +MILLISECONDS_PER_SECOND = 1000 + +INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS +""" +We wait 5 minutes to send the first set of stats as the server can be quite busy the +first few minutes +""" + +PHONE_HOME_INTERVAL_SECONDS = 3 * ONE_HOUR_SECONDS +""" +Phone home stats are sent every 3 hours +""" + # Contains the list of processes we will be monitoring # currently either 0 or 1 _stats_process: List[Tuple[int, "resource.struct_rusage"]] = [] @@ -185,12 +201,14 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, 5 * 60 * 1000 + hs.get_datastores().main.generate_user_daily_visits, + 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, ) # monthly active user limiting functionality clock.looping_call( - hs.get_datastores().main.reap_monthly_active_users, 1000 * 60 * 60 + hs.get_datastores().main.reap_monthly_active_users, + ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) hs.get_datastores().main.reap_monthly_active_users() @@ -221,7 +239,12 @@ def start_phone_stats_home(hs: "HomeServer") -> None: if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000, hs, stats) + clock.looping_call( + phone_stats_home, + PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + hs, + stats, + ) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process @@ -229,4 +252,6 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes - clock.call_later(5 * 60, phone_stats_home, hs, stats) + clock.call_later( + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats + ) diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py new file mode 100644 index 0000000000..5339d649df --- /dev/null +++ b/tests/metrics/test_phone_home_stats.py @@ -0,0 +1,263 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging +from unittest.mock import AsyncMock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.app.phone_stats_home import ( + PHONE_HOME_INTERVAL_SECONDS, + start_phone_stats_home, +) +from synapse.rest import admin, login, register, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest +from tests.server import ThreadedMemoryReactorClock + +TEST_REPORT_STATS_ENDPOINT = "https://fake.endpoint/stats" +TEST_SERVER_CONTEXT = "test-server-context" + + +class PhoneHomeStatsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + register.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + # Configure the homeserver to enable stats reporting. + config = self.default_config() + config["report_stats"] = True + config["report_stats_endpoint"] = TEST_REPORT_STATS_ENDPOINT + + # Configure the server context so we can check it ends up being reported + config["server_context"] = TEST_SERVER_CONTEXT + + # Allow guests to be registered + config["allow_guest_access"] = True + + hs = self.setup_test_homeserver(config=config) + + # Replace the proxied http client with a mock, so we can inspect outbound requests to + # the configured stats endpoint. + self.put_json_mock = AsyncMock(return_value={}) + hs.get_proxied_http_client().put_json = self.put_json_mock # type: ignore[method-assign] + return hs + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = homeserver.get_datastores().main + + # Wait for the background updates to add the database triggers that keep the + # `event_stats` table up-to-date. + self.wait_for_background_updates() + + # Force stats reporting to occur + start_phone_stats_home(hs=homeserver) + + super().prepare(reactor, clock, homeserver) + + def _get_latest_phone_home_stats(self) -> JsonDict: + # Wait for `phone_stats_home` to be called again + a healthy margin (50s). + self.reactor.advance(2 * PHONE_HOME_INTERVAL_SECONDS + 50) + + # Extract the reported stats from our http client mock + mock_calls = self.put_json_mock.call_args_list + report_stats_calls = [] + for call in mock_calls: + if call.args[0] == TEST_REPORT_STATS_ENDPOINT: + report_stats_calls.append(call) + + self.assertGreaterEqual( + (len(report_stats_calls)), + 1, + "Expected at-least one call to the report_stats endpoint", + ) + + # Extract the phone home stats from the call + phone_home_stats = report_stats_calls[0].args[1] + + return phone_home_stats + + def _perform_user_actions(self) -> None: + """ + Perform some actions on the homeserver that would bump the phone home + stats. + + This creates a few users (including a guest), a room, and sends some messages. + Expected number of events: + - 10 unencrypted messages + - 5 encrypted messages + - 24 total events (including room state, etc) + """ + + # Create some users + user_1_mxid = self.register_user( + username="test_user_1", + password="test", + ) + user_2_mxid = self.register_user( + username="test_user_2", + password="test", + ) + # Note: `self.register_user` does not support guest registration, and updating the + # Admin API it calls to add a new parameter would cause the `mac` parameter to fail + # in a backwards-incompatible manner. Hence, we make a manual request here. + _guest_user_mxid = self.make_request( + method="POST", + path="/_matrix/client/v3/register?kind=guest", + content={ + "username": "guest_user", + "password": "test", + }, + shorthand=False, + ) + + # Log in to each user + user_1_token = self.login(username=user_1_mxid, password="test") + user_2_token = self.login(username=user_2_mxid, password="test") + + # Create a room between the two users + room_1_id = self.helper.create_room_as( + is_public=False, + tok=user_1_token, + ) + + # Mark this room as end-to-end encrypted + self.helper.send_state( + room_id=room_1_id, + event_type="m.room.encryption", + body={ + "algorithm": "m.megolm.v1.aes-sha2", + "rotation_period_ms": 604800000, + "rotation_period_msgs": 100, + }, + state_key="", + tok=user_1_token, + ) + + # User 1 invites user 2 + self.helper.invite( + room=room_1_id, + src=user_1_mxid, + targ=user_2_mxid, + tok=user_1_token, + ) + + # User 2 joins + self.helper.join( + room=room_1_id, + user=user_2_mxid, + tok=user_2_token, + ) + + # User 1 sends 10 unencrypted messages + for _ in range(10): + self.helper.send( + room_id=room_1_id, + body="Zoinks Scoob! A message!", + tok=user_1_token, + ) + + # User 2 sends 5 encrypted "messages" + for _ in range(5): + self.helper.send_event( + room_id=room_1_id, + type="m.room.encrypted", + content={ + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "some_key", + "ciphertext": { + "some_key": { + "type": 0, + "body": "encrypted_payload", + }, + }, + }, + tok=user_2_token, + ) + + def test_phone_home_stats(self) -> None: + """ + Test that the phone home stats contain the stats we expect based on + the scenario carried out in `prepare` + """ + # Do things to bump the stats + self._perform_user_actions() + + # Wait for the stats to be reported + phone_home_stats = self._get_latest_phone_home_stats() + + self.assertEqual( + phone_home_stats["homeserver"], self.hs.config.server.server_name + ) + + self.assertTrue(isinstance(phone_home_stats["memory_rss"], int)) + self.assertTrue(isinstance(phone_home_stats["cpu_average"], int)) + + self.assertEqual(phone_home_stats["server_context"], TEST_SERVER_CONTEXT) + + self.assertTrue(isinstance(phone_home_stats["timestamp"], int)) + self.assertTrue(isinstance(phone_home_stats["uptime_seconds"], int)) + self.assertTrue(isinstance(phone_home_stats["python_version"], str)) + + # We expect only our test users to exist on the homeserver + self.assertEqual(phone_home_stats["total_users"], 3) + self.assertEqual(phone_home_stats["total_nonbridged_users"], 3) + self.assertEqual(phone_home_stats["daily_user_type_native"], 2) + self.assertEqual(phone_home_stats["daily_user_type_guest"], 1) + self.assertEqual(phone_home_stats["daily_user_type_bridged"], 0) + self.assertEqual(phone_home_stats["total_room_count"], 1) + self.assertEqual(phone_home_stats["daily_active_users"], 2) + self.assertEqual(phone_home_stats["monthly_active_users"], 2) + self.assertEqual(phone_home_stats["daily_active_rooms"], 1) + self.assertEqual(phone_home_stats["daily_active_e2ee_rooms"], 1) + self.assertEqual(phone_home_stats["daily_messages"], 10) + self.assertEqual(phone_home_stats["daily_e2ee_messages"], 5) + self.assertEqual(phone_home_stats["daily_sent_messages"], 10) + self.assertEqual(phone_home_stats["daily_sent_e2ee_messages"], 5) + + # Our users have not been around for >30 days, hence these are all 0. + self.assertEqual(phone_home_stats["r30v2_users_all"], 0) + self.assertEqual(phone_home_stats["r30v2_users_android"], 0) + self.assertEqual(phone_home_stats["r30v2_users_ios"], 0) + self.assertEqual(phone_home_stats["r30v2_users_electron"], 0) + self.assertEqual(phone_home_stats["r30v2_users_web"], 0) + self.assertEqual( + phone_home_stats["cache_factor"], self.hs.config.caches.global_factor + ) + self.assertEqual( + phone_home_stats["event_cache_size"], + self.hs.config.caches.event_cache_size, + ) + self.assertEqual( + phone_home_stats["database_engine"], + self.hs.config.database.databases[0].config["name"], + ) + self.assertEqual( + phone_home_stats["database_server_version"], + self.hs.get_datastores().main.database_engine.server_version, + ) + + synapse_logger = logging.getLogger("synapse") + log_level = synapse_logger.getEffectiveLevel() + self.assertEqual(phone_home_stats["log_level"], logging.getLevelName(log_level)) From 553e124f766584456fbdb6d1aa37fdd12ad54dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Tue, 20 May 2025 17:53:30 +0100 Subject: [PATCH 332/691] Include room ID in room deletion status response (#18318) When querying by `delete_id` it's handy to see which room the delete pertains to. --- changelog.d/18318.feature | 1 + docs/admin_api/rooms.md | 7 ++++++- synapse/rest/admin/rooms.py | 1 + tests/rest/admin/test_room.py | 7 +++++++ 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18318.feature diff --git a/changelog.d/18318.feature b/changelog.d/18318.feature new file mode 100644 index 0000000000..fba0e83577 --- /dev/null +++ b/changelog.d/18318.feature @@ -0,0 +1 @@ +Include room ID in room deletion status response. diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index bfc2cd4376..bdda9b47ad 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -794,6 +794,7 @@ A response body like the following is returned: "results": [ { "delete_id": "delete_id1", + "room_id": "!roomid:example.com", "status": "failed", "error": "error message", "shutdown_room": { @@ -804,6 +805,7 @@ A response body like the following is returned: } }, { "delete_id": "delete_id2", + "room_id": "!roomid:example.com", "status": "purging", "shutdown_room": { "kicked_users": [ @@ -842,6 +844,8 @@ A response body like the following is returned: ```json { "status": "purging", + "delete_id": "bHkCNQpHqOaFhPtK", + "room_id": "!roomid:example.com", "shutdown_room": { "kicked_users": [ "@foobar:example.com" @@ -869,7 +873,8 @@ The following fields are returned in the JSON response body: - `results` - An array of objects, each containing information about one task. This field is omitted from the result when you query by `delete_id`. Task objects contain the following fields: - - `delete_id` - The ID for this purge if you query by `room_id`. + - `delete_id` - The ID for this purge + - `room_id` - The ID of the room being deleted - `status` - The status will be one of: - `shutting_down` - The process is removing users from the room. - `purging` - The process is purging the room and event data from database. diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 3097cb1a9d..f8c5bf18d4 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -150,6 +150,7 @@ class RoomRestV2Servlet(RestServlet): def _convert_delete_task_to_response(task: ScheduledTask) -> JsonDict: return { "delete_id": task.id, + "room_id": task.resource_id, "status": task.status, "shutdown_room": task.result, } diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 165d175ab2..8d806082aa 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -758,6 +758,8 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(2, len(channel.json_body["results"])) self.assertEqual("complete", channel.json_body["results"][0]["status"]) self.assertEqual("complete", channel.json_body["results"][1]["status"]) + self.assertEqual(self.room_id, channel.json_body["results"][0]["room_id"]) + self.assertEqual(self.room_id, channel.json_body["results"][1]["room_id"]) delete_ids = {delete_id1, delete_id2} self.assertTrue(channel.json_body["results"][0]["delete_id"] in delete_ids) delete_ids.remove(channel.json_body["results"][0]["delete_id"]) @@ -777,6 +779,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(1, len(channel.json_body["results"])) self.assertEqual("complete", channel.json_body["results"][0]["status"]) self.assertEqual(delete_id2, channel.json_body["results"][0]["delete_id"]) + self.assertEqual(self.room_id, channel.json_body["results"][0]["room_id"]) # get status after more than clearing time for all tasks self.reactor.advance(TaskScheduler.KEEP_TASKS_FOR_MS / 1000 / 2) @@ -1237,6 +1240,9 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual( delete_id, channel_room_id.json_body["results"][0]["delete_id"] ) + self.assertEqual( + self.room_id, channel_room_id.json_body["results"][0]["room_id"] + ) # get information by delete_id channel_delete_id = self.make_request( @@ -1249,6 +1255,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): channel_delete_id.code, msg=channel_delete_id.json_body, ) + self.assertEqual(self.room_id, channel_delete_id.json_body["room_id"]) # test values that are the same in both responses for content in [ From b7d48419476f70e54dc24ecd986562ba22be52ec Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 21 May 2025 16:09:09 -0600 Subject: [PATCH 333/691] Policy server part 1: Actually call the policy server (#18387) Roughly reviewable commit-by-commit. This is the first part of adding policy server support to Synapse. Other parts (unordered), which may or may not be bundled into fewer PRs, include: * Implementation of a bulk API * Supporting a moderation server config (the `fallback_*` options of https://github.com/element-hq/policyserv_spam_checker ) * Adding an "early event hook" for appservices to receive federation transactions *before* events are processed formally * Performance and stability improvements ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: turt2live <1190097+turt2live@users.noreply.github.com> Co-authored-by: Devon Hudson --- changelog.d/18387.feature | 1 + synapse/federation/federation_base.py | 34 ++++ synapse/federation/federation_client.py | 57 ++++++ synapse/federation/transport/client.py | 27 +++ synapse/handlers/message.py | 15 +- synapse/handlers/room_policy.py | 89 ++++++++++ synapse/server.py | 5 + synapse/types/handlers/policy_server.py | 16 ++ tests/handlers/test_room_policy.py | 226 ++++++++++++++++++++++++ 9 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18387.feature create mode 100644 synapse/handlers/room_policy.py create mode 100644 synapse/types/handlers/policy_server.py create mode 100644 tests/handlers/test_room_policy.py diff --git a/changelog.d/18387.feature b/changelog.d/18387.feature new file mode 100644 index 0000000000..2d9ff2cea2 --- /dev/null +++ b/changelog.d/18387.feature @@ -0,0 +1 @@ +Add support for calling Policy Servers ([MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284)) to mark events as spam. \ No newline at end of file diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 3796bff5e7..45593430e8 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -30,6 +30,7 @@ from synapse.crypto.keyring import Keyring from synapse.events import EventBase, make_event_from_dict from synapse.events.utils import prune_event, validate_canonicaljson from synapse.federation.units import filter_pdus_for_valid_depth +from synapse.handlers.room_policy import RoomPolicyHandler from synapse.http.servlet import assert_params_in_dict from synapse.logging.opentracing import log_kv, trace from synapse.types import JsonDict, get_domain_from_id @@ -64,6 +65,24 @@ class FederationBase: self._clock = hs.get_clock() self._storage_controllers = hs.get_storage_controllers() + # We need to define this lazily otherwise we get a cyclic dependency. + # self._policy_handler = hs.get_room_policy_handler() + self._policy_handler: Optional[RoomPolicyHandler] = None + + def _lazily_get_policy_handler(self) -> RoomPolicyHandler: + """Lazily get the room policy handler. + + This is required to avoid an import cycle: RoomPolicyHandler requires a + FederationClient, which requires a FederationBase, which requires a + RoomPolicyHandler. + + Returns: + RoomPolicyHandler: The room policy handler. + """ + if self._policy_handler is None: + self._policy_handler = self.hs.get_room_policy_handler() + return self._policy_handler + @trace async def _check_sigs_and_hash( self, @@ -80,6 +99,10 @@ class FederationBase: Also runs the event through the spam checker; if it fails, redacts the event and flags it as soft-failed. + Also checks that the event is allowed by the policy server, if the room uses + a policy server. If the event is not allowed, the event is flagged as + soft-failed but not redacted. + Args: room_version: The room version of the PDU pdu: the event to be checked @@ -145,6 +168,17 @@ class FederationBase: ) return redacted_event + policy_allowed = await self._lazily_get_policy_handler().is_event_allowed(pdu) + if not policy_allowed: + logger.warning( + "Event not allowed by policy server, soft-failing %s", pdu.event_id + ) + pdu.internal_metadata.soft_failed = True + # Note: we don't redact the event so admins can inspect the event after the + # fact. Other processes may redact the event, but that won't be applied to + # the database copy of the event until the server's config requires it. + return pdu + spam_check = await self._spam_checker_module_callbacks.check_event_for_spam(pdu) if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 9fc5b70e9a..7c485aa7e0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -75,6 +75,7 @@ from synapse.http.client import is_unknown_endpoint from synapse.http.types import QueryParams from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id +from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.retryutils import NotRetryingDestination @@ -421,6 +422,62 @@ class FederationClient(FederationBase): return None + @trace + @tag_args + async def get_pdu_policy_recommendation( + self, destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> str: + """Requests that the destination server (typically a policy server) + check the event and return its recommendation on how to handle the + event. + + If the policy server could not be contacted or the policy server + returned an unknown recommendation, this returns an OK recommendation. + This type fixing behaviour is done because the typical caller will be + in a critical call path and would generally interpret a `None` or similar + response as "weird value; don't care; move on without taking action". We + just frontload that logic here. + + + Args: + destination: The remote homeserver to ask (a policy server) + pdu: The event to check + timeout: How long to try (in ms) the destination for before + giving up. None indicates no timeout. + + Returns: + The policy recommendation, or RECOMMENDATION_OK if the policy server was + uncontactable or returned an unknown recommendation. + """ + + logger.debug( + "get_pdu_policy_recommendation for event_id=%s from %s", + pdu.event_id, + destination, + ) + + try: + res = await self.transport_layer.get_policy_recommendation_for_pdu( + destination, pdu, timeout=timeout + ) + recommendation = res.get("recommendation") + if not isinstance(recommendation, str): + raise InvalidResponseError("recommendation is not a string") + if recommendation not in (RECOMMENDATION_OK, RECOMMENDATION_SPAM): + logger.warning( + "get_pdu_policy_recommendation: unknown recommendation: %s", + recommendation, + ) + return RECOMMENDATION_OK + return recommendation + except Exception as e: + logger.warning( + "get_pdu_policy_recommendation: server %s responded with error, assuming OK recommendation: %s", + destination, + e, + ) + return RECOMMENDATION_OK + @trace @tag_args async def get_pdu( diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 206e91ed14..62bf96ce91 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -143,6 +143,33 @@ class TransportLayerClient: destination, path=path, timeout=timeout, try_trailing_slash_on_400=True ) + async def get_policy_recommendation_for_pdu( + self, destination: str, event: EventBase, timeout: Optional[int] = None + ) -> JsonDict: + """Requests the policy recommendation for the given pdu from the given policy server. + + Args: + destination: The host name of the remote homeserver checking the event. + event: The event to check. + timeout: How long to try (in ms) the destination for before giving up. + None indicates no timeout. + + Returns: + The full recommendation object from the remote server. + """ + logger.debug( + "get_policy_recommendation_for_pdu dest=%s, event_id=%s", + destination, + event.event_id, + ) + return await self.client.post_json( + destination=destination, + path=f"/_matrix/policy/unstable/org.matrix.msc4284/event/{event.event_id}/check", + data=event.get_pdu_json(), + ignore_backoff=True, + timeout=timeout, + ) + async def backfill( self, destination: str, room_id: str, event_tuples: Collection[str], limit: int ) -> Optional[Union[JsonDict, list]]: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index ff6eb5a514..cb6de02309 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -495,6 +495,7 @@ class EventCreationHandler: self._instance_name = hs.get_instance_name() self._notifier = hs.get_notifier() self._worker_lock_handler = hs.get_worker_locks_handler() + self._policy_handler = hs.get_room_policy_handler() self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state @@ -1108,6 +1109,18 @@ class EventCreationHandler: event.sender, ) + policy_allowed = await self._policy_handler.is_event_allowed(event) + if not policy_allowed: + logger.warning( + "Event not allowed by policy server, rejecting %s", + event.event_id, + ) + raise SynapseError( + 403, + "This message has been rejected as probable spam", + Codes.FORBIDDEN, + ) + spam_check_result = ( await self._spam_checker_module_callbacks.check_event_for_spam( event @@ -1119,7 +1132,7 @@ class EventCreationHandler: [code, dict] = spam_check_result raise SynapseError( 403, - "This message had been rejected as probable spam", + "This message has been rejected as probable spam", code, dict, ) diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py new file mode 100644 index 0000000000..dcfebb128c --- /dev/null +++ b/synapse/handlers/room_policy.py @@ -0,0 +1,89 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2016-2021 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from typing import TYPE_CHECKING + +from synapse.events import EventBase +from synapse.types.handlers.policy_server import RECOMMENDATION_OK +from synapse.util.stringutils import parse_and_validate_server_name + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class RoomPolicyHandler: + def __init__(self, hs: "HomeServer"): + self._hs = hs + self._store = hs.get_datastores().main + self._storage_controllers = hs.get_storage_controllers() + self._event_auth_handler = hs.get_event_auth_handler() + self._federation_client = hs.get_federation_client() + + async def is_event_allowed(self, event: EventBase) -> bool: + """Check if the given event is allowed in the room by the policy server. + + Note: This will *always* return True if the room's policy server is Synapse + itself. This is because Synapse can't be a policy server (currently). + + If no policy server is configured in the room, this returns True. Similarly, if + the policy server is invalid in any way (not joined, not a server, etc), this + returns True. + + If a valid and contactable policy server is configured in the room, this returns + True if that server suggests the event is not spammy, and False otherwise. + + Args: + event: The event to check. This should be a fully-formed PDU. + + Returns: + bool: True if the event is allowed in the room, False otherwise. + """ + policy_event = await self._storage_controllers.state.get_current_state_event( + event.room_id, "org.matrix.msc4284.policy", "" + ) + if not policy_event: + return True # no policy server == default allow + + policy_server = policy_event.content.get("via", "") + if policy_server is None or not isinstance(policy_server, str): + return True # no policy server == default allow + + if policy_server == self._hs.hostname: + return True # Synapse itself can't be a policy server (currently) + + try: + parse_and_validate_server_name(policy_server) + except ValueError: + return True # invalid policy server == default allow + + is_in_room = await self._event_auth_handler.is_host_in_room( + event.room_id, policy_server + ) + if not is_in_room: + return True # policy server not in room == default allow + + # At this point, the server appears valid and is in the room, so ask it to check + # the event. + recommendation = await self._federation_client.get_pdu_policy_recommendation( + policy_server, event + ) + if recommendation != RECOMMENDATION_OK: + return False + + return True # default allow diff --git a/synapse/server.py b/synapse/server.py index bd2faa61b9..2add4d4e6e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -107,6 +107,7 @@ from synapse.handlers.room_member import ( RoomMemberMasterHandler, ) from synapse.handlers.room_member_worker import RoomMemberWorkerHandler +from synapse.handlers.room_policy import RoomPolicyHandler from synapse.handlers.room_summary import RoomSummaryHandler from synapse.handlers.search import SearchHandler from synapse.handlers.send_email import SendEmailHandler @@ -807,6 +808,10 @@ class HomeServer(metaclass=abc.ABCMeta): return OidcHandler(self) + @cache_in_self + def get_room_policy_handler(self) -> RoomPolicyHandler: + return RoomPolicyHandler(self) + @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: return EventClientSerializer(self) diff --git a/synapse/types/handlers/policy_server.py b/synapse/types/handlers/policy_server.py new file mode 100644 index 0000000000..bfc09dabf4 --- /dev/null +++ b/synapse/types/handlers/policy_server.py @@ -0,0 +1,16 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +RECOMMENDATION_OK = "ok" +RECOMMENDATION_SPAM = "spam" diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py new file mode 100644 index 0000000000..26642c18ea --- /dev/null +++ b/tests/handlers/test_room_policy.py @@ -0,0 +1,226 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from typing import Optional +from unittest import mock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.events import EventBase, make_event_from_dict +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID +from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM +from synapse.util import Clock + +from tests import unittest +from tests.test_utils import event_injection + + +class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): + """Tests room policy handler.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # mock out the federation transport client + self.mock_federation_transport_client = mock.Mock( + spec=["get_policy_recommendation_for_pdu"] + ) + self.mock_federation_transport_client.get_policy_recommendation_for_pdu = ( + mock.AsyncMock() + ) + return super().setup_test_homeserver( + federation_transport_client=self.mock_federation_transport_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.handler = hs.get_room_policy_handler() + main_store = self.hs.get_datastores().main + + # Create a room + self.creator = self.register_user("creator", "test1234") + self.creator_token = self.login("creator", "test1234") + self.room_id = self.helper.create_room_as( + room_creator=self.creator, tok=self.creator_token + ) + room_version = self.get_success(main_store.get_room_version(self.room_id)) + + # Create some sample events + self.spammy_event = make_event_from_dict( + room_version=room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is a spammy event.", + }, + }, + ) + self.not_spammy_event = make_event_from_dict( + room_version=room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@not_spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is a NOT spammy event.", + }, + }, + ) + + # Prepare the policy server mock to decide spam vs not spam on those events + self.call_count = 0 + + async def get_policy_recommendation_for_pdu( + destination: str, + pdu: EventBase, + timeout: Optional[int] = None, + ) -> JsonDict: + self.call_count += 1 + self.assertEqual(destination, self.OTHER_SERVER_NAME) + if pdu.event_id == self.spammy_event.event_id: + return {"recommendation": RECOMMENDATION_SPAM} + elif pdu.event_id == self.not_spammy_event.event_id: + return {"recommendation": RECOMMENDATION_OK} + else: + self.fail("Unexpected event ID") + + self.mock_federation_transport_client.get_policy_recommendation_for_pdu.side_effect = get_policy_recommendation_for_pdu + + def _add_policy_server_to_room(self) -> None: + # Inject a member event into the room + policy_user_id = f"@policy:{self.OTHER_SERVER_NAME}" + self.get_success( + event_injection.inject_member_event( + self.hs, self.room_id, policy_user_id, "join" + ) + ) + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": self.OTHER_SERVER_NAME, + }, + tok=self.creator_token, + state_key="", + ) + + def test_no_policy_event_set(self) -> None: + # We don't need to modify the room state at all - we're testing the default + # case where a room doesn't use a policy server. + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_empty_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + # empty content (no `via`) + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_nonstring_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": 42, # should be a server name + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_self_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + # We ignore events when the policy server is ourselves (for now?) + "via": (UserID.from_string(self.creator)).domain, + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_invalid_server_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": "|this| is *not* a (valid) server name.com", + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_not_in_room_policy_event_set(self) -> None: + self.helper.send_state( + self.room_id, + "org.matrix.msc4284.policy", + { + "via": f"x.{self.OTHER_SERVER_NAME}", + }, + tok=self.creator_token, + state_key="", + ) + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 0) + + def test_spammy_event_is_spam(self) -> None: + self._add_policy_server_to_room() + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, False) + self.assertEqual(self.call_count, 1) + + def test_not_spammy_event_is_not_spam(self) -> None: + self._add_policy_server_to_room() + + ok = self.get_success(self.handler.is_event_allowed(self.not_spammy_event)) + self.assertEqual(ok, True) + self.assertEqual(self.call_count, 1) From ed6b7ba9c3da3add7a1551069411fa3697b0efc4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 21 May 2025 23:12:01 +0100 Subject: [PATCH 334/691] Bump pyo3 from 0.23.5 to 0.24.2 (#18460) Also bump pythonize from 0.23.0 to 0.24.0, otherwise we couldn't compile as pythonize 0.23.0 required pyo3 "^0.23.0". Addresses [RUSTSEC-2025-0020](https://rustsec.org/advisories/RUSTSEC-2025-0020), although Synapse is not affected as we don't make use of `PyString::from_object`. [pyo3 0.24.x](https://github.com/PyO3/pyo3/releases/tag/v0.24.0) include some performance optimisations apparently, and no breaking changes. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- Cargo.lock | 28 ++++++++++++++-------------- changelog.d/18460.misc | 1 + rust/Cargo.toml | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 changelog.d/18460.misc diff --git a/Cargo.lock b/Cargo.lock index 13156e67b5..980dff6987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" dependencies = [ "anyhow", "cfg-if", @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" dependencies = [ "once_cell", "target-lexicon", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" dependencies = [ "libc", "pyo3-build-config", @@ -327,9 +327,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" dependencies = [ "heck", "proc-macro2", @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91a6ee7a084f913f98d70cdc3ebec07e852b735ae3059a1500db2661265da9ff" +checksum = "d5bcac0d0b71821f0d69e42654f1e15e5c94b85196446c4de9588951a2117e7b" dependencies = [ "pyo3", "serde", @@ -532,9 +532,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "typenum" diff --git a/changelog.d/18460.misc b/changelog.d/18460.misc new file mode 100644 index 0000000000..5aa19683eb --- /dev/null +++ b/changelog.d/18460.misc @@ -0,0 +1 @@ +Bump pyo3 from 0.23.5 to 0.24.2. \ No newline at end of file diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 651b268f86..840988e74e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,14 +30,14 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.23.5", features = [ +pyo3 = { version = "0.24.2", features = [ "macros", "anyhow", "abi3", "abi3-py39", ] } pyo3-log = "0.12.0" -pythonize = "0.23.0" +pythonize = "0.24.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } From 162407319103a9f553225a925017cf3f951a4644 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 21 May 2025 23:24:12 +0100 Subject: [PATCH 335/691] Bump Tornado from 6.4.2 to 6.5.0 (#18459) Bumps tornado 6.5.0 to mitigate [CVE-2025-47287](https://nvd.nist.gov/vuln/detail/CVE-2025-47287). This dependency is only used indirectly through our sentry dependency. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18459.misc | 1 + poetry.lock | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 changelog.d/18459.misc diff --git a/changelog.d/18459.misc b/changelog.d/18459.misc new file mode 100644 index 0000000000..e148825696 --- /dev/null +++ b/changelog.d/18459.misc @@ -0,0 +1 @@ +Bump tornado from 6.4.2 to 6.5.0. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index ada0646215..9938e46780 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2767,24 +2767,25 @@ files = [ [[package]] name = "tornado" -version = "6.4.2" +version = "6.5" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"opentracing\"" files = [ - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, - {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, - {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, - {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, + {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c625b9d03f1fb4d64149c47d0135227f0434ebb803e2008040eb92906b0105a"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a0d8d2309faf015903080fb5bdd969ecf9aa5ff893290845cf3fd5b2dd101bc"}, + {file = "tornado-6.5-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03576ab51e9b1677e4cdaae620d6700d9823568b7939277e4690fe4085886c55"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab75fe43d0e1b3a5e3ceddb2a611cb40090dd116a84fc216a07a298d9e000471"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:119c03f440a832128820e87add8a175d211b7f36e7ee161c631780877c28f4fb"}, + {file = "tornado-6.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:231f2193bb4c28db2bdee9e57bc6ca0cd491f345cd307c57d79613b058e807e0"}, + {file = "tornado-6.5-cp39-abi3-win32.whl", hash = "sha256:fd20c816e31be1bbff1f7681f970bbbd0bb241c364220140228ba24242bcdc59"}, + {file = "tornado-6.5-cp39-abi3-win_amd64.whl", hash = "sha256:007f036f7b661e899bd9ef3fa5f87eb2cb4d1b2e7d67368e778e140a2f101a7a"}, + {file = "tornado-6.5-cp39-abi3-win_arm64.whl", hash = "sha256:542e380658dcec911215c4820654662810c06ad872eefe10def6a5e9b20e9633"}, + {file = "tornado-6.5.tar.gz", hash = "sha256:c70c0a26d5b2d85440e4debd14a8d0b463a0cf35d92d3af05f5f1ffa8675c826"}, ] [[package]] From 24e849e483820762fa2d231ad702e6aeaa23829c Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 23 May 2025 01:59:40 -0700 Subject: [PATCH 336/691] Don't move invited users to new room when shutting down room (#18471) This is confusing to users who received unwanted invites. --- changelog.d/18471.misc | 1 + synapse/handlers/room.py | 23 ++++++++++---------- tests/rest/admin/test_room.py | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 changelog.d/18471.misc diff --git a/changelog.d/18471.misc b/changelog.d/18471.misc new file mode 100644 index 0000000000..b36712bea3 --- /dev/null +++ b/changelog.d/18471.misc @@ -0,0 +1 @@ +Don't move invited users to new room when shutting down room. \ No newline at end of file diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 386375d64b..763f99e028 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1806,7 +1806,7 @@ class RoomShutdownHandler: ] = None, ) -> Optional[ShutdownRoomResponse]: """ - Shuts down a room. Moves all local users and room aliases automatically + Shuts down a room. Moves all joined local users and room aliases automatically to a new room if `new_room_user_id` is set. Otherwise local users only leave the room without any information. @@ -1949,16 +1949,17 @@ class RoomShutdownHandler: # Join users to new room if new_room_user_id: - assert new_room_id is not None - await self.room_member_handler.update_membership( - requester=target_requester, - target=target_requester.user, - room_id=new_room_id, - action=Membership.JOIN, - content={}, - ratelimit=False, - require_consent=False, - ) + if membership == Membership.JOIN: + assert new_room_id is not None + await self.room_member_handler.update_membership( + requester=target_requester, + target=target_requester.user, + room_id=new_room_id, + action=Membership.JOIN, + content={}, + ratelimit=False, + require_consent=False, + ) result["kicked_users"].append(user_id) if update_result_fct: diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 8d806082aa..e22dfcba1b 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -369,6 +369,47 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(200, channel.code, msg=channel.json_body) self._is_blocked(room_id) + def test_invited_users_not_joined_to_new_room(self) -> None: + """ + Test that when a new room id is provided, users who are only invited + but have not joined original room are not moved to new room. + """ + invitee = self.register_user("invitee", "pass") + + self.helper.invite( + self.room_id, self.other_user, invitee, tok=self.other_user_tok + ) + + # verify that user is invited + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{self.room_id}/members?membership=invite", + access_token=self.other_user_tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(len(channel.json_body["chunk"]), 1) + invite = channel.json_body["chunk"][0] + self.assertEqual(invite["state_key"], invitee) + + # shutdown room + channel = self.make_request( + "DELETE", + self.url, + {"new_room_user_id": self.admin_user}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(len(channel.json_body["kicked_users"]), 2) + + # joined member is moved to new room but invited user is not + users_in_room = self.get_success( + self.store.get_users_in_room(channel.json_body["new_room_id"]) + ) + self.assertNotIn(invitee, users_in_room) + self.assertIn(self.other_user, users_in_room) + self._is_purged(self.room_id) + self._has_no_members(self.room_id) + def test_shutdown_room_consent(self) -> None: """Test that we can shutdown rooms with local users who have not yet accepted the privacy policy. This used to fail when we tried to From 33ba8860c43d4770ea119a09a4fcbbf366f3b32e Mon Sep 17 00:00:00 2001 From: 3nprob <74199244+3nprob@users.noreply.github.com> Date: Mon, 26 May 2025 14:21:43 +0000 Subject: [PATCH 337/691] fix(device-handler): make _maybe_retry_device_resync thread-safe (#18391) A race-condition may render concurrent retry loops. Use an actual `Lock` for guarding single access of device resyncing retrying. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18391.bugfix | 1 + synapse/handlers/device.py | 13 +++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18391.bugfix diff --git a/changelog.d/18391.bugfix b/changelog.d/18391.bugfix new file mode 100644 index 0000000000..bbcb7b7a28 --- /dev/null +++ b/changelog.d/18391.bugfix @@ -0,0 +1 @@ +Prevent race-condition in `_maybe_retry_device_resync` entrance. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 1efd039f22..f8b547bbed 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -20,6 +20,7 @@ # # import logging +from threading import Lock from typing import ( TYPE_CHECKING, AbstractSet, @@ -1237,7 +1238,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): ) # Attempt to resync out of sync device lists every 30s. - self._resync_retry_in_progress = False + self._resync_retry_lock = Lock() self.clock.looping_call( run_as_background_process, 30 * 1000, @@ -1419,13 +1420,10 @@ class DeviceListUpdater(DeviceListWorkerUpdater): """Retry to resync device lists that are out of sync, except if another retry is in progress. """ - if self._resync_retry_in_progress: + # If the lock can not be acquired we want to always return immediately instead of blocking here + if not self._resync_retry_lock.acquire(blocking=False): return - try: - # Prevent another call of this function to retry resyncing device lists so - # we don't send too many requests. - self._resync_retry_in_progress = True # Get all of the users that need resyncing. need_resync = await self.store.get_user_ids_requiring_device_list_resync() @@ -1466,8 +1464,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): e, ) finally: - # Allow future calls to retry resyncinc out of sync device lists. - self._resync_retry_in_progress = False + self._resync_retry_lock.release() async def multi_user_device_resync( self, user_ids: List[str], mark_failed_as_stale: bool = True From 07468a0f1c6ef07fe056e6a77347d045d34c69ff Mon Sep 17 00:00:00 2001 From: gui-yue Date: Wed, 28 May 2025 01:17:04 +0800 Subject: [PATCH 338/691] Increase timeout for test_lock_contention on RISC-V (#18430) This PR addresses a test failure for `tests.handlers.test_worker_lock.WorkerLockTestCase.test_lock_contention` which consistently times out on the RISC-V (specifically `riscv64`) architecture. The test simulates high lock contention and has a default timeout of 5 seconds, which seems sufficient for architectures like x86_64 but proves too short for current RISC-V hardware/environment performance characteristics, leading to spurious `tests.utils.TestTimeout` failures. This fix introduces architecture detection using `platform.machine()`. If a RISC-V architecture is detected: * The timeout for this specific test is increased (e.g., to 15 seconds ). The original, stricter timeout (5 seconds) and lock count (500) are maintained for all other architectures to avoid masking potential performance regressions elsewhere. This change has been tested locally on RISC-V, where the test now passes reliably, and on x86_64, where it continues to pass with the original constraints. --- ### Pull Request Checklist * [X] Pull request is based on the develop branch *(Assuming you based it correctly)* * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). *(See below)* * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) *(Please run linters locally)* --- changelog.d/18430.bugfix | 1 + tests/handlers/test_worker_lock.py | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18430.bugfix diff --git a/changelog.d/18430.bugfix b/changelog.d/18430.bugfix new file mode 100644 index 0000000000..d61fd56c33 --- /dev/null +++ b/changelog.d/18430.bugfix @@ -0,0 +1 @@ +Fix the `tests.handlers.test_worker_lock.WorkerLockTestCase.test_lock_contention` test which could spuriously time out on RISC-V architectures due to performance differences. \ No newline at end of file diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 6e9a15c8ee..0691d3f99c 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -19,6 +19,9 @@ # # +import logging +import platform + from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor @@ -29,6 +32,8 @@ from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.utils import test_timeout +logger = logging.getLogger(__name__) + class WorkerLockTestCase(unittest.HomeserverTestCase): def prepare( @@ -53,12 +58,27 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" - + nb_locks_to_test = 500 + current_machine = platform.machine().lower() + if current_machine.startswith("riscv"): + # RISC-V specific settings + timeout_seconds = 15 # Increased timeout for RISC-V + # add a print or log statement here for visibility in CI logs + logger.info( # use logger.info + f"Detected RISC-V architecture ({current_machine}). " + f"Adjusting test_lock_contention: timeout={timeout_seconds}s" + ) + else: + # Settings for other architectures + timeout_seconds = 5 # It takes around 0.5s on a 5+ years old laptop - with test_timeout(5): - nb_locks = 500 - d = self._take_locks(nb_locks) - self.assertEqual(self.get_success(d), nb_locks) + with test_timeout(timeout_seconds): # Use the dynamically set timeout + d = self._take_locks( + nb_locks_to_test + ) # Use the (potentially adjusted) number of locks + self.assertEqual( + self.get_success(d), nb_locks_to_test + ) # Assert against the used number of locks async def _take_locks(self, nb_locks: int) -> int: locks = [ From c8733be8aa0b5853d9a508e24d20c454ad149ec6 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Wed, 28 May 2025 11:58:08 +0200 Subject: [PATCH 339/691] Add option to limit key queries to users sharing rooms as per MSC4263 (#18180) This implements https://github.com/matrix-org/matrix-spec-proposals/pull/4263. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Signed-off-by: Johannes Marbach --- changelog.d/18180.feature | 1 + synapse/config/experimental.py | 6 + synapse/handlers/e2e_keys.py | 37 ++++- synapse/storage/databases/main/roommember.py | 67 +++++++++ tests/handlers/test_e2e_keys.py | 150 +++++++++++++++++++ 5 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18180.feature diff --git a/changelog.d/18180.feature b/changelog.d/18180.feature new file mode 100644 index 0000000000..fbf226e51c --- /dev/null +++ b/changelog.d/18180.feature @@ -0,0 +1 @@ +Add `msc4263_limit_key_queries_to_users_who_share_rooms` config option as per [MSC4263](https://github.com/matrix-org/matrix-spec-proposals/pull/4263). diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 1226eaa58a..2dc75a778e 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -560,3 +560,9 @@ class ExperimentalConfig(Config): # MSC4076: Add `disable_badge_count`` to pusher configuration self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) + + # MSC4263: Preventing MXID enumeration via key queries + self.msc4263_limit_key_queries_to_users_who_share_rooms = experimental.get( + "msc4263_limit_key_queries_to_users_who_share_rooms", + False, + ) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index f2b2e30bf4..6171aaf29f 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -158,7 +158,37 @@ class E2eKeysHandler: the number of in-flight queries at a time. """ async with self._query_devices_linearizer.queue((from_user_id, from_device_id)): - device_keys_query: Dict[str, List[str]] = query_body.get("device_keys", {}) + + async def filter_device_key_query( + query: Dict[str, List[str]], + ) -> Dict[str, List[str]]: + if not self.config.experimental.msc4263_limit_key_queries_to_users_who_share_rooms: + # Only ignore invalid user IDs, which is the same behaviour as if + # the user existed but had no keys. + return { + user_id: v + for user_id, v in query.items() + if UserID.is_valid(user_id) + } + + # Strip invalid user IDs and user IDs the requesting user does not share rooms with. + valid_user_ids = [ + user_id for user_id in query.keys() if UserID.is_valid(user_id) + ] + allowed_user_ids = set( + await self.store.do_users_share_a_room_joined_or_invited( + from_user_id, valid_user_ids + ) + ) + return { + user_id: v + for user_id, v in query.items() + if user_id in allowed_user_ids + } + + device_keys_query: Dict[str, List[str]] = await filter_device_key_query( + query_body.get("device_keys", {}) + ) # separate users by domain. # make a map from domain to user_id to device_ids @@ -166,11 +196,6 @@ class E2eKeysHandler: remote_queries = {} for user_id, device_ids in device_keys_query.items(): - if not UserID.is_valid(user_id): - # Ignore invalid user IDs, which is the same behaviour as if - # the user existed but had no keys. - continue - # we use UserID.from_string to catch invalid user ids if self.is_mine(UserID.from_string(user_id)): local_query[user_id] = device_ids diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 2084776543..7ca73abb83 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -871,6 +871,73 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return {u for u, share_room in user_dict.items() if share_room} + @cached(max_entries=10000) + async def does_pair_of_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_id: str + ) -> bool: + raise NotImplementedError() + + @cachedList( + cached_method_name="does_pair_of_users_share_a_room_joined_or_invited", + list_name="other_user_ids", + ) + async def _do_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_ids: Collection[str] + ) -> Mapping[str, Optional[bool]]: + """Return mapping from user ID to whether they share a room with the + given user via being either joined or invited. + + Note: `None` and `False` are equivalent and mean they don't share a + room. + """ + + def do_users_share_a_room_joined_or_invited_txn( + txn: LoggingTransaction, user_ids: Collection[str] + ) -> Dict[str, bool]: + clause, args = make_in_list_sql_clause( + self.database_engine, "state_key", user_ids + ) + + # This query works by fetching both the list of rooms for the target + # user and the set of other users, and then checking if there is any + # overlap. + sql = f""" + SELECT DISTINCT b.state_key + FROM ( + SELECT room_id FROM current_state_events + WHERE type = 'm.room.member' AND (membership = 'join' OR membership = 'invite') AND state_key = ? + ) AS a + INNER JOIN ( + SELECT room_id, state_key FROM current_state_events + WHERE type = 'm.room.member' AND (membership = 'join' OR membership = 'invite') AND {clause} + ) AS b using (room_id) + """ + + txn.execute(sql, (user_id, *args)) + return {u: True for (u,) in txn} + + to_return = {} + for batch_user_ids in batch_iter(other_user_ids, 1000): + res = await self.db_pool.runInteraction( + "do_users_share_a_room_joined_or_invited", + do_users_share_a_room_joined_or_invited_txn, + batch_user_ids, + ) + to_return.update(res) + + return to_return + + async def do_users_share_a_room_joined_or_invited( + self, user_id: str, other_user_ids: Collection[str] + ) -> Set[str]: + """Return the set of users who share a room with the first users via being either joined or invited""" + + user_dict = await self._do_users_share_a_room_joined_or_invited( + user_id, other_user_ids + ) + + return {u for u, share_room in user_dict.items() if share_room} + async def get_users_who_share_room_with_user(self, user_id: str) -> Set[str]: """Returns the set of users who share a room with `user_id`""" room_ids = await self.get_rooms_for_user(user_id) diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index e67efcc17f..70fc4263e7 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -1896,3 +1896,153 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.assertEqual( remaining_key_ids, {"AAAAAAAAAA", "BAAAAA", "BAAAAB", "BAAAAAAAAA"} ) + + @override_config( + { + "experimental_features": { + "msc4263_limit_key_queries_to_users_who_share_rooms": True + } + } + ) + def test_query_devices_remote_restricted_not_in_shared_room(self) -> None: + """Tests that querying keys for a remote user that we don't share a room + with returns nothing. + """ + + remote_user_id = "@test:other" + local_user_id = "@test:test" + + # Do *not* pretend we're sharing a room with the user we're querying. + + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + self.hs.get_federation_client().query_client_keys = mock.AsyncMock( # type: ignore[method-assign] + return_value={ + "device_keys": {remote_user_id: {}}, + "master_keys": { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + }, + "self_signing_keys": { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + + remote_self_signing_key: remote_self_signing_key + }, + } + }, + } + ) + + e2e_handler = self.hs.get_e2e_keys_handler() + + query_result = self.get_success( + e2e_handler.query_devices( + { + "device_keys": {remote_user_id: []}, + }, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + + self.assertEqual( + query_result, + { + "device_keys": {}, + "failures": {}, + "master_keys": {}, + "self_signing_keys": {}, + "user_signing_keys": {}, + }, + ) + + @override_config( + { + "experimental_features": { + "msc4263_limit_key_queries_to_users_who_share_rooms": True + } + } + ) + def test_query_devices_remote_restricted_in_shared_room(self) -> None: + """Tests that querying keys for a remote user that we share a room + with returns the cross signing keys correctly. + """ + + remote_user_id = "@test:other" + local_user_id = "@test:test" + + # Pretend we're sharing a room with the user we're querying. If not, + # `query_devices` will filter out the user ID and `_query_devices_for_destination` + # will return early. + self.store.do_users_share_a_room_joined_or_invited = mock.AsyncMock( # type: ignore[method-assign] + return_value=[remote_user_id] + ) + self.store.get_rooms_for_user = mock.AsyncMock(return_value={"some_room_id"}) + + remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" + remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" + + self.hs.get_federation_client().query_user_devices = mock.AsyncMock( # type: ignore[method-assign] + return_value={ + "user_id": remote_user_id, + "stream_id": 1, + "devices": [], + "master_key": { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + }, + "self_signing_key": { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + }, + } + ) + + e2e_handler = self.hs.get_e2e_keys_handler() + + query_result = self.get_success( + e2e_handler.query_devices( + { + "device_keys": {remote_user_id: []}, + }, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + + self.assertEqual(query_result["failures"], {}) + self.assertEqual( + query_result["master_keys"], + { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["master"], + "keys": {"ed25519:" + remote_master_key: remote_master_key}, + } + }, + ) + self.assertEqual( + query_result["self_signing_keys"], + { + remote_user_id: { + "user_id": remote_user_id, + "usage": ["self_signing"], + "keys": { + "ed25519:" + remote_self_signing_key: remote_self_signing_key + }, + } + }, + ) From 3680de63a7528e3b4ed77611b5e6af64c156adf8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 11:58:30 +0200 Subject: [PATCH 340/691] Bump types-jsonschema from 4.23.0.20241208 to 4.23.0.20250516 (#18481) Bumps [types-jsonschema](https://github.com/typeshed-internal/stub_uploader) from 4.23.0.20241208 to 4.23.0.20250516.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-jsonschema&package-manager=pip&previous-version=4.23.0.20241208&new-version=4.23.0.20250516)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9938e46780..8b58fdd942 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2973,14 +2973,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.23.0.20241208" +version = "4.23.0.20250516" description = "Typing stubs for jsonschema" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_jsonschema-4.23.0.20241208-py3-none-any.whl", hash = "sha256:87934bd9231c99d8eff94cacfc06ba668f7973577a9bd9e1f9de957c5737313e"}, - {file = "types_jsonschema-4.23.0.20241208.tar.gz", hash = "sha256:e8b15ad01f290ecf6aea53f93fbdf7d4730e4600313e89e8a7f95622f7e87b7c"}, + {file = "types_jsonschema-4.23.0.20250516-py3-none-any.whl", hash = "sha256:e7d0dd7db7e59e63c26e3230e26ffc64c4704cc5170dc21270b366a35ead1618"}, + {file = "types_jsonschema-4.23.0.20250516.tar.gz", hash = "sha256:9ace09d9d35c4390a7251ccd7d833b92ccc189d24d1b347f26212afce361117e"}, ] [package.dependencies] From 58e8521313498ed58012bd6220ac2f082ccf0553 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 11:58:45 +0200 Subject: [PATCH 341/691] Bump ruff from 0.11.10 to 0.11.11 (#18482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.10 to 0.11.11.
Release notes

Sourced from ruff's releases.

0.11.11

Release Notes

Preview features

  • [airflow] Add autofixes for AIR302 and AIR312 (#17942)
  • [airflow] Move rules from AIR312 to AIR302 (#17940)
  • [airflow] Update AIR301 and AIR311 with the latest Airflow implementations (#17985)
  • [flake8-simplify] Enable fix in preview mode (SIM117) (#18208)

Bug fixes

  • Fix inconsistent formatting of match-case on [] and _ (#18147)
  • [pylint] Fix PLW1514 not recognizing the encoding positional argument of codecs.open (#18109)

CLI

  • Add full option name in formatter warning (#18217)

Documentation

  • Fix rendering of admonition in docs (#18163)
  • [flake8-print] Improve print/pprint docs for T201 and T203 (#18130)
  • [flake8-simplify] Add fix safety section (SIM110,SIM210) (#18114,#18100)
  • [pylint] Fix docs example that produced different output (PLW0603) (#18216)

Contributors

... (truncated)

Changelog

Sourced from ruff's changelog.

0.11.11

Preview features

  • [airflow] Add autofixes for AIR302 and AIR312 (#17942)
  • [airflow] Move rules from AIR312 to AIR302 (#17940)
  • [airflow] Update AIR301 and AIR311 with the latest Airflow implementations (#17985)
  • [flake8-simplify] Enable fix in preview mode (SIM117) (#18208)

Bug fixes

  • Fix inconsistent formatting of match-case on [] and _ (#18147)
  • [pylint] Fix PLW1514 not recognizing the encoding positional argument of codecs.open (#18109)

CLI

  • Add full option name in formatter warning (#18217)

Documentation

  • Fix rendering of admonition in docs (#18163)
  • [flake8-print] Improve print/pprint docs for T201 and T203 (#18130)
  • [flake8-simplify] Add fix safety section (SIM110,SIM210) (#18114,#18100)
  • [pylint] Fix docs example that produced different output (PLW0603) (#18216)
Commits
  • 0397682 Bump 0.11.11 (#18259)
  • bcefa45 [ty] Rename call-possibly-unbound-method to `possibly-unbound-implicit-call...
  • 91b7a57 [ty] Implement Python's floor division semantics for Literal ints (#18249)
  • 98da200 [ty] Fix server panic when calling system_mut (#18252)
  • 029085f [ty] Clarify ty check output default in documentation. (#18246)
  • 6df10c6 [pylint] Fix docs example that produced different output (PLW0603) (#18216)
  • bdf4884 Preserve tuple parentheses in case patterns (#18147)
  • 01eeb2f [ty] Support frozen dataclasses (#17974)
  • cb04343 [ty] Split invalid-base error code into two error codes (#18245)
  • 02394b8 [ty] Improve invalid-type-form diagnostic where a module-literal type is us...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ruff&package-manager=pip&previous-version=0.11.10&new-version=0.11.11)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b58fdd942..16d6e43f8a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2440,30 +2440,30 @@ files = [ [[package]] name = "ruff" -version = "0.11.10" +version = "0.11.11" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.11.10-py3-none-linux_armv6l.whl", hash = "sha256:859a7bfa7bc8888abbea31ef8a2b411714e6a80f0d173c2a82f9041ed6b50f58"}, - {file = "ruff-0.11.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:968220a57e09ea5e4fd48ed1c646419961a0570727c7e069842edd018ee8afed"}, - {file = "ruff-0.11.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1067245bad978e7aa7b22f67113ecc6eb241dca0d9b696144256c3a879663bca"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4854fd09c7aed5b1590e996a81aeff0c9ff51378b084eb5a0b9cd9518e6cff2"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b4564e9f99168c0f9195a0fd5fa5928004b33b377137f978055e40008a082c5"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b6a9cc5b62c03cc1fea0044ed8576379dbaf751d5503d718c973d5418483641"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:607ecbb6f03e44c9e0a93aedacb17b4eb4f3563d00e8b474298a201622677947"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b3a522fa389402cd2137df9ddefe848f727250535c70dafa840badffb56b7a4"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f071b0deed7e9245d5820dac235cbdd4ef99d7b12ff04c330a241ad3534319f"}, - {file = "ruff-0.11.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a60e3a0a617eafba1f2e4186d827759d65348fa53708ca547e384db28406a0b"}, - {file = "ruff-0.11.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da8ec977eaa4b7bf75470fb575bea2cb41a0e07c7ea9d5a0a97d13dbca697bf2"}, - {file = "ruff-0.11.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ddf8967e08227d1bd95cc0851ef80d2ad9c7c0c5aab1eba31db49cf0a7b99523"}, - {file = "ruff-0.11.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5a94acf798a82db188f6f36575d80609072b032105d114b0f98661e1679c9125"}, - {file = "ruff-0.11.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3afead355f1d16d95630df28d4ba17fb2cb9c8dfac8d21ced14984121f639bad"}, - {file = "ruff-0.11.10-py3-none-win32.whl", hash = "sha256:dc061a98d32a97211af7e7f3fa1d4ca2fcf919fb96c28f39551f35fc55bdbc19"}, - {file = "ruff-0.11.10-py3-none-win_amd64.whl", hash = "sha256:5cc725fbb4d25b0f185cb42df07ab6b76c4489b4bfb740a175f3a59c70e8a224"}, - {file = "ruff-0.11.10-py3-none-win_arm64.whl", hash = "sha256:ef69637b35fb8b210743926778d0e45e1bffa850a7c61e428c6b971549b5f5d1"}, - {file = "ruff-0.11.10.tar.gz", hash = "sha256:d522fb204b4959909ecac47da02830daec102eeb100fb50ea9554818d47a5fa6"}, + {file = "ruff-0.11.11-py3-none-linux_armv6l.whl", hash = "sha256:9924e5ae54125ed8958a4f7de320dab7380f6e9fa3195e3dc3b137c6842a0092"}, + {file = "ruff-0.11.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c8a93276393d91e952f790148eb226658dd275cddfde96c6ca304873f11d2ae4"}, + {file = "ruff-0.11.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d6e333dbe2e6ae84cdedefa943dfd6434753ad321764fd937eef9d6b62022bcd"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7885d9a5e4c77b24e8c88aba8c80be9255fa22ab326019dac2356cff42089fc6"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b5ab797fcc09121ed82e9b12b6f27e34859e4227080a42d090881be888755d4"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e231ff3132c1119ece836487a02785f099a43992b95c2f62847d29bace3c75ac"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a97c9babe1d4081037a90289986925726b802d180cca784ac8da2bbbc335f709"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8c4ddcbe8a19f59f57fd814b8b117d4fcea9bee7c0492e6cf5fdc22cfa563c8"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6224076c344a7694c6fbbb70d4f2a7b730f6d47d2a9dc1e7f9d9bb583faf390b"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:882821fcdf7ae8db7a951df1903d9cb032bbe838852e5fc3c2b6c3ab54e39875"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:dcec2d50756463d9df075a26a85a6affbc1b0148873da3997286caf1ce03cae1"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99c28505ecbaeb6594701a74e395b187ee083ee26478c1a795d35084d53ebd81"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9263f9e5aa4ff1dec765e99810f1cc53f0c868c5329b69f13845f699fe74f639"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:64ac6f885e3ecb2fdbb71de2701d4e34526651f1e8503af8fb30d4915a3fe345"}, + {file = "ruff-0.11.11-py3-none-win32.whl", hash = "sha256:1adcb9a18802268aaa891ffb67b1c94cd70578f126637118e8099b8e4adcf112"}, + {file = "ruff-0.11.11-py3-none-win_amd64.whl", hash = "sha256:748b4bb245f11e91a04a4ff0f96e386711df0a30412b9fe0c74d5bdc0e4a531f"}, + {file = "ruff-0.11.11-py3-none-win_arm64.whl", hash = "sha256:6c51f136c0364ab1b774767aa8b86331bd8e9d414e2d107db7a2189f35ea1f7b"}, + {file = "ruff-0.11.11.tar.gz", hash = "sha256:7774173cc7c1980e6bf67569ebb7085989a78a103922fb83ef3dfe230cd0687d"}, ] [[package]] @@ -3395,4 +3395,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "522f5bacf5610646876452e0e397038dd5c959692d2ab76214431bff78562d01" +content-hash = "9824e42dfc0e128129ee0c8641f7fe639bf47574cdd3f052dd995941abc6e44b" diff --git a/pyproject.toml b/pyproject.toml index 9507c85fee..e7a5e9c183 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -320,7 +320,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.11.10" +ruff = "0.11.11" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From d82ad6e554e00462d70c053652fc23c334f5dbe7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 11:59:59 +0200 Subject: [PATCH 342/691] Bump lxml from 5.3.0 to 5.4.0 (#18480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [lxml](https://github.com/lxml/lxml) from 5.3.0 to 5.4.0.
Release notes

Sourced from lxml's releases.

lxml-5.4.0

5.4.0 (2025-04-22)

Bugs fixed

  • LP#2107279: Binary wheels use libxml2 2.13.8 and libxslt 1.1.43 to resolve several CVEs. (Binary wheels for Windows continue to use a patched libxml2 2.11.9 and libxslt 1.1.39.) Issue found by Anatoly Katyushin, see https://bugs.launchpad.net/lxml/+bug/2107279

lxml-5.3.2

No release notes provided.

lxml-5.3.1

No release notes provided.

Changelog

Sourced from lxml's changelog.

5.4.0 (2025-04-22)

Bugs fixed

  • LP#2107279: Binary wheels use libxml2 2.13.8 and libxslt 1.1.43 to resolve several CVEs. (Binary wheels for Windows continue to use a patched libxml2 2.11.9 and libxslt 1.1.39.) Issue found by Anatoly Katyushin.

5.3.2 (2025-04-05)

This release resolves CVE-2025-24928 as described in https://gitlab.gnome.org/GNOME/libxml2/-/issues/847

Bugs fixed

  • Binary wheels use libxml2 2.12.10 and libxslt 1.1.42.

  • Binary wheels for Windows use a patched libxml2 2.11.9 and libxslt 1.1.39.

5.3.1 (2025-02-09)

Bugs fixed

  • GH#440: Some tests were adapted for libxml2 2.14.0. Patch by Nick Wellnhofer.

  • LP#2097175: DTD(external_id="…") erroneously required a byte string as ID value.

  • GH#450: iterparse() internally triggered the `DeprecationWarning`` added in lxml 5.3.0 when parsing HTML.

Other changes

  • GH#442: Binary wheels for macOS no longer use the linker flag -flat_namespace.
Commits
  • 6e76d57 Build: Exclude slow Py3.9 wheel builds for s390/ppc and Py3.7 for ARM64.
  • ee10c02 Prepare release of lxml 5.4.0.
  • 0e4f3c3 Prepare release of lxml 5.3.3.
  • b4703fc Update changelog.
  • db723bb Build: Use libxslt 1.1.43 instead of 1.1.42 to resolve some CVEs.
  • a664877 Build: Use libxml2 2.13.8 instead of 2.12.x to resolve some CVEs.
  • df4633e Remove appveyor usage.
  • 820db89 CI: Allow Py3.14 jobs to fail.
  • 93ad02a docs: Add a note about C compiler installation to error message (GH-454)
  • 16878da Add some hints to the documentation on how to build lxml (GH-453)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=lxml&package-manager=pip&previous-version=5.3.0&new-version=5.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 276 +++++++++++++++++++++++++--------------------------- 1 file changed, 135 insertions(+), 141 deletions(-) diff --git a/poetry.lock b/poetry.lock index 16d6e43f8a..cbed01b564 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1039,159 +1039,153 @@ pyasn1 = ">=0.4.6" [[package]] name = "lxml" -version = "5.3.0" +version = "5.4.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.6" groups = ["main"] markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, - {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"}, - {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"}, - {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"}, - {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"}, - {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"}, - {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"}, - {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"}, - {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"}, - {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"}, - {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"}, - {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"}, - {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"}, - {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"}, - {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"}, - {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"}, - {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"}, - {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"}, - {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"}, - {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"}, - {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"}, - {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"}, - {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"}, - {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"}, - {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"}, - {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"}, - {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"}, - {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"}, - {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"}, - {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"}, - {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"}, - {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"}, - {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"}, - {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"}, - {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"}, - {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"}, - {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"}, - {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"}, - {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"}, - {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"}, - {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"}, - {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"}, - {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"}, - {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"}, - {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"}, - {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"}, - {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"}, - {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"}, - {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"}, - {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"}, - {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"}, - {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"}, + {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, + {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696ea9e87442467819ac22394ca36cb3d01848dad1be6fac3fb612d3bd5a12cf"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef80aeac414f33c24b3815ecd560cee272786c3adfa5f31316d8b349bfade28"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9c2754cef6963f3408ab381ea55f47dabc6f78f4b8ebb0f0b25cf1ac1f7609"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a62cc23d754bb449d63ff35334acc9f5c02e6dae830d78dab4dd12b78a524f4"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f82125bc7203c5ae8633a7d5d20bcfdff0ba33e436e4ab0abc026a53a8960b7"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b67319b4aef1a6c56576ff544b67a2a6fbd7eaee485b241cabf53115e8908b8f"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:a8ef956fce64c8551221f395ba21d0724fed6b9b6242ca4f2f7beb4ce2f41997"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:0a01ce7d8479dce84fc03324e3b0c9c90b1ece9a9bb6a1b6c9025e7e4520e78c"}, + {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91505d3ddebf268bb1588eb0f63821f738d20e1e7f05d3c647a5ca900288760b"}, + {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3bcdde35d82ff385f4ede021df801b5c4a5bcdfb61ea87caabcebfc4945dc1b"}, + {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aea7c06667b987787c7d1f5e1dfcd70419b711cdb47d6b4bb4ad4b76777a0563"}, + {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7fb111eef4d05909b82152721a59c1b14d0f365e2be4c742a473c5d7372f4f5"}, + {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43d549b876ce64aa18b2328faff70f5877f8c6dede415f80a2f799d31644d776"}, + {file = "lxml-5.4.0-cp310-cp310-win32.whl", hash = "sha256:75133890e40d229d6c5837b0312abbe5bac1c342452cf0e12523477cd3aa21e7"}, + {file = "lxml-5.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:de5b4e1088523e2b6f730d0509a9a813355b7f5659d70eb4f319c76beea2e250"}, + {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9"}, + {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8"}, + {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86"}, + {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056"}, + {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7"}, + {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd"}, + {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751"}, + {file = "lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4"}, + {file = "lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539"}, + {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4"}, + {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7"}, + {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079"}, + {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20"}, + {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8"}, + {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f"}, + {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc"}, + {file = "lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f"}, + {file = "lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2"}, + {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0"}, + {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8"}, + {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982"}, + {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61"}, + {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54"}, + {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b"}, + {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a"}, + {file = "lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82"}, + {file = "lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f"}, + {file = "lxml-5.4.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7be701c24e7f843e6788353c055d806e8bd8466b52907bafe5d13ec6a6dbaecd"}, + {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb54f7c6bafaa808f27166569b1511fc42701a7713858dddc08afdde9746849e"}, + {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97dac543661e84a284502e0cf8a67b5c711b0ad5fb661d1bd505c02f8cf716d7"}, + {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:c70e93fba207106cb16bf852e421c37bbded92acd5964390aad07cb50d60f5cf"}, + {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9c886b481aefdf818ad44846145f6eaf373a20d200b5ce1a5c8e1bc2d8745410"}, + {file = "lxml-5.4.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:fa0e294046de09acd6146be0ed6727d1f42ded4ce3ea1e9a19c11b6774eea27c"}, + {file = "lxml-5.4.0-cp36-cp36m-win32.whl", hash = "sha256:61c7bbf432f09ee44b1ccaa24896d21075e533cd01477966a5ff5a71d88b2f56"}, + {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, + {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, + {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, + {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, + {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, + {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, + {file = "lxml-5.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eaf24066ad0b30917186420d51e2e3edf4b0e2ea68d8cd885b14dc8afdcf6556"}, + {file = "lxml-5.4.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b31a3a77501d86d8ade128abb01082724c0dfd9524f542f2f07d693c9f1175f"}, + {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e108352e203c7afd0eb91d782582f00a0b16a948d204d4dec8565024fafeea5"}, + {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a11a96c3b3f7551c8a8109aa65e8594e551d5a84c76bf950da33d0fb6dfafab7"}, + {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:ca755eebf0d9e62d6cb013f1261e510317a41bf4650f22963474a663fdfe02aa"}, + {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4cd915c0fb1bed47b5e6d6edd424ac25856252f09120e3e8ba5154b6b921860e"}, + {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:226046e386556a45ebc787871d6d2467b32c37ce76c2680f5c608e25823ffc84"}, + {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:b108134b9667bcd71236c5a02aad5ddd073e372fb5d48ea74853e009fe38acb6"}, + {file = "lxml-5.4.0-cp38-cp38-win32.whl", hash = "sha256:1320091caa89805df7dcb9e908add28166113dcd062590668514dbd510798c88"}, + {file = "lxml-5.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:073eb6dcdf1f587d9b88c8c93528b57eccda40209cf9be549d469b942b41d70b"}, + {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bda3ea44c39eb74e2488297bb39d47186ed01342f0022c8ff407c250ac3f498e"}, + {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ceaf423b50ecfc23ca00b7f50b64baba85fb3fb91c53e2c9d00bc86150c7e40"}, + {file = "lxml-5.4.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:664cdc733bc87449fe781dbb1f309090966c11cc0c0cd7b84af956a02a8a4729"}, + {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67ed8a40665b84d161bae3181aa2763beea3747f748bca5874b4af4d75998f87"}, + {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b4a3bd174cc9cdaa1afbc4620c049038b441d6ba07629d89a83b408e54c35cd"}, + {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b0989737a3ba6cf2a16efb857fb0dfa20bc5c542737fddb6d893fde48be45433"}, + {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:dc0af80267edc68adf85f2a5d9be1cdf062f973db6790c1d065e45025fa26140"}, + {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:639978bccb04c42677db43c79bdaa23785dc7f9b83bfd87570da8207872f1ce5"}, + {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a99d86351f9c15e4a901fc56404b485b1462039db59288b203f8c629260a142"}, + {file = "lxml-5.4.0-cp39-cp39-win32.whl", hash = "sha256:3e6d5557989cdc3ebb5302bbdc42b439733a841891762ded9514e74f60319ad6"}, + {file = "lxml-5.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:a8c9b7f16b63e65bbba889acb436a1034a82d34fa09752d754f88d708eca80e1"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1b717b00a71b901b4667226bba282dd462c42ccf618ade12f9ba3674e1fabc55"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27a9ded0f0b52098ff89dd4c418325b987feed2ea5cc86e8860b0f844285d740"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7ce10634113651d6f383aa712a194179dcd496bd8c41e191cec2099fa09de5"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53370c26500d22b45182f98847243efb518d268374a9570409d2e2276232fd37"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6364038c519dffdbe07e3cf42e6a7f8b90c275d4d1617a69bb59734c1a2d571"}, + {file = "lxml-5.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b12cb6527599808ada9eb2cd6e0e7d3d8f13fe7bbb01c6311255a15ded4c7ab4"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5f11a1526ebd0dee85e7b1e39e39a0cc0d9d03fb527f56d8457f6df48a10dc0c"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48b4afaf38bf79109bb060d9016fad014a9a48fb244e11b94f74ae366a64d252"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de6f6bb8a7840c7bf216fb83eec4e2f79f7325eca8858167b68708b929ab2172"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5cca36a194a4eb4e2ed6be36923d3cffd03dcdf477515dea687185506583d4c9"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b7c86884ad23d61b025989d99bfdd92a7351de956e01c61307cb87035960bcb1"}, + {file = "lxml-5.4.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:53d9469ab5460402c19553b56c3648746774ecd0681b1b27ea74d5d8a3ef5590"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:56dbdbab0551532bb26c19c914848d7251d73edb507c3079d6805fa8bba5b706"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14479c2ad1cb08b62bb941ba8e0e05938524ee3c3114644df905d2331c76cd57"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32697d2ea994e0db19c1df9e40275ffe84973e4232b5c274f47e7c1ec9763cdd"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:24f6df5f24fc3385f622c0c9d63fe34604893bc1a5bdbb2dbf5870f85f9a404a"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:151d6c40bc9db11e960619d2bf2ec5829f0aaffb10b41dcf6ad2ce0f3c0b2325"}, + {file = "lxml-5.4.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4025bf2884ac4370a3243c5aa8d66d3cb9e15d3ddd0af2d796eccc5f0244390e"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9459e6892f59ecea2e2584ee1058f5d8f629446eab52ba2305ae13a32a059530"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47fb24cc0f052f0576ea382872b3fc7e1f7e3028e53299ea751839418ade92a6"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50441c9de951a153c698b9b99992e806b71c1f36d14b154592580ff4a9d0d877"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ab339536aa798b1e17750733663d272038bf28069761d5be57cb4a9b0137b4f8"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9776af1aad5a4b4a1317242ee2bea51da54b2a7b7b48674be736d463c999f37d"}, + {file = "lxml-5.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:63e7968ff83da2eb6fdda967483a7a023aa497d85ad8f05c3ad9b1f2e8c84987"}, + {file = "lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd"}, ] [package.extras] cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml-html-clean"] +html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=3.0.11)"] +source = ["Cython (>=3.0.11,<3.1.0)"] [[package]] name = "lxml-stubs" From 22db145da3e3afd0cfccbfbeee9837381fbf0616 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 28 May 2025 12:29:07 +0200 Subject: [PATCH 343/691] 1.131.0rc1 --- CHANGES.md | 50 +++++++++++++++++++++++++++++++++++++++ changelog.d/18180.feature | 1 - changelog.d/18262.feature | 1 - changelog.d/18286.doc | 1 - changelog.d/18318.feature | 1 - changelog.d/18380.misc | 1 - changelog.d/18387.feature | 1 - changelog.d/18391.bugfix | 1 - changelog.d/18409.misc | 1 - changelog.d/18417.misc | 1 - changelog.d/18419.misc | 1 - changelog.d/18420.misc | 1 - changelog.d/18430.bugfix | 1 - changelog.d/18434.bugfix | 1 - changelog.d/18440.misc | 1 - changelog.d/18445.doc | 1 - changelog.d/18451.misc | 1 - changelog.d/18454.misc | 1 - changelog.d/18459.misc | 1 - changelog.d/18460.misc | 1 - changelog.d/18463.misc | 1 - changelog.d/18471.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- 24 files changed, 57 insertions(+), 22 deletions(-) delete mode 100644 changelog.d/18180.feature delete mode 100644 changelog.d/18262.feature delete mode 100644 changelog.d/18286.doc delete mode 100644 changelog.d/18318.feature delete mode 100644 changelog.d/18380.misc delete mode 100644 changelog.d/18387.feature delete mode 100644 changelog.d/18391.bugfix delete mode 100644 changelog.d/18409.misc delete mode 100644 changelog.d/18417.misc delete mode 100644 changelog.d/18419.misc delete mode 100644 changelog.d/18420.misc delete mode 100644 changelog.d/18430.bugfix delete mode 100644 changelog.d/18434.bugfix delete mode 100644 changelog.d/18440.misc delete mode 100644 changelog.d/18445.doc delete mode 100644 changelog.d/18451.misc delete mode 100644 changelog.d/18454.misc delete mode 100644 changelog.d/18459.misc delete mode 100644 changelog.d/18460.misc delete mode 100644 changelog.d/18463.misc delete mode 100644 changelog.d/18471.misc diff --git a/CHANGES.md b/CHANGES.md index d29027bbfb..3912cfb38c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,53 @@ +# Synapse 1.131.0rc1 (2025-05-28) + +### Features + +- Add `msc4263_limit_key_queries_to_users_who_share_rooms` config option as per [MSC4263](https://github.com/matrix-org/matrix-spec-proposals/pull/4263). ([\#18180](https://github.com/element-hq/synapse/issues/18180)) +- Add option to allow registrations that begin with `_`. Contributed by `_` (@hex5f). ([\#18262](https://github.com/element-hq/synapse/issues/18262)) +- Include room ID in room deletion status response. ([\#18318](https://github.com/element-hq/synapse/issues/18318)) +- Add support for calling Policy Servers ([MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284)) to mark events as spam. ([\#18387](https://github.com/element-hq/synapse/issues/18387)) + +### Bugfixes + +- Prevent race-condition in `_maybe_retry_device_resync` entrance. ([\#18391](https://github.com/element-hq/synapse/issues/18391)) +- Fix the `tests.handlers.test_worker_lock.WorkerLockTestCase.test_lock_contention` test which could spuriously time out on RISC-V architectures due to performance differences. ([\#18430](https://github.com/element-hq/synapse/issues/18430)) +- Fix admin redaction endpoint not redacting encrypted messages. ([\#18434](https://github.com/element-hq/synapse/issues/18434)) + +### Improved Documentation + +- Update `room_list_publication_rules` docs to consider defaults that changed in v1.126.0. Contributed by @HarHarLinks. ([\#18286](https://github.com/element-hq/synapse/issues/18286)) +- Add advice for upgrading between major PostgreSQL versions to the database documentation. ([\#18445](https://github.com/element-hq/synapse/issues/18445)) + +### Internal Changes + +- Fix a memory leak in `_NotifierUserStream`. ([\#18380](https://github.com/element-hq/synapse/issues/18380)) +- Fix a couple type annotations in the `RootConfig`/`Config`. ([\#18409](https://github.com/element-hq/synapse/issues/18409)) +- Explicitly enable PyPy builds in `cibuildwheel`s config to avoid it being disabled on a future upgrade to `cibuildwheel` v3. ([\#18417](https://github.com/element-hq/synapse/issues/18417)) +- Update the PR review template to remove an erroneous line break from the final bullet point. ([\#18419](https://github.com/element-hq/synapse/issues/18419)) +- Explain why we `flush_buffer()` for Python `print(...)` output. ([\#18420](https://github.com/element-hq/synapse/issues/18420)) +- Add lint to ensure we don't add a `CREATE/DROP INDEX` in a schema delta. ([\#18440](https://github.com/element-hq/synapse/issues/18440)) +- Allow checking only for the existence of a field in an SSO provider's response, rather than requiring the value(s) to check. ([\#18454](https://github.com/element-hq/synapse/issues/18454)) +- Add unit tests for homeserver usage statistics. ([\#18463](https://github.com/element-hq/synapse/issues/18463)) +- Don't move invited users to new room when shutting down room. ([\#18471](https://github.com/element-hq/synapse/issues/18471)) + + + +### Updates to locked dependencies + +* Bump actions/setup-python from 5.5.0 to 5.6.0. ([\#18398](https://github.com/element-hq/synapse/issues/18398)) +* Bump authlib from 1.5.1 to 1.5.2. ([\#18452](https://github.com/element-hq/synapse/issues/18452)) +* Bump docker/build-push-action from 6.15.0 to 6.17.0. ([\#18397](https://github.com/element-hq/synapse/issues/18397), [\#18449](https://github.com/element-hq/synapse/issues/18449)) +* Bump lxml from 5.3.0 to 5.4.0. ([\#18480](https://github.com/element-hq/synapse/issues/18480)) +* Bump mypy-zope from 1.0.9 to 1.0.11. ([\#18428](https://github.com/element-hq/synapse/issues/18428)) +* Bump pyo3 from 0.23.5 to 0.24.2. ([\#18460](https://github.com/element-hq/synapse/issues/18460)) +* Bump pyo3-log from 0.12.3 to 0.12.4. ([\#18453](https://github.com/element-hq/synapse/issues/18453)) +* Bump pyopenssl from 25.0.0 to 25.1.0. ([\#18450](https://github.com/element-hq/synapse/issues/18450)) +* Bump ruff from 0.7.3 to 0.11.11. ([\#18451](https://github.com/element-hq/synapse/issues/18451), [\#18482](https://github.com/element-hq/synapse/issues/18482)) +* Bump tornado from 6.4.2 to 6.5.0. ([\#18459](https://github.com/element-hq/synapse/issues/18459)) +* Bump setuptools from 72.1.0 to 78.1.1. ([\#18461](https://github.com/element-hq/synapse/issues/18461)) +* Bump types-jsonschema from 4.23.0.20241208 to 4.23.0.20250516. ([\#18481](https://github.com/element-hq/synapse/issues/18481)) +* Bump types-requests from 2.32.0.20241016 to 2.32.0.20250328. ([\#18427](https://github.com/element-hq/synapse/issues/18427)) + # Synapse 1.130.0 (2025-05-20) ### Bugfixes diff --git a/changelog.d/18180.feature b/changelog.d/18180.feature deleted file mode 100644 index fbf226e51c..0000000000 --- a/changelog.d/18180.feature +++ /dev/null @@ -1 +0,0 @@ -Add `msc4263_limit_key_queries_to_users_who_share_rooms` config option as per [MSC4263](https://github.com/matrix-org/matrix-spec-proposals/pull/4263). diff --git a/changelog.d/18262.feature b/changelog.d/18262.feature deleted file mode 100644 index c8249faa76..0000000000 --- a/changelog.d/18262.feature +++ /dev/null @@ -1 +0,0 @@ -Add option to allow registrations that begin with `_`. Contributed by `_` (@hex5f). diff --git a/changelog.d/18286.doc b/changelog.d/18286.doc deleted file mode 100644 index 37728351c5..0000000000 --- a/changelog.d/18286.doc +++ /dev/null @@ -1 +0,0 @@ -Update `room_list_publication_rules` docs to consider defaults that changed in v1.126.0. Contributed by @HarHarLinks. diff --git a/changelog.d/18318.feature b/changelog.d/18318.feature deleted file mode 100644 index fba0e83577..0000000000 --- a/changelog.d/18318.feature +++ /dev/null @@ -1 +0,0 @@ -Include room ID in room deletion status response. diff --git a/changelog.d/18380.misc b/changelog.d/18380.misc deleted file mode 100644 index 5d6017be26..0000000000 --- a/changelog.d/18380.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a memory leak in `_NotifierUserStream`. \ No newline at end of file diff --git a/changelog.d/18387.feature b/changelog.d/18387.feature deleted file mode 100644 index 2d9ff2cea2..0000000000 --- a/changelog.d/18387.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for calling Policy Servers ([MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284)) to mark events as spam. \ No newline at end of file diff --git a/changelog.d/18391.bugfix b/changelog.d/18391.bugfix deleted file mode 100644 index bbcb7b7a28..0000000000 --- a/changelog.d/18391.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent race-condition in `_maybe_retry_device_resync` entrance. diff --git a/changelog.d/18409.misc b/changelog.d/18409.misc deleted file mode 100644 index bbb9bdbb1b..0000000000 --- a/changelog.d/18409.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a couple type annotations in the `RootConfig`/`Config`. diff --git a/changelog.d/18417.misc b/changelog.d/18417.misc deleted file mode 100644 index 5f650a202a..0000000000 --- a/changelog.d/18417.misc +++ /dev/null @@ -1 +0,0 @@ -Explicitly enable PyPy builds in `cibuildwheel`s config to avoid it being disabled on a future upgrade to `cibuildwheel` v3. \ No newline at end of file diff --git a/changelog.d/18419.misc b/changelog.d/18419.misc deleted file mode 100644 index 0ff36e27b8..0000000000 --- a/changelog.d/18419.misc +++ /dev/null @@ -1 +0,0 @@ -Update the PR review template to remove an erroneous line break from the final bullet point. \ No newline at end of file diff --git a/changelog.d/18420.misc b/changelog.d/18420.misc deleted file mode 100644 index d52175af91..0000000000 --- a/changelog.d/18420.misc +++ /dev/null @@ -1 +0,0 @@ -Explain why we `flush_buffer()` for Python `print(...)` output. diff --git a/changelog.d/18430.bugfix b/changelog.d/18430.bugfix deleted file mode 100644 index d61fd56c33..0000000000 --- a/changelog.d/18430.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `tests.handlers.test_worker_lock.WorkerLockTestCase.test_lock_contention` test which could spuriously time out on RISC-V architectures due to performance differences. \ No newline at end of file diff --git a/changelog.d/18434.bugfix b/changelog.d/18434.bugfix deleted file mode 100644 index dd094c83e8..0000000000 --- a/changelog.d/18434.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix admin redaction endpoint not redacting encrypted messages. \ No newline at end of file diff --git a/changelog.d/18440.misc b/changelog.d/18440.misc deleted file mode 100644 index 6aaa6dde5c..0000000000 --- a/changelog.d/18440.misc +++ /dev/null @@ -1 +0,0 @@ -Add lint to ensure we don't add a `CREATE/DROP INDEX` in a schema delta. diff --git a/changelog.d/18445.doc b/changelog.d/18445.doc deleted file mode 100644 index 1e05a791b2..0000000000 --- a/changelog.d/18445.doc +++ /dev/null @@ -1 +0,0 @@ -Add advice for upgrading between major PostgreSQL versions to the database documentation. diff --git a/changelog.d/18451.misc b/changelog.d/18451.misc deleted file mode 100644 index 593e83eb7f..0000000000 --- a/changelog.d/18451.misc +++ /dev/null @@ -1 +0,0 @@ -Bump ruff from 0.7.3 to 0.11.10. \ No newline at end of file diff --git a/changelog.d/18454.misc b/changelog.d/18454.misc deleted file mode 100644 index 892fbd1d94..0000000000 --- a/changelog.d/18454.misc +++ /dev/null @@ -1 +0,0 @@ -Allow checking only for the existence of a field in an SSO provider's response, rather than requiring the value(s) to check. \ No newline at end of file diff --git a/changelog.d/18459.misc b/changelog.d/18459.misc deleted file mode 100644 index e148825696..0000000000 --- a/changelog.d/18459.misc +++ /dev/null @@ -1 +0,0 @@ -Bump tornado from 6.4.2 to 6.5.0. \ No newline at end of file diff --git a/changelog.d/18460.misc b/changelog.d/18460.misc deleted file mode 100644 index 5aa19683eb..0000000000 --- a/changelog.d/18460.misc +++ /dev/null @@ -1 +0,0 @@ -Bump pyo3 from 0.23.5 to 0.24.2. \ No newline at end of file diff --git a/changelog.d/18463.misc b/changelog.d/18463.misc deleted file mode 100644 index 1264758d7c..0000000000 --- a/changelog.d/18463.misc +++ /dev/null @@ -1 +0,0 @@ -Add unit tests for homeserver usage statistics. \ No newline at end of file diff --git a/changelog.d/18471.misc b/changelog.d/18471.misc deleted file mode 100644 index b36712bea3..0000000000 --- a/changelog.d/18471.misc +++ /dev/null @@ -1 +0,0 @@ -Don't move invited users to new room when shutting down room. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 56776a7d86..294e5e449b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.131.0~rc1) stable; urgency=medium + + * New synapse release 1.131.0rc1. + + -- Synapse Packaging team Wed, 28 May 2025 10:25:44 +0000 + matrix-synapse-py3 (1.130.0) stable; urgency=medium * New Synapse release 1.130.0. diff --git a/pyproject.toml b/pyproject.toml index e7a5e9c183..762a095926 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.130.0" +version = "1.131.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 461571fcf2094b37078b67da95d3cd56b5aff24e Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 28 May 2025 12:36:28 +0200 Subject: [PATCH 344/691] Changelog fixes Co-Authored-By: Andrew Morgan --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3912cfb38c..2ddf511110 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ - Add `msc4263_limit_key_queries_to_users_who_share_rooms` config option as per [MSC4263](https://github.com/matrix-org/matrix-spec-proposals/pull/4263). ([\#18180](https://github.com/element-hq/synapse/issues/18180)) - Add option to allow registrations that begin with `_`. Contributed by `_` (@hex5f). ([\#18262](https://github.com/element-hq/synapse/issues/18262)) -- Include room ID in room deletion status response. ([\#18318](https://github.com/element-hq/synapse/issues/18318)) +- Include room ID in response to the [Room Deletion Status Admin API](https://element-hq.github.io/synapse/latest/admin_api/rooms.html#status-of-deleting-rooms). ([\#18318](https://github.com/element-hq/synapse/issues/18318)) - Add support for calling Policy Servers ([MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284)) to mark events as spam. ([\#18387](https://github.com/element-hq/synapse/issues/18387)) ### Bugfixes From 2436512a2522c480f33dcf5ce54f8d5f9371ba54 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 28 May 2025 07:20:27 -0400 Subject: [PATCH 345/691] Mark dehydrated devices in admin get devices endpoint (#18252) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18252.misc | 1 + docs/admin_api/user_admin_api.md | 7 +++- synapse/rest/admin/devices.py | 11 ++++++ tests/rest/admin/test_device.py | 64 ++++++++++++++++++++++++++++++-- 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18252.misc diff --git a/changelog.d/18252.misc b/changelog.d/18252.misc new file mode 100644 index 0000000000..6c76aca29c --- /dev/null +++ b/changelog.d/18252.misc @@ -0,0 +1 @@ +Mark dehydrated devices in the [List All User Devices Admin API](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#list-all-devices). \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 875876081f..ee22b0db50 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -954,7 +954,8 @@ A response body like the following is returned: "last_seen_ip": "1.2.3.4", "last_seen_user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0", "last_seen_ts": 1474491775024, - "user_id": "" + "user_id": "", + "dehydrated": false }, { "device_id": "AUIECTSRND", @@ -962,7 +963,8 @@ A response body like the following is returned: "last_seen_ip": "1.2.3.5", "last_seen_user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0", "last_seen_ts": 1474491775025, - "user_id": "" + "user_id": "", + "dehydrated": false } ], "total": 2 @@ -992,6 +994,7 @@ The following fields are returned in the JSON response body: - `last_seen_ts` - The timestamp (in milliseconds since the unix epoch) when this devices was last seen. (May be a few minutes out of date, for efficiency reasons). - `user_id` - Owner of device. + - `dehydrated` - Whether the device is a dehydrated device. - `total` - Total number of user's devices. diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 125ed8c491..09baf8ce21 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -145,6 +145,17 @@ class DevicesGetRestServlet(RestServlet): devices = await self.device_worker_handler.get_devices_by_user( target_user.to_string() ) + + # mark the dehydrated device by adding a "dehydrated" flag + dehydrated_device_info = await self.device_worker_handler.get_dehydrated_device( + target_user.to_string() + ) + if dehydrated_device_info: + dehydrated_device_id = dehydrated_device_info[0] + for device in devices: + is_dehydrated = device["device_id"] == dehydrated_device_id + device["dehydrated"] = is_dehydrated + return HTTPStatus.OK, {"devices": devices, "total": len(devices)} diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index a88c77bd19..531162a6e9 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -27,7 +27,7 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes from synapse.handlers.device import DeviceHandler -from synapse.rest.client import login +from synapse.rest.client import devices, login from synapse.server import HomeServer from synapse.util import Clock @@ -299,6 +299,7 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): class DevicesRestTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, + devices.register_servlets, login.register_servlets, ] @@ -390,15 +391,63 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): self.assertEqual(0, channel.json_body["total"]) self.assertEqual(0, len(channel.json_body["devices"])) + @unittest.override_config( + {"experimental_features": {"msc2697_enabled": False, "msc3814_enabled": True}} + ) def test_get_devices(self) -> None: """ Tests that a normal lookup for devices is successfully """ # Create devices number_devices = 5 - for _ in range(number_devices): + # we create 2 fewer devices in the loop, because we will create another + # login after the loop, and we will create a dehydrated device + for _ in range(number_devices - 2): self.login("user", "pass") + other_user_token = self.login("user", "pass") + dehydrated_device_url = ( + "/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device" + ) + content = { + "device_data": { + "algorithm": "m.dehydration.v1.olm", + }, + "device_id": "dehydrated_device", + "initial_device_display_name": "foo bar", + "device_keys": { + "user_id": "@user:test", + "device_id": "dehydrated_device", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], + "keys": { + ":": "", + }, + "signatures": { + "@user:test": {":": ""} + }, + }, + "fallback_keys": { + "alg1:device1": "f4llb4ckk3y", + "signed_:": { + "fallback": "true", + "key": "f4llb4ckk3y", + "signatures": { + "@user:test": {":": ""} + }, + }, + }, + "one_time_keys": {"alg1:k1": "0net1m3k3y"}, + } + self.make_request( + "PUT", + dehydrated_device_url, + access_token=other_user_token, + content=content, + ) + # Get devices channel = self.make_request( "GET", @@ -410,13 +459,22 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): self.assertEqual(number_devices, channel.json_body["total"]) self.assertEqual(number_devices, len(channel.json_body["devices"])) self.assertEqual(self.other_user, channel.json_body["devices"][0]["user_id"]) - # Check that all fields are available + # Check that all fields are available, and that the dehydrated device is marked as dehydrated + found_dehydrated = False for d in channel.json_body["devices"]: self.assertIn("user_id", d) self.assertIn("device_id", d) self.assertIn("display_name", d) self.assertIn("last_seen_ip", d) self.assertIn("last_seen_ts", d) + if d["device_id"] == "dehydrated_device": + self.assertTrue(d.get("dehydrated")) + found_dehydrated = True + else: + # Either the field is not present, or set to False + self.assertFalse(d.get("dehydrated")) + + self.assertTrue(found_dehydrated) class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): From fae72f181b06a771fef70dc421ae357e60c2d9d5 Mon Sep 17 00:00:00 2001 From: V02460 Date: Tue, 3 Jun 2025 11:29:38 +0200 Subject: [PATCH 346/691] Machine-readable config description (#17892) --- .github/workflows/docs.yaml | 12 + .github/workflows/schema.yaml | 54 + changelog.d/17892.doc | 1 + docs/README.md | 12 + docs/usage/administration/admin_faq.md | 2 +- .../configuration/config_documentation.md | 4492 ++++++------- schema/synapse-config.schema.yaml | 5711 +++++++++++++++++ schema/v1/Do not edit files in this folder | 2 + schema/v1/meta.schema.json | 29 + schema/v1/vocab/documentation.html | 11 + scripts-dev/gen_config_documentation.py | 503 ++ scripts-dev/release.py | 6 + 12 files changed, 8356 insertions(+), 2479 deletions(-) create mode 100644 .github/workflows/schema.yaml create mode 100644 changelog.d/17892.doc create mode 100644 schema/synapse-config.schema.yaml create mode 100644 schema/v1/Do not edit files in this folder create mode 100644 schema/v1/meta.schema.json create mode 100644 schema/v1/vocab/documentation.html create mode 100755 scripts-dev/gen_config_documentation.py diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 930c71a8b4..64fb437b42 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -78,6 +78,18 @@ jobs: mdbook build cp book/welcome_and_overview.html book/index.html + - name: Prepare and publish schema files + run: | + sudo apt-get update && sudo apt-get install -y yq + mkdir -p book/schema + # Remove developer notice before publishing. + rm schema/v*/Do\ not\ edit\ files\ in\ this\ folder + # Copy schema files that are independent from current Synapse version. + cp -r -t book/schema schema/v*/ + # Convert config schema from YAML source file to JSON. + yq < schema/synapse-config.schema.yaml \ + > book/schema/synapse-config.schema.json + # Deploy to the target directory. - name: Deploy to gh pages uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml new file mode 100644 index 0000000000..41f3cd01be --- /dev/null +++ b/.github/workflows/schema.yaml @@ -0,0 +1,54 @@ +name: Schema + +on: + pull_request: + paths: + - schema/** + - docs/usage/configuration/config_documentation.md + +jobs: + validate-schema: + name: Ensure Synapse config schema is valid + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + with: + python-version: "3.x" + - name: Install check-jsonschema + run: pip install check-jsonschema==0.33.0 + + - name: Validate meta schema + run: check-jsonschema --check-metaschema schema/v*/meta.schema.json + - name: Validate schema + run: |- + # Please bump on introduction of a new meta schema. + LATEST_META_SCHEMA_VERSION=v1 + check-jsonschema \ + --schemafile="schema/$LATEST_META_SCHEMA_VERSION/meta.schema.json" \ + schema/synapse-config.schema.yaml + - name: Validate default config + # Populates the empty instance with default values and checks against the schema. + run: |- + echo "{}" | check-jsonschema \ + --fill-defaults --schemafile=schema/synapse-config.schema.yaml - + + check-doc-generation: + name: Ensure generated documentation is up-to-date + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + with: + python-version: "3.x" + - name: Install PyYAML + run: pip install PyYAML==6.0.2 + + - name: Regenerate config documentation + run: | + scripts-dev/gen_config_documentation.py \ + schema/synapse-config.schema.yaml \ + > docs/usage/configuration/config_documentation.md + - name: Error in case of any differences + # Errors if there are now any modified files (untracked files are ignored). + run: 'git diff || ! git status --porcelain=1 | grep "^ M"' diff --git a/changelog.d/17892.doc b/changelog.d/17892.doc new file mode 100644 index 0000000000..83c03190dd --- /dev/null +++ b/changelog.d/17892.doc @@ -0,0 +1 @@ +Generate config documentation from JSON Schema file. diff --git a/docs/README.md b/docs/README.md index 0b2b910c73..7802d3c3ce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -63,6 +63,18 @@ mdbook serve The URL at which the docs can be viewed at will be logged. +## Synapse configuration documentation + +The [Configuration +Manual](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html) +page is generated from a YAML file, +[schema/synapse-config.schema.yaml](../schema/synapse-config.schema.yaml). To +add new options or modify existing ones, first edit that file, then run +[scripts-dev/gen_config_documentation.py](../scripts-dev/gen_config_documentation.py) +to generate an updated Configuration Manual markdown file. + +Build the book as described above to preview it in a web browser. + ## Configuration and theming The look and behaviour of the website is configured by the [book.toml](../book.toml) file diff --git a/docs/usage/administration/admin_faq.md b/docs/usage/administration/admin_faq.md index 1be432000c..20f8c6a157 100644 --- a/docs/usage/administration/admin_faq.md +++ b/docs/usage/administration/admin_faq.md @@ -255,7 +255,7 @@ line to `/etc/default/matrix-synapse`: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -*Note*: You may need to set `PYTHONMALLOC=malloc` to ensure that `jemalloc` can accurately calculate memory usage. By default, Python uses its internal small-object allocator, which may interfere with jemalloc's ability to track memory consumption correctly. This could prevent the [cache_autotuning](../configuration/config_documentation.md#caches-and-associated-values) feature from functioning as expected, as the Python allocator may not reach the memory threshold set by `max_cache_memory_usage`, thus not triggering the cache eviction process. +*Note*: You may need to set `PYTHONMALLOC=malloc` to ensure that `jemalloc` can accurately calculate memory usage. By default, Python uses its internal small-object allocator, which may interfere with jemalloc's ability to track memory consumption correctly. This could prevent the [cache_autotuning](../configuration/config_documentation.md#caches) feature from functioning as expected, as the Python allocator may not reach the memory threshold set by `max_cache_memory_usage`, thus not triggering the cache eviction process. This made a significant difference on Python 2.7 - it's unclear how much of an improvement it provides on Python 3.x. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 3927b9ca14..b9845dd78f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1,3 +1,5 @@ + + # Configuring Synapse This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified @@ -90,32 +92,32 @@ apply if you want your config file to be read properly. A few helpful things to the sub-options, if any, are identified and listed in the body of the section. In addition, each setting has an example of its usage, with the proper indentation shown. - +--- ## Modules Server admins can expand Synapse's functionality with external modules. -See [here](../../modules/index.md) for more -documentation on how to configure or create custom modules for Synapse. - +See [here](../../modules/index.md) for more documentation on how to configure or create custom modules for Synapse. --- ### `modules` -Use the `module` sub-option to add modules under this option to extend functionality. -The `module` setting then has a sub-option, `config`, which can be used to define some configuration -for the `module`. +*(array)* Use the `module` sub-option to add modules under this option to extend functionality. The `module` setting then has a sub-option, `config`, which can be used to define some configuration for the `module`. Defaults to `[]`. -Defaults to none. +Options for each entry include: + +* `module` (string): Path to the Python class of the module. + +* `config` (object): Configuration options for the module. Example configuration: ```yaml modules: - - module: my_super_module.MySuperClass - config: - do_thing: true - - module: my_other_super_module.SomeClass - config: {} +- module: my_super_module.MySuperClass + config: + do_thing: true +- module: my_other_super_module.SomeClass + config: {} ``` --- ## Server @@ -125,37 +127,28 @@ Define your homeserver name and other base options. --- ### `server_name` -This sets the public-facing domain of the server. +*(string)* This sets the public-facing domain of the server. -The `server_name` name will appear at the end of usernames and room addresses -created on your server. For example if the `server_name` was example.com, -usernames on your server would be in the format `@user:example.com` +The `server_name` name will appear at the end of usernames and room addresses created on your server. For example if the `server_name` was example.com, usernames on your server would be in the format `@user:example.com`. -In most cases you should avoid using a matrix specific subdomain such as -matrix.example.com or synapse.example.com as the `server_name` for the same -reasons you wouldn't use user@email.example.com as your email address. -See [here](../../delegate.md) -for information on how to host Synapse on a subdomain while preserving -a clean `server_name`. +In most cases you should avoid using a matrix specific subdomain such as matrix.example.com or synapse.example.com as the `server_name` for the same reasons you wouldn't use user@email.example.com as your email address. See [here](../../delegate.md) for information on how to host Synapse on a subdomain while preserving a clean `server_name`. -The `server_name` cannot be changed later so it is important to -configure this correctly before you start Synapse. It should be all -lowercase and may contain an explicit port. +The `server_name` cannot be changed later so it is important to configure this correctly before you start Synapse. It should be all lowercase and may contain an explicit port. There is no default for this option. -Example configuration #1: +Example configurations: ```yaml server_name: matrix.org ``` -Example configuration #2: + ```yaml server_name: localhost:8080 ``` --- ### `pid_file` -When running Synapse as a daemon, the file to store the pid in. Defaults to none. +*(string|null)* When running Synapse as a daemon, the file to store the pid in. Defaults to `null`. Example configuration: ```yaml @@ -164,14 +157,14 @@ pid_file: DATADIR/homeserver.pid --- ### `daemonize` -Specifies whether Synapse should be started as a daemon process. If Synapse is being -managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to -`false`. +*(boolean)* Specifies whether Synapse should be started as a daemon process. If Synapse is being managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to `false`. This can also be set by the `--daemonize` (`-D`) argument when starting Synapse. See `worker_daemonize` for more information on daemonizing workers. +Defaults to `false`. + Example configuration: ```yaml daemonize: true @@ -179,10 +172,12 @@ daemonize: true --- ### `print_pidfile` -Print the path to the pidfile just before daemonizing. Defaults to false. +*(boolean)* Print the path to the pidfile just before daemonizing. This can also be set by the `--print-pidfile` argument when starting Synapse. +Defaults to `false`. + Example configuration: ```yaml print_pidfile: true @@ -190,19 +185,16 @@ print_pidfile: true --- ### `user_agent_suffix` -A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). Defaults -to None +*(string|null)* A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). Defaults to `null`. Example configuration: ```yaml -user_agent_suffix: " (I'm a teapot; Linux x86_64)" +user_agent_suffix: ' (I''m a teapot; Linux x86_64)' ``` --- ### `use_frozen_dicts` -Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing -prevents bugs where we accidentally share e.g. signature dicts. However, freezing a -dict is expensive. Defaults to false. +*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. Defaults to `false`. Example configuration: ```yaml @@ -211,7 +203,7 @@ use_frozen_dicts: true --- ### `web_client_location` -The absolute URL to the web client which `/` will redirect to. Defaults to none. +*(string|null)* The absolute URL to the web client which `/` will redirect to. Defaults to `null`. Example configuration: ```yaml @@ -220,14 +212,11 @@ web_client_location: https://riot.example.com/ --- ### `public_baseurl` -The public-facing base URL that clients use to access this Homeserver (not -including _matrix/...). This is the same URL a user might enter into the -'Custom Homeserver URL' field on their client. If you use Synapse with a -reverse proxy, this should be the URL to reach Synapse via the proxy. -Otherwise, it should be the URL to reach Synapse's client HTTP listener (see -['listeners'](#listeners) below). +*(string|null)* The public-facing base URL that clients use to access this Homeserver (not including _matrix/...). This is the same URL a user might enter into the "Custom Homeserver URL" field on their client. If you use Synapse with a reverse proxy, this should be the URL to reach Synapse via the proxy. Otherwise, it should be the URL to reach Synapse's client HTTP listener (see [`listeners`](#listeners) below). -Defaults to `https:///`. +If unset or null, `https:///` is used. + +Defaults to `null`. Example configuration: ```yaml @@ -236,46 +225,39 @@ public_baseurl: https://example.com/ --- ### `serve_server_wellknown` -By default, other servers will try to reach our server on port 8448, which can -be inconvenient in some environments. +*(boolean)* By default, other servers will try to reach our server on port 8448, which can be inconvenient in some environments. -Provided `https:///` on port 443 is routed to Synapse, this -option configures Synapse to serve a file at `https:///.well-known/matrix/server`. -This will tell other servers to send traffic to port 443 instead. +Provided `https:///` on port 443 is routed to Synapse, this option configures Synapse to serve a file at `https:///.well-known/matrix/server`. This will tell other servers to send traffic to port 443 instead. This option currently defaults to false. -See [Delegation of incoming federation traffic](../../delegate.md) for more -information. +See [Delegation of incoming federation traffic](../../delegate.md) for more information. + +Defaults to `false`. Example configuration: ```yaml serve_server_wellknown: true ``` --- -### `extra_well_known_client_content ` +### `extra_well_known_client_content` -This option allows server runners to add arbitrary key-value pairs to the [client-facing `.well-known` response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). -Note that the `public_baseurl` config option must be provided for Synapse to serve a response to `/.well-known/matrix/client` at all. +*(object)* This option allows server runners to add arbitrary key-value pairs to the [client-facing `.well-known` response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). Note that the `public_baseurl` config option must be provided for Synapse to serve a response to `/.well-known/matrix/client` at all. -If this option is provided, it parses the given yaml to json and -serves it on `/.well-known/matrix/client` endpoint -alongside the standard properties. +If this option is provided, it parses the given yaml to json and serves it on `/.well-known/matrix/client` endpoint alongside the standard properties. *Added in Synapse 1.62.0.* Example configuration: ```yaml -extra_well_known_client_content : +extra_well_known_client_content: option1: value1 option2: value2 ``` --- ### `soft_file_limit` -Set the soft limit on the number of file descriptors synapse can use. -Zero is used to indicate synapse should set the soft limit to the hard limit. -Defaults to 0. +*(integer)* Set the soft limit on the number of file descriptors synapse can use. Zero is used to indicate synapse should set the soft limit to the hard limit. Defaults to `0`. Example configuration: ```yaml @@ -284,10 +266,19 @@ soft_file_limit: 3 --- ### `presence` -Presence tracking allows users to see the state (e.g online/offline) -of other local and remote users. Set the `enabled` sub-option to false to -disable presence tracking on this homeserver. Defaults to true. -This option replaces the previous top-level 'use_presence' option. +*(object)* Presence tracking allows users to see the state (e.g online/offline) of other local and remote users. This option replaces the previous top-level `use_presence` option. + +This setting has the following sub-options: + +* `enabled` (boolean|string): Set to false to disable presence tracking on this homeserver. + + Can also be set to a special value of "untracked" which ignores updates received via clients and federation, while still accepting updates from the [module API](../../modules/index.md). + + *The "untracked" option was added in Synapse 1.96.0.* + + Defaults to `true`. + +* `include_offline_users_on_sync` (boolean): When clients perform an initial or `full_state` sync, presence results for offline users are not included by default. Setting `include_offline_users_on_sync` to `true` will always include offline users in the results. Defaults to `false`. Example configuration: ```yaml @@ -295,23 +286,10 @@ presence: enabled: false include_offline_users_on_sync: false ``` - -`enabled` can also be set to a special value of "untracked" which ignores updates -received via clients and federation, while still accepting updates from the -[module API](../../modules/index.md). - -*The "untracked" option was added in Synapse 1.96.0.* - -When clients perform an initial or `full_state` sync, presence results for offline users are -not included by default. Setting `include_offline_users_on_sync` to `true` will always include -offline users in the results. Defaults to false. - --- ### `require_auth_for_profile_requests` -Whether to require authentication to retrieve profile data (avatars, display names) of other -users through the client API. Defaults to false. Note that profile data is also available -via the federation API, unless `allow_profile_lookup_over_federation` is set to false. +*(boolean)* Whether to require authentication to retrieve profile data (avatars, display names) of other users through the client API. Note that profile data is also available via the federation API, unless `allow_profile_lookup_over_federation` is set to false. Defaults to `false`. Example configuration: ```yaml @@ -320,10 +298,7 @@ require_auth_for_profile_requests: true --- ### `limit_profile_requests_to_users_who_share_rooms` -Use this option to require a user to share a room with another user in order -to retrieve their profile information. Only checked on Client-Server -requests. Profile requests from other servers should be checked by the -requesting server. Defaults to false. +*(boolean)* Use this option to require a user to share a room with another user in order to retrieve their profile information. Only checked on Client-Server requests. Profile requests from other servers should be checked by the requesting server. Defaults to `false`. Example configuration: ```yaml @@ -332,11 +307,7 @@ limit_profile_requests_to_users_who_share_rooms: true --- ### `include_profile_data_on_invite` -Use this option to prevent a user's profile data from being retrieved and -displayed in a room until they have joined it. By default, a user's -profile data is included in an invite event, regardless of the values -of the above two settings, and whether or not the users share a server. -Defaults to true. +*(boolean)* Use this option to prevent a user's profile data from being retrieved and displayed in a room until they have joined it. By default, a user's profile data is included in an invite event, regardless of the values of the above two settings, and whether or not the users share a server. Defaults to `true`. Example configuration: ```yaml @@ -345,9 +316,7 @@ include_profile_data_on_invite: false --- ### `allow_public_rooms_without_auth` -If set to true, removes the need for authentication to access the server's -public rooms directory through the client API, meaning that anyone can -query the room directory. Defaults to false. +*(boolean)* If set to true, removes the need for authentication to access the server's public rooms directory through the client API, meaning that anyone can query the room directory. Defaults to `false`. Example configuration: ```yaml @@ -356,8 +325,7 @@ allow_public_rooms_without_auth: true --- ### `allow_public_rooms_over_federation` -If set to true, allows any other homeserver to fetch the server's public -rooms directory via federation. Defaults to false. +*(boolean)* If set to true, allows any other homeserver to fetch the server's public rooms directory via federation. Defaults to `false`. Example configuration: ```yaml @@ -366,50 +334,56 @@ allow_public_rooms_over_federation: true --- ### `default_room_version` -The default room version for newly created rooms on this server. +*(string)* The default room version for newly created rooms on this server. Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) -For example, for room version 1, `default_room_version` should be set -to "1". - -Currently defaults to ["10"](https://spec.matrix.org/v1.5/rooms/v10/). +For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). +Defaults to `"10"`. + Example configuration: ```yaml -default_room_version: "8" +default_room_version: '8' ``` --- ### `gc_thresholds` -The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. -Defaults to none. +*(array|null)* The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. Defaults to `null`. Example configuration: ```yaml -gc_thresholds: [700, 10, 10] +gc_thresholds: +- 700 +- 10 +- 10 ``` --- ### `gc_min_interval` -The minimum time in seconds between each GC for a generation, regardless of -the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` -indicates that a second must pass between consecutive generation 0 GCs, etc. +*(array)* The minimum time in seconds between each GC for a generation, regardless of the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive generation 0 GCs, etc. -Defaults to `[1s, 10s, 30s]`. +Default configuration: +```yaml +gc_min_interval: +- 1s +- 10s +- 30s +``` Example configuration: ```yaml -gc_min_interval: [0.5s, 30s, 1m] +gc_min_interval: +- 0.5s +- 30s +- 1m ``` --- ### `filter_timeline_limit` -Set the limit on the returned events in the timeline in the get -and sync operations. Defaults to 100. A value of -1 means no upper limit. - +*(integer)* Set the limit on the returned events in the timeline in the get and sync operations. A value of -1 means no upper limit. Defaults to `100`. Example configuration: ```yaml @@ -418,8 +392,7 @@ filter_timeline_limit: 5000 --- ### `block_non_admin_invites` -Whether room invites to users on this server should be blocked -(except those sent by local server admins). Defaults to false. +*(boolean)* Whether room invites to users on this server should be blocked (except those sent by local server admins). Defaults to `false`. Example configuration: ```yaml @@ -428,8 +401,7 @@ block_non_admin_invites: true --- ### `enable_search` -If set to false, new messages will not be indexed for searching and users -will receive errors when searching for messages. Defaults to true. +*(boolean)* If set to false, new messages will not be indexed for searching and users will receive errors when searching for messages. Defaults to `true`. Example configuration: ```yaml @@ -438,126 +410,61 @@ enable_search: false --- ### `ip_range_blacklist` -This option prevents outgoing requests from being sent to the specified blacklisted IP address -CIDR ranges. If this option is not specified then it defaults to private IP -address ranges (see the example below). +*(array)* This option prevents outgoing requests from being sent to the specified blacklisted IP address CIDR ranges. If this option is not specified then it defaults to private IP address ranges (see the example below). -The blacklist applies to the outbound requests for federation, identity servers, -push servers, and for checking key validity for third-party invite events. +The blacklist applies to the outbound requests for federation, identity servers, push servers, and for checking key validity for third-party invite events. -(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -listed here, since they correspond to unroutable addresses.) +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly listed here, since they correspond to unroutable addresses.) This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. Note: The value is ignored when an HTTP proxy is in use. -Example configuration: +Default configuration: ```yaml ip_range_blacklist: - - '127.0.0.0/8' - - '10.0.0.0/8' - - '172.16.0.0/12' - - '192.168.0.0/16' - - '100.64.0.0/10' - - '192.0.0.0/24' - - '169.254.0.0/16' - - '192.88.99.0/24' - - '198.18.0.0/15' - - '192.0.2.0/24' - - '198.51.100.0/24' - - '203.0.113.0/24' - - '224.0.0.0/4' - - '::1/128' - - 'fe80::/10' - - 'fc00::/7' - - '2001:db8::/32' - - 'ff00::/8' - - 'fec0::/10' +- 127.0.0.0/8 +- 10.0.0.0/8 +- 172.16.0.0/12 +- 192.168.0.0/16 +- 100.64.0.0/10 +- 192.0.0.0/24 +- 169.254.0.0/16 +- 192.88.99.0/24 +- 198.18.0.0/15 +- 192.0.2.0/24 +- 198.51.100.0/24 +- 203.0.113.0/24 +- 224.0.0.0/4 +- ::1/128 +- fe80::/10 +- fc00::/7 +- 2001:db8::/32 +- ff00::/8 +- fec0::/10 ``` --- ### `ip_range_whitelist` -List of IP address CIDR ranges that should be allowed for federation, -identity servers, push servers, and for checking key validity for -third-party invite events. This is useful for specifying exceptions to -wide-ranging blacklisted target IP ranges - e.g. for communication with -a push server only visible in your network. +*(array)* List of IP address CIDR ranges that should be allowed for federation, identity servers, push servers, and for checking key validity for third-party invite events. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges – e.g. for communication with a push server only visible in your network. -This whitelist overrides `ip_range_blacklist` and defaults to an empty -list. +This whitelist overrides `ip_range_blacklist`. + +Defaults to `[]`. Example configuration: ```yaml ip_range_whitelist: - - '192.168.1.1' +- 192.168.1.1 ``` --- ### `listeners` -List of ports that Synapse should listen on, their purpose and their -configuration. - -Sub-options for each listener include: - -* `port`: the TCP port to bind to. - -* `tag`: An alias for the port in the logger name. If set the tag is logged instead -of the port. Default to `None`, is optional and only valid for listener with `type: http`. -See the docs [request log format](../administration/request_log.md). - -* `bind_addresses`: a list of local addresses to listen on. The default is - 'all local interfaces'. - -* `type`: the type of listener. Normally `http`, but other valid options are: - - * `manhole`: (see the docs [here](../../manhole.md)), - - * `metrics`: (see the docs [here](../../metrics-howto.md)), - -* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. - -* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is - behind a [reverse-proxy](../../reverse_proxy.md). - -* `request_id_header`: The header extracted from each incoming request that is - used as the basis for the request ID. The request ID is used in - [logs](../administration/request_log.md#request-log-format) and tracing to - correlate and match up requests. When unset, Synapse will automatically - generate sequential request IDs. This option is useful when Synapse is behind - a [reverse-proxy](../../reverse_proxy.md). - - _Added in Synapse 1.68.0._ - -* `resources`: Only valid for an 'http' listener. A list of resources to host - on this port. Sub-options for each resource are: - - * `names`: a list of names of HTTP resources. See below for a list of valid resource names. - - * `compress`: set to true to enable gzip compression on HTTP bodies for this resource. This is currently only supported with the - `client`, `consent`, `metrics` and `federation` resources. - -* `additional_resources`: Only valid for an 'http' listener. A map of - additional endpoints which should be loaded via dynamic modules. - -Unix socket support (_Added in Synapse 1.89.0_): -* `path`: A path and filename for a Unix socket. Make sure it is located in a - directory with read and write permissions, and that it already exists (the directory - will not be created). Defaults to `None`. - * **Note**: The use of both `path` and `port` options for the same `listener` is not - compatible. - * The `x_forwarded` option defaults to true when using Unix sockets and can be omitted. - * Other options that would not make sense to use with a UNIX socket, such as - `bind_addresses` and `tls` will be ignored and can be removed. -* `mode`: The file permissions to set on the UNIX socket. Defaults to `666` -* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). - Also make sure that `metrics` is not included in `resources` -> `names` - +*(array)* List of ports that Synapse should listen on, their purpose and their configuration. Valid resource names are: -* `client`: the client-server API (/_matrix/client). Also implies `media` and `static`. - If configuring the main process, the Synapse Admin API (/_synapse/admin) is also implied. +* `client`: the client-server API (/_matrix/client). Also implies `media` and `static`. If configuring the main process, the Synapse Admin API (/_synapse/admin) is also implied. * `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. @@ -573,82 +480,108 @@ Valid resource names are: * `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). -* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for "fallback authentication".) -* `health`: the [health check endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint - is by default active for all other resources and does not have to be activated separately. - This is only useful if you want to use the health endpoint explicitly on a dedicated port or - for [workers](../../workers.md) and containers without listener e.g. - [application services](../../workers.md#notifying-application-services). +* `health`: the [health check endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint is by default active for all other resources and does not have to be activated separately. This is only useful if you want to use the health endpoint explicitly on a dedicated port or for [workers](../../workers.md) and containers without listener e.g. [application services](../../workers.md#notifying-application-services). -Example configuration #1: +Defaults to `[]`. + +Options for each entry include: + +* `port` (integer): The TCP port to bind to. + +* `tag` (string|null): An alias for the port in the logger name. If set the tag is logged instead of the port. Default to `None`, is optional and only valid for listener with `type: http`. See the docs [request log format](../administration/request_log.md). + +* `bind_addresses` (array|null): A list of local addresses to listen on. The default is "all local interfaces". + +* `type` (string): The type of listener. Normally `http`, but other valid options are [`manhole`](../../manhole.md) and [`metrics`](../../metrics-howto.md). + +* `tls` (boolean): Set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path/tls_certificate_path. + +* `x_forwarded` (boolean): Only valid for an `http` listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + +* `request_id_header` (string|null): The header extracted from each incoming request that is used as the basis for the request ID. The request ID is used in [logs](../administration/request_log.md#request-log-format) and tracing to correlate and match up requests. When unset, Synapse will automatically generate sequential request IDs. This option is useful when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + + _Added in Synapse 1.68.0._ + +* `resources` (array): Only valid for an `http` listener. A list of resources to host on this port. + + Options for each entry include: + + * `names` (array): A list of names of HTTP resources. See below for a list of valid resource names. + + * `compress` (boolean): Set to true to enable gzip compression on HTTP bodies for this resource. This is currently only supported with the `client`, `consent`, `metrics` and `federation` resources. + +* `additional_resources` (object): Only valid for an `http` listener. A map of additional endpoints which should be loaded via dynamic modules. + +* `path` (string): A path and filename for a Unix socket. Make sure it is located in a directory with read and write permissions, and that it already exists (the directory will not be created). Defaults to `None`. + * **Note**: The use of both `path` and `port` options for the same `listener` is not compatible. + * The `x_forwarded` option defaults to true when using Unix sockets and can be omitted. + * Other options that would not make sense to use with a UNIX socket, such as `bind_addresses` and `tls` will be ignored and can be removed. + + _Added in Synapse 1.89.0_: Unix socket support + +* `mode` (integer|null): The file permissions to set on the UNIX socket. Defaults to `666` if unset or null. + + **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). Also make sure that `metrics` is not included in `resources` -> `names` + + _Added in Synapse 1.89.0_: Unix socket support + +Example configurations: ```yaml listeners: - # TLS-enabled listener: for when matrix traffic is sent directly to synapse. - # - # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section - # below.) - # - - port: 8448 - type: http - tls: true - resources: - - names: [client, federation] +- port: 8448 + type: http + tls: true + resources: + - names: + - client + - federation ``` -Example configuration #2: + ```yaml listeners: - # Insecure HTTP listener: for when matrix traffic passes through a reverse proxy - # that unwraps TLS. - # - # If you plan to use a reverse proxy, please see - # https://element-hq.github.io/synapse/latest/reverse_proxy.html. - # - - port: 8008 - tls: false - type: http - x_forwarded: true - bind_addresses: ['::1', '127.0.0.1'] - - resources: - - names: [client, federation] - compress: false - - # example additional_resources: - additional_resources: - "/_matrix/my/custom/endpoint": - module: my_module.CustomRequestHandler - config: {} - - # Turn on the twisted ssh manhole service on localhost on the given - # port. - - port: 9000 - bind_addresses: ['::1', '127.0.0.1'] - type: manhole +- port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: + - ::1 + - 127.0.0.1 + resources: + - names: + - client + - federation + compress: false + additional_resources: + /_matrix/my/custom/endpoint: + module: my_module.CustomRequestHandler + config: {} +- port: 9000 + bind_addresses: + - ::1 + - 127.0.0.1 + type: manhole ``` -Example configuration #3: + ```yaml listeners: - # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering - # lightweight interprocess communication without TCP/IP overhead, avoid port - # conflicts, and providing enhanced security through system file permissions. - # - # Note that x_forwarded will default to true, when using a UNIX socket. Please see - # https://element-hq.github.io/synapse/latest/reverse_proxy.html. - # - - path: /run/synapse/main_public.sock - type: http - resources: - - names: [client, federation] +- path: /run/synapse/main_public.sock + type: http + resources: + - names: + - client + - federation ``` - --- ### `manhole` -Turn on the Twisted telnet manhole service on the given port. Defaults to none. +*(integer|null)* Turn on the Twisted telnet manhole service on the given port. This can also be set by the `--manhole` argument when starting Synapse. +Defaults to `null`. + Example configuration: ```yaml manhole: 1234 @@ -656,13 +589,17 @@ manhole: 1234 --- ### `manhole_settings` -Connection settings for the manhole. You can find more information -on the manhole [here](../../manhole.md). Manhole sub-options include: -* `username` : the username for the manhole. This defaults to 'matrix'. -* `password`: The password for the manhole. This defaults to 'rabbithole'. -* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. - If these are left unset, then hardcoded and non-secret keys are used, - which could allow traffic to be intercepted if sent over a public network. +*(object)* Connection settings for the manhole. You can find more information on the manhole [here](../../manhole.md). + +This setting has the following sub-options: + +* `username` (string|null): The username for the manhole. This defaults to "matrix". + +* `password` (string|null): The password for the manhole. This defaults to "rabbithole". + +* `ssh_priv_key_path` (string|null): The private SSH key used to encrypt the manhole traffic. If left unset, then hardcoded and non-secret keys are used, which could allow traffic to be intercepted if sent over a public network. + +* `ssh_pub_key_path` (string|null): The public SSH key corresponsing to `ssh_priv_key_path`. If left unset, a hardcoded key is used. Example configuration: ```yaml @@ -675,15 +612,11 @@ manhole_settings: --- ### `dummy_events_threshold` -Forward extremities can build up in a room due to networking delays between -homeservers. Once this happens in a large room, calculation of the state of -that room can become quite expensive. To mitigate this, once the number of -forward extremities reaches a given threshold, Synapse will send an -`org.matrix.dummy_event` event, which will reduce the forward extremities -in the room. +*(integer)* Forward extremities can build up in a room due to networking delays between homeservers. Once this happens in a large room, calculation of the state of that room can become quite expensive. To mitigate this, once the number of forward extremities reaches a given threshold, Synapse will send an `org.matrix.dummy_event` event, which will reduce the forward extremities in the room. This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. -The default value is 10. + +Defaults to `10`. Example configuration: ```yaml @@ -692,14 +625,13 @@ dummy_events_threshold: 5 --- ### `delete_stale_devices_after` -An optional duration. If set, Synapse will run a daily background task to log out and -delete any device that hasn't been accessed for more than the specified amount of time. +An optional duration. If set, Synapse will run a daily background task to log out and delete any device that hasn't been accessed for more than the specified amount of time. -Defaults to no duration, which means devices are never pruned. +A value of null means devices are never pruned. -**Note:** This task will always run on the main process, regardless of the value of -`run_background_tasks_on`. This is due to workers currently not having the ability to -delete devices. +**Note:** This task will always run on the main process, regardless of the value of `run_background_tasks_on`. This is due to workers currently not having the ability to delete devices. + +Defaults to `null`. Example configuration: ```yaml @@ -708,167 +640,168 @@ delete_stale_devices_after: 1y --- ### `email` -Configuration for sending emails from Synapse. +*(object)* Configuration for sending emails from Synapse. -Server admins can configure custom templates for email content. See -[here](../../templates.md) for more information. +Server admins can configure custom templates for email content. See [here](../../templates.md) for more information. This setting has the following sub-options: -* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. -* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 465 if `force_tls` is true, else 25. + +* `smtp_host` (string): The hostname of the outgoing SMTP server to use. Defaults to `"localhost"`. + +* `smtp_port` (string|null): The port on the mail server for outgoing SMTP. If null or unset, 465 is used if `force_tls` is true, else 25. _Changed in Synapse 1.64.0:_ the default port is now aware of `force_tls`. -* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no - authentication is attempted. -* `force_tls`: By default, Synapse connects over plain text and then optionally upgrades - to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS), - and the option `require_transport_security` is ignored. - It is recommended to enable this if supported by your mail server. + + Defaults to `null`. + +* `smtp_user` (string|null): Username for authentication to the SMTP server. Defaults to `null`. + +* `smtp_pass` (string|null): Password for authentication to the SMTP server. Defaults to `null`. + +* `force_tls` (boolean): By default, Synapse connects over plain text and then optionally upgrades to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS), and the option `require_transport_security` is ignored. It is recommended to enable this if supported by your mail server. _New in Synapse 1.64.0._ -* `require_transport_security`: Set to true to require TLS transport security for SMTP. - By default, Synapse will connect over plain text, and will then switch to - TLS via STARTTLS *if the SMTP server supports it*. If this option is set, - Synapse will refuse to connect unless the server supports STARTTLS. -* `enable_tls`: By default, if the server supports TLS, it will be used, and the server - must present a certificate that is valid for `tlsname`. If this option - is set to false, TLS will not be used. -* `tlsname`: The domain name the SMTP server's TLS certificate must be valid for, defaulting to `smtp_host`. -* `notif_from`: defines the "From" address to use when sending emails. - It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, - which is normally set in `app_name`, but may be overridden by the - Matrix client application. Note that the placeholder must be written '%(app)s', including the - trailing 's'. -* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email - subjects. It defaults to 'Matrix'. -* `enable_notifs`: Set to true to allow users to receive e-mail notifications. If this is not set, - users can configure e-mail notifications but will not receive them. Disabled by default. -* `notif_for_new_users`: Set to false to disable automatic subscription to email - notifications for new users. Enabled by default. -* `notif_delay_before_mail`: The time to wait before emailing about a notification. - This gives the user a chance to view the message via push or an open client. - Defaults to 10 minutes. + + Defaults to `false`. + +* `require_transport_security` (boolean): Set to true to require TLS transport security for SMTP. By default, Synapse will connect over plain text, and will then switch to TLS via STARTTLS *if the SMTP server supports it*. If this option is set, Synapse will refuse to connect unless the server supports STARTTLS. Defaults to `false`. + +* `enable_tls` (boolean): By default, if the server supports TLS, it will be used, and the server must present a certificate that is valid for `tlsname`. If this option is set to false, TLS will not be used. Defaults to `true`. + +* `tlsname` (string): The domain name the SMTP server's TLS certificate must be valid for, defaulting to `smtp_host`. + +* `notif_from` (string|null): Defines the "From" address to use when sending emails. It must be set if email sending is enabled. The placeholder `%(app)s` will be replaced by the application name, which is normally set in `app_name`, but may be overridden by the Matrix client application. Note that the placeholder must be written `%(app)s`, including the trailing 's'. Defaults to `null`. + +* `app_name` (string): Defines the default value for `%(app)s` in `notif_from` and email subjects. Defaults to `"Matrix"`. + +* `enable_notifs` (boolean): Set to true to allow users to receive e-mail notifications. If this is not set, users can configure e-mail notifications but will not receive them. Defaults to `false`. + +* `notif_for_new_users` (boolean): Set to false to disable automatic subscription to email notifications for new users. Defaults to `true`. + +* `notif_delay_before_mail` (duration): The time to wait before emailing about a notification. This gives the user a chance to view the message via push or an open client. _New in Synapse 1.99.0._ -* `client_base_url`: Custom URL for client links within the email notifications. By default - links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; - the old name is still supported for backwards-compatibility but is now deprecated.) -* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. - Defaults to 1h. -* `invite_client_location`: The web client location to direct users to during an invite. This is passed - to the identity server as the `org.matrix.web_client_location` key. Defaults - to unset, giving no guidance to the identity server. -* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will - be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. - In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname - of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the - message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will - can use the '%(server_name)s' placeholder, which will be replaced by the value of the - `server_name` setting in your Synapse configuration. - Here is a list of subjects for notification emails that can be set: - * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a - room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." - * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a - room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." - * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in - a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." - * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a - name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." - * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. - Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." - * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in - multiple rooms. This is similar to the setting above except it's used when - the room in which the notification was triggered has no name. Defaults to - "[%(app)s] You have messages on %(app)s from %(person)s and others..." - * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. - Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." - * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a - name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." - * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" - * `email_validation`: Subject to use when sending a verification email to assert an address's - ownership. Defaults to "[%(server_name)s] Validate your email" + Defaults to `"10m"`. + +* `client_base_url` (string): Custom URL for client links within the email notifications. (This setting used to be called `riot_base_url`; the old name is still supported for backwards-compatibility but is now deprecated.) Defaults to `"https://matrix.to"`. + +* `validation_token_lifetime` (duration): Configures the time that a validation email will expire after sending. Defaults to `"1h"`. + +* `invite_client_location` (string|null): The web client location to direct users to during an invite. This is passed to the identity server as the `org.matrix.web_client_location` key. If null or unset no guidance is given to the identity server. Defaults to `null`. + +* `subjects` (object): Subjects to use when sending emails from Synapse. The placeholder `%(app)s` will be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. In addition, each subject can use the following placeholders: `%(person)s`, which will be replaced by the displayname of the user(s) that sent the message(s), e.g. "Alice and Bob", and `%(room)s`, which will be replaced by the name of the room the message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will can use the `%(server_name)s` placeholder, which will be replaced by the value of the `server_name` setting in your Synapse configuration. + + This setting has the following sub-options: + + * `message_from_person_in_room` (string): Subject to use to notify about one message from one or more user(s) in a room which has a name. Defaults to `"[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."`. + + * `message_from_person` (string): Subject to use to notify about one message from one or more user(s) in a room which doesn't have a name. Defaults to `"[%(app)s] You have a message on %(app)s from %(person)s..."`. + + * `messages_from_person` (string): Subject to use to notify about multiple messages from one or more users in a room which doesn't have a name. Defaults to `"[%(app)s] You have messages on %(app)s from %(person)s..."`. + + * `messages_in_room` (string): Subject to use to notify about multiple messages in a room which has a name. Defaults to `"[%(app)s] You have messages on %(app)s in the %(room)s room..."`. + + * `messages_in_room_and_others` (string): Subject to use to notify about multiple messages in multiple rooms. Defaults to `"[%(app)s] You have messages on %(app)s in the %(room)s room and others..."`. + + * `messages_from_person_and_others` (string): Subject to use to notify about multiple messages from multiple persons in multiple rooms. This is similar to the setting above except it's used when the room in which the notification was triggered has no name. Defaults to `"[%(app)s] You have messages on %(app)s from %(person)s and others..."`. + + * `invite_from_person_to_room` (string): Subject to use to notify about an invite to a room which has a name. Defaults to `"[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."`. + + * `invite_from_person` (string): Subject to use to notify about an invite to a room which doesn't have a name. Defaults to `"[%(app)s] %(person)s has invited you to chat on %(app)s..."`. + + * `password_reset` (string): Subject to use when sending a password reset email. Defaults to `"[%(server_name)s] Password reset"`. + + * `email_validation` (string): Subject to use when sending a verification email to assert an address's ownership. Defaults to `"[%(server_name)s] Validate your email"`. Example configuration: - ```yaml email: smtp_host: mail.server smtp_port: 587 - smtp_user: "exampleusername" - smtp_pass: "examplepassword" + smtp_user: exampleusername + smtp_pass: examplepassword force_tls: true require_transport_security: true enable_tls: false tlsname: mail.server.example.com - notif_from: "Your Friendly %(app)s homeserver " + notif_from: Your Friendly %(app)s homeserver app_name: my_branded_matrix_server enable_notifs: true notif_for_new_users: false - client_base_url: "http://localhost/riot" + client_base_url: http://localhost/riot validation_token_lifetime: 15m invite_client_location: https://app.element.io - subjects: - message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." - message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." - messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." - messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." - messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." - messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." - invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." - invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." - password_reset: "[%(server_name)s] Password reset" - email_validation: "[%(server_name)s] Validate your email" + message_from_person_in_room: '[%(app)s] You have a message on %(app)s from %(person)s + in the %(room)s room...' + message_from_person: '[%(app)s] You have a message on %(app)s from %(person)s...' + messages_from_person: '[%(app)s] You have messages on %(app)s from %(person)s...' + messages_in_room: '[%(app)s] You have messages on %(app)s in the %(room)s room...' + messages_in_room_and_others: '[%(app)s] You have messages on %(app)s in the %(room)s + room and others...' + messages_from_person_and_others: '[%(app)s] You have messages on %(app)s from + %(person)s and others...' + invite_from_person_to_room: '[%(app)s] %(person)s has invited you to join the + %(room)s room on %(app)s...' + invite_from_person: '[%(app)s] %(person)s has invited you to chat on %(app)s...' + password_reset: '[%(server_name)s] Password reset' + email_validation: '[%(server_name)s] Validate your email' ``` --- ### `max_event_delay_duration` -The maximum allowed duration by which sent events can be delayed, as per -[MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). -Must be a positive value if set. +The maximum allowed duration by which sent events can be delayed, as per [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). Must be a positive value if set. -Defaults to no duration (`null`), which disallows sending delayed events. +If null or unset, sending of delayed events is disallowed. + +Defaults to `null`. Example configuration: ```yaml max_event_delay_duration: 24h ``` - +--- ## Homeserver blocking + Useful options for Synapse admins. --- - ### `admin_contact` -How to reach the server admin, used in `ResourceLimitError`. Defaults to none. +*(string|null)* How to reach the server admin, used in `ResourceLimitError`. Defaults to `null`. Example configuration: ```yaml -admin_contact: 'mailto:admin@server.com' +admin_contact: mailto:admin@server.com ``` --- -### `hs_disabled` and `hs_disabled_message` +### `hs_disabled` -Blocks users from connecting to the homeserver and provides a human-readable reason -why the connection was blocked. Defaults to false. +*(boolean)* Blocks users from connecting to the homeserver and provides the human-readable reason given in `hs_disabled_message`. Defaults to `false`. Example configuration: ```yaml hs_disabled: true -hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +### `hs_disabled_message` + +*(string)* Human-readable reason why the connection was blocked. Defaults to `"Homeserver is currently blocked"`. + +Example configuration: +```yaml +hs_disabled_message: Reason for why the HS is blocked ``` --- ### `limit_usage_by_mau` -This option disables/enables monthly active user blocking. Used in cases where the admin or -server owner wants to limit to the number of monthly active users. When enabled and a limit is -reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. -Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. +*(boolean)* This option disables/enables monthly active user blocking. Used in cases where the admin or server owner wants to limit to the number of monthly active users. When enabled and a limit is reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. If this is enabled, a value for `max_mau_value` must also be set. See [Monthly Active Users](../administration/monthly_active_users.md) for details on how to configure MAU. +Defaults to `false`. + Example configuration: ```yaml limit_usage_by_mau: true @@ -876,8 +809,7 @@ limit_usage_by_mau: true --- ### `max_mau_value` -This option sets the hard limit of monthly active users above which the server will start -blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. +*(integer)* This option sets the hard limit of monthly active users above which the server will start blocking user actions if `limit_usage_by_mau` is enabled. Defaults to `0`. Example configuration: ```yaml @@ -886,11 +818,7 @@ max_mau_value: 50 --- ### `mau_trial_days` -The option `mau_trial_days` is a means to add a grace period for active users. It -means that users must be active for the specified number of days before they -can be considered active and guards against the case where lots of users -sign up in a short space of time never to return after their initial -session. Defaults to 0. +*(integer)* The option `mau_trial_days` is a means to add a grace period for active users. It means that users must be active for the specified number of days before they can be considered active and guards against the case where lots of users sign up in a short space of time never to return after their initial session. Defaults to `0`. Example configuration: ```yaml @@ -899,10 +827,7 @@ mau_trial_days: 5 --- ### `mau_appservice_trial_days` -The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different -trial number if the user was registered by an appservice. A value -of 0 means no trial days are applied. Appservices not listed in this dictionary -use the value of `mau_trial_days` instead. +*(object)* The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different trial number if the user was registered by an appservice. A value of 0 means no trial days are applied. Appservices not listed in this dictionary use the value of `mau_trial_days` instead. Defaults to `{}`. Example configuration: ```yaml @@ -913,11 +838,7 @@ mau_appservice_trial_days: --- ### `mau_limit_alerting` -The option `mau_limit_alerting` is a means of limiting client-side alerting -should the mau limit be reached. This is useful for small instances -where the admin has 5 mau seats (say) for 5 specific people and no -interest increasing the mau limit further. Defaults to true, which -means that alerting is enabled. +*(boolean)* Limit client-side alerting should the mau limit be reached. This is useful for small instances where the admin has 5 mau seats (say) for 5 specific people and no interest increasing the mau limit further. Defaults to `true`. Example configuration: ```yaml @@ -926,9 +847,7 @@ mau_limit_alerting: false --- ### `mau_stats_only` -If enabled, the metrics for the number of monthly active users will -be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` -is true, this is implied to be true. Defaults to false. +*(boolean)* If enabled, the metrics for the number of monthly active users will be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` is true, this is implied to be true. Defaults to `false`. Example configuration: ```yaml @@ -937,22 +856,24 @@ mau_stats_only: true --- ### `mau_limit_reserved_threepids` -Sometimes the server admin will want to ensure certain accounts are -never blocked by mau checking. These accounts are specified by this option. -Defaults to none. Add accounts by specifying the `medium` and `address` of the -reserved threepid (3rd party identifier). +*(array)* Sometimes the server admin will want to ensure certain accounts are never blocked by mau checking. These accounts are specified by this option. Add accounts by specifying the `medium` and `address` of the reserved threepid (3rd party identifier). Defaults to `[]`. + +Options for each entry include: + +* `medium` (string): Medium of the account threepid. + +* `address` (string): Address of the account threepid. Example configuration: ```yaml mau_limit_reserved_threepids: - - medium: 'email' - address: 'reserved_user@example.com' +- medium: email + address: reserved_user@example.com ``` --- ### `server_context` -This option is used by phonehome stats to group together related servers. -Defaults to none. +*(string|null)* This option is used by phonehome stats to group together related servers. Defaults to `null`. Example configuration: ```yaml @@ -961,32 +882,30 @@ server_context: context --- ### `limit_remote_rooms` -When this option is enabled, the room "complexity" will be checked before a user -joins a new remote room. If it is above the complexity limit, the server will -disallow joining, or will instantly leave. This is useful for homeservers that are -resource-constrained. Options for this setting include: -* `enabled`: whether this check is enabled. Defaults to false. -* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. -* `complexity_error`: override the error which is returned when the room is too complex with a - custom message. -* `admins_can_join`: allow server admins to join complex rooms. Default is false. +*(object)* When this option is enabled, the room "complexity" will be checked before a user joins a new remote room. If it is above the complexity limit, the server will disallow joining, or will instantly leave. This is useful for homeservers that are resource-constrained. Room complexity is an arbitrary measure based on factors such as the number of users in the room. -Room complexity is an arbitrary measure based on factors such as the number of -users in the room. +This setting has the following sub-options: + +* `enabled` (boolean): Whether this check is enabled. Defaults to `false`. + +* `complexity` (number): The limit above which rooms cannot be joined. Defaults to `1.0`. + +* `complexity_error` (string): Override the error which is returned when the room is too complex with a custom message. Defaults to `"Your homeserver is unable to join rooms this large or complex. Please speak to your server administrator, or upgrade your instance to join this room."`. + +* `admins_can_join` (boolean): Allow server admins to join complex rooms. Defaults to `false`. Example configuration: ```yaml limit_remote_rooms: enabled: true complexity: 0.5 - complexity_error: "I can't let you do that, Dave." + complexity_error: I can't let you do that, Dave. admins_can_join: true ``` --- ### `require_membership_for_aliases` -Whether to require a user to be in the room to add an alias to it. -Defaults to true. +*(boolean)* Whether to require a user to be in the room to add an alias to it. Defaults to `true`. Example configuration: ```yaml @@ -995,9 +914,7 @@ require_membership_for_aliases: false --- ### `allow_per_room_profiles` -Whether to allow per-room membership profiles through the sending of membership -events with profile information that differs from the target's global profile. -Defaults to true. +*(boolean)* Whether to allow per-room membership profiles through the sending of membership events with profile information that differs from the target's global profile. Defaults to `true`. Example configuration: ```yaml @@ -1006,11 +923,12 @@ allow_per_room_profiles: false --- ### `max_avatar_size` -The largest permissible file size in bytes for a user avatar. Defaults to no restriction. -Use M for MB and K for KB. +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. Use M for MB and K for KB. Note that user avatar changes will not work if this is set without using Synapse's media repository. +Defaults to `null`. + Example configuration: ```yaml max_avatar_size: 10M @@ -1018,26 +936,27 @@ max_avatar_size: 10M --- ### `allowed_avatar_mimetypes` -The MIME types allowed for user avatars. Defaults to no restriction. +*(array|null)* The MIME types allowed for user avatars. Defaults to no restriction. -Note that user avatar changes will not work if this is set without -using Synapse's media repository. +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Defaults to `null`. Example configuration: ```yaml -allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +allowed_avatar_mimetypes: +- image/png +- image/jpeg +- image/gif ``` --- ### `redaction_retention_period` -How long to keep redacted events in unredacted form in the database. After -this period redacted events get replaced with their redacted form in the DB. +How long to keep redacted events in unredacted form in the database. After this period redacted events get replaced with their redacted form in the DB. -Synapse will check whether the rentention period has concluded for redacted -events every 5 minutes. Thus, even if this option is set to `0`, Synapse may -still take up to 5 minutes to purge redacted events from the database. +Synapse will check whether the rentention period has concluded for redacted events every 5 minutes. Thus, even if this option is set to `0`, Synapse may still take up to 5 minutes to purge redacted events from the database. Set to `null` to disable. -Defaults to `7d`. Set to `null` to disable. +Defaults to `"7d"`. Example configuration: ```yaml @@ -1046,9 +965,7 @@ redaction_retention_period: 28d --- ### `forgotten_room_retention_period` -How long to keep locally forgotten rooms before purging them from the DB. - -Defaults to `null`, meaning it's disabled. +How long to keep locally forgotten rooms before purging them from the DB. A value of `null` means it's disabled. Defaults to `null`. Example configuration: ```yaml @@ -1057,9 +974,7 @@ forgotten_room_retention_period: 28d --- ### `user_ips_max_age` -How long to track users' last seen time and IPs in the database. - -Defaults to `28d`. Set to `null` to disable clearing out of old rows. +How long to track users' last seen time and IPs in the database. Set to `null` to disable clearing out of old rows. Defaults to `"28d"`. Example configuration: ```yaml @@ -1068,13 +983,7 @@ user_ips_max_age: 14d --- ### `request_token_inhibit_3pid_errors` -Inhibits the `/requestToken` endpoints from returning an error that might leak -information about whether an e-mail address is in use or not on this -homeserver. Defaults to false. -Note that for some endpoints the error situation is the e-mail already being -used, and for others the error is entering the e-mail being unused. -If this option is enabled, instead of returning an error, these endpoints will -act as if no error happened and return a fake session ID ('sid') to clients. +*(boolean)* Inhibits the `/requestToken` endpoints from returning an error that might leak information about whether an e-mail address is in use or not on this homeserver. Note that for some endpoints the error situation is the e-mail already being used, and for others the error is entering the e-mail being unused. If this option is enabled, instead of returning an error, these endpoints will act as if no error happened and return a fake session ID (`sid`) to clients. Defaults to `false`. Example configuration: ```yaml @@ -1083,36 +992,30 @@ request_token_inhibit_3pid_errors: true --- ### `next_link_domain_whitelist` -A list of domains that the domain portion of `next_link` parameters -must match. +*(array|null)* A list of domains that the domain portion of `next_link` parameters must match. -This parameter is optionally provided by clients while requesting -validation of an email or phone number, and maps to a link that -users will be automatically redirected to after validation -succeeds. Clients can make use this parameter to aid the validation -process. +This parameter is optionally provided by clients while requesting validation of an email or phone number, and maps to a link that users will be automatically redirected to after validation succeeds. Clients can make use this parameter to aid the validation process. The whitelist is applied whether the homeserver or an identity server is handling validation. -The default value is no whitelist functionality; all domains are -allowed. Setting this value to an empty list will instead disallow -all domains. +The default value is no whitelist functionality; all domains are allowed. Setting this value to an empty list will instead disallow all domains. + +Defaults to `null`. Example configuration: ```yaml -next_link_domain_whitelist: ["matrix.org"] +next_link_domain_whitelist: matrix.org ``` --- -### `templates` and `custom_template_directory` +### `templates` -These options define templates to use when generating email or HTML page contents. -The `custom_template_directory` determines which directory Synapse will try to -find template files in to use to generate email or HTML page contents. -If not set, or a file is not found within the template directory, a default -template from within the Synapse package will be used. +*(object)* These options define templates to use when generating email or HTML page contents. -See [here](../../templates.md) for more -information about using custom templates. +See [here](../../templates.md) for more information about using custom templates. + +This setting has the following sub-options: + +* `custom_template_directory` (string|null): Determines which directory Synapse will try to find template files in to use to generate email or HTML page contents. If not set, or a file is not found within the template directory, a default template from within the Synapse package will be used. Defaults to `null`. Example configuration: ```yaml @@ -1122,62 +1025,49 @@ templates: --- ### `retention` -This option and the associated options determine message retention policy at the -server level. +*(object)* This option and the associated options determine message retention policy at the server level. -Room admins and mods can define a retention period for their rooms using the -`m.room.retention` state event, and server admins can cap this period by setting -the `allowed_lifetime_min` and `allowed_lifetime_max` config options. +Room admins and mods can define a retention period for their rooms using the `m.room.retention` state event, and server admins can cap this period by setting the `allowed_lifetime_min` and `allowed_lifetime_max` config options. -If this feature is enabled, Synapse will regularly look for and purge events -which are older than the room's maximum retention period. Synapse will also -filter events received over federation so that events that should have been -purged are ignored and not stored again. +If this feature is enabled, Synapse will regularly look for and purge events which are older than the room's maximum retention period. Synapse will also filter events received over federation so that events that should have been purged are ignored and not stored again. -The message retention policies feature is disabled by default. You can read more -about this feature [here](../../message_retention_policies.md). +The message retention policies feature is disabled by default. You can read more about this feature [here](../../message_retention_policies.md). This setting has the following sub-options: -* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the - 'm.room.retention' state event. This option is further specified by the - `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the - value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. -* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If - set, and the state of a room contains a `m.room.retention` event in its state - which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, - Synapse will cap the room's policy to these limits when running purge jobs. +* `enabled` (boolean): Enforce message retention policies Defaults to `false`. -* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: - Server admins can define the settings of the background jobs purging the - events whose lifetime has expired under the `purge_jobs` section. +* `default_policy` (object): Default message retention policy. If set, Synapse will apply it to rooms that lack the `m.room.retention` state event. - If no configuration is provided for this option, a single job will be set up to delete - expired events in every room daily. + This setting has the following sub-options: - Each job's configuration defines which range of message lifetimes the job - takes care of. For example, if `shortest_max_lifetime` is '2d' and - `longest_max_lifetime` is '3d', the job will handle purging expired events in - rooms whose state defines a `max_lifetime` that's both higher than 2 days, and - lower than or equal to 3 days. Both the minimum and the maximum value of a - range are optional, e.g. a job with no `shortest_max_lifetime` and a - `longest_max_lifetime` of '3d' will handle every room with a retention policy - whose `max_lifetime` is lower than or equal to three days. + * `min_lifetime`: Minimum message retention time of the default message retention policy. Synapse doesn't take this option into account yet. Defaults to `null`. - The rationale for this per-job configuration is that some rooms might have a - retention policy with a low `max_lifetime`, where history needs to be purged - of outdated messages on a more frequent basis than for the rest of the rooms - (e.g. every 12h), but not want that purge to be performed by a job that's - iterating over every room it knows, which could be heavy on the server. + * `max_lifetime`: Maximum message retention time of the default message retention policy. Defaults to `null`. - If any purge job is configured, it is strongly recommended to have at least - a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` - set, or one job without `shortest_max_lifetime` and one job without - `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if - `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a - room's policy to these values is done after the policies are retrieved from - Synapse's database (which is done using the range specified in a purge job's - configuration). +* `allowed_lifetime_min`: Retention policy limit. If set, and the state of a room contains a `m.room.retention` event in its state which contains a `min_lifetime` that's beyond this bound, Synapse will cap the room's policy to these limits when running purge jobs. Defaults to `null`. + +* `allowed_lifetime_max`: Retention policy limit. If set, and the state of a room contains a `m.room.retention` event in its state which contains a `max_lifetime` that's beyond this bound, Synapse will cap the room's policy to these limits when running purge jobs. Defaults to `null`. + +* `purge_jobs` (array|null): Server admins can define the settings of the background jobs purging the events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job takes care of. For example, if `shortest_max_lifetime` is "2d" and `longest_max_lifetime` is "3d", the job will handle purging expired events in rooms whose state defines a `max_lifetime` that's both higher than 2 days, and lower than or equal to 3 days. Both the minimum and the maximum value of a range are optional, e.g. a job with no `shortest_max_lifetime` and a `longest_max_lifetime` of "3d" will handle every room with a retention policy whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a retention policy with a low `max_lifetime`, where history needs to be purged of outdated messages on a more frequent basis than for the rest of the rooms (e.g. every 12h), but not want that purge to be performed by a job that's iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` set, or one job without `shortest_max_lifetime` and one job without `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a room's policy to these values is done after the policies are retrieved from Synapse's database (which is done using the range specified in a purge job's configuration). + + Defaults to `null`. + + Options for each entry include: + + * `shortest_max_lifetime`: Apply job to rooms that have a `max_lifetime` higher than `shortest_max_lifetime`. A value of `null` never excludes any room. + + * `longest_max_lifetime`: Apply job to rooms that have a `max_lifetime` lower than or equal to `shortest_max_lifetime`. A value of `null` never excludes any room. + + * `interval` (duration): How often to run the job. Example configuration: ```yaml @@ -1189,10 +1079,10 @@ retention: allowed_lifetime_min: 1d allowed_lifetime_max: 1y purge_jobs: - - longest_max_lifetime: 3d - interval: 12h - - shortest_max_lifetime: 3d - interval: 1d + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d ``` --- ## TLS @@ -1202,32 +1092,29 @@ Options related to TLS. --- ### `tls_certificate_path` -This option specifies a PEM-encoded X509 certificate for TLS. -This certificate, as of Synapse 1.0, will need to be a valid and verifiable -certificate, signed by a recognised Certificate Authority. Defaults to none. +*(string|null)* This option specifies a PEM-encoded X509 certificate for TLS. This certificate, as of Synapse 1.0, will need to be a valid and verifiable certificate, signed by a recognised Certificate Authority. -Be sure to use a `.pem` file that includes the full certificate chain including -any intermediate certificates (for instance, if using certbot, use -`fullchain.pem` as your certificate, not `cert.pem`). +Be sure to use a `.pem` file that includes the full certificate chain including any intermediate certificates (for instance, if using certbot, use `fullchain.pem` as your certificate, not `cert.pem`). + +Defaults to `null`. Example configuration: ```yaml -tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +tls_certificate_path: CONFDIR/SERVERNAME.tls.crt ``` --- ### `tls_private_key_path` -PEM-encoded private key for TLS. Defaults to none. +*(string|null)* PEM-encoded private key for TLS. Defaults to `null`. Example configuration: ```yaml -tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +tls_private_key_path: CONFDIR/SERVERNAME.tls.key ``` --- ### `federation_verify_certificates` -Whether to verify TLS server certificates for outbound federation requests. -Defaults to true. To disable certificate verification, set the option to false. +*(boolean)* Whether to verify TLS server certificates for outbound federation requests. To disable certificate verification, set the option to false. Defaults to `true`. Example configuration: ```yaml @@ -1236,53 +1123,51 @@ federation_verify_certificates: false --- ### `federation_client_minimum_tls_version` -The minimum TLS version that will be used for outbound federation requests. +*(string)* The minimum TLS version that will be used for outbound federation requests. -Defaults to `"1"`. Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note -that setting this value higher than `"1.2"` will prevent federation to most -of the public Matrix network: only configure it to `"1.3"` if you have an -entirely private federation setup and you can ensure TLS 1.3 support. +Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note that setting this value higher than `"1.2"` will prevent federation to most of the public Matrix network: only configure it to `"1.3"` if you have an entirely private federation setup and you can ensure TLS 1.3 support. + +Defaults to `"1"`. Example configuration: ```yaml -federation_client_minimum_tls_version: "1.2" +federation_client_minimum_tls_version: '1.2' ``` --- ### `federation_certificate_verification_whitelist` -Skip federation certificate verification on a given whitelist -of domains. +*(array)* Skip federation certificate verification on a given whitelist of domains. -This setting should only be used in very specific cases, such as -federation over Tor hidden services and similar. For private networks -of homeservers, you likely want to use a private CA instead. +This setting should only be used in very specific cases, such as federation over Tor hidden services and similar. For private networks of homeservers, you likely want to use a private CA instead. Only effective if `federation_verify_certificates` is `true`. +Defaults to `[]`. + Example configuration: ```yaml federation_certificate_verification_whitelist: - - lon.example.com - - "*.domain.com" - - "*.onion" +- lon.example.com +- '*.domain.com' +- '*.onion' ``` --- ### `federation_custom_ca_list` -List of custom certificate authorities for federation traffic. +*(array)* List of custom certificate authorities for federation traffic. -This setting should only normally be used within a private network of -homeservers. +This setting should only normally be used within a private network of homeservers. -Note that this list will replace those that are provided by your -operating environment. Certificates must be in PEM format. +Note that this list will replace those that are provided by your operating environment. Certificates must be in PEM format. + +Defaults to `[]`. Example configuration: ```yaml federation_custom_ca_list: - - myCA1.pem - - myCA2.pem - - myCA3.pem +- myCA1.pem +- myCA2.pem +- myCA3.pem ``` --- ## Federation @@ -1292,31 +1177,25 @@ Options related to federation. --- ### `federation_domain_whitelist` -Restrict federation to the given whitelist of domains. -N.B. we recommend also firewalling your federation listener to limit -inbound federation traffic as early as possible, rather than relying -purely on this application-layer restriction. If not specified, the -default is to whitelist everything. +*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. If not specified, the default is to whitelist everything. -Note: this does not stop a server from joining rooms that servers not on the -whitelist are in. As such, this option is really only useful to establish a -"private federation", where a group of servers all whitelist each other and have -the same whitelist. +Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. + +Defaults to `[]`. Example configuration: ```yaml federation_domain_whitelist: - - lon.example.com - - nyc.example.com - - syd.example.com +- lon.example.com +- nyc.example.com +- syd.example.com ``` --- ### `federation_whitelist_endpoint_enabled` -Enables an endpoint for fetching the federation whitelist config. +*(boolean)* Enables an endpoint for fetching the federation whitelist config. -The request method and path is `GET /_synapse/client/v1/config/federation_whitelist`, and the -response format is: +The request method and path is `GET /_synapse/client/v1/config/federation_whitelist`, and the response format is: ```json { @@ -1331,6 +1210,8 @@ If `whitelist_enabled` is `false` then the server is permitted to federate with The endpoint requires authentication. +Defaults to `false`. + Example configuration: ```yaml federation_whitelist_endpoint_enabled: true @@ -1338,25 +1219,18 @@ federation_whitelist_endpoint_enabled: true --- ### `federation_metrics_domains` -Report prometheus metrics on the age of PDUs being sent to and received from -the given domains. This can be used to give an idea of "delay" on inbound -and outbound federation, though be aware that any delay can be due to problems -at either end or with the intermediate network. - -By default, no domains are monitored in this way. +*(array)* Report prometheus metrics on the age of PDUs being sent to and received from the given domains. This can be used to give an idea of "delay" on inbound and outbound federation, though be aware that any delay can be due to problems at either end or with the intermediate network. Defaults to `[]`. Example configuration: ```yaml federation_metrics_domains: - - matrix.org - - example.com +- matrix.org +- example.com ``` --- ### `allow_profile_lookup_over_federation` -Set to false to disable profile lookup over federation. By default, the -Federation API allows other homeservers to obtain profile data of any user -on this homeserver. +*(boolean)* Set to false to disable profile lookup over federation. By default, the Federation API allows other homeservers to obtain profile data of any user on this homeserver. Defaults to `true`. Example configuration: ```yaml @@ -1365,9 +1239,7 @@ allow_profile_lookup_over_federation: false --- ### `allow_device_name_lookup_over_federation` -Set this option to true to allow device display name lookup over federation. By default, the -Federation API prevents other homeservers from obtaining the display names of any user devices -on this homeserver. +*(boolean)* Set this option to true to allow device display name lookup over federation. By default, the Federation API prevents other homeservers from obtaining the display names of any user devices on this homeserver. Defaults to `false`. Example configuration: ```yaml @@ -1376,27 +1248,29 @@ allow_device_name_lookup_over_federation: true --- ### `federation` -The federation section defines some sub-options related to federation. +*(object)* The federation section defines some sub-options related to federation. -The following options are related to configuring timeout and retry logic for one request, -independently of the others. -Short retry algorithm is used when something or someone will wait for the request to have an -answer, while long retry is used for requests that happen in the background, -like sending a federation transaction. +The following options are related to configuring timeout and retry logic for one request, independently of the others. Short retry algorithm is used when something or someone will wait for the request to have an answer, while long retry is used for requests that happen in the background, like sending a federation transaction. -* `client_timeout`: timeout for the federation requests. Default to 60s. -* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s. -* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s. -* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts. -* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts. +`destination_*` options control the retry logic when communicating with a specific homeserver destination. Unlike the previous configuration options, these values apply across all requests for a given destination and the state of the backoff is stored in the database. -The following options control the retry logic when communicating with a specific homeserver destination. -Unlike the previous configuration options, these values apply across all requests -for a given destination and the state of the backoff is stored in the database. +This setting has the following sub-options: -* `destination_min_retry_interval`: the initial backoff, after the first request fails. Defaults to 10m. -* `destination_retry_multiplier`: how much we multiply the backoff by after each subsequent fail. Defaults to 2. -* `destination_max_retry_interval`: a cap on the backoff. Defaults to a week. +* `client_timeout` (duration): Timeout for the federation requests. Defaults to `"60s"`. + +* `max_short_retry_delay` (duration): Maximum delay to be used for the short retry algo. Defaults to `"2s"`. + +* `max_long_retry_delay` (duration): Maximum delay to be used for the long retry algo. Defaults to `"60s"`. + +* `max_short_retries` (integer): Maximum number of retries for the short retry algo. Defaults to `3`. + +* `max_long_retries` (integer): Maximum number of retries for the long retry algo. Defaults to `10`. + +* `destination_min_retry_interval` (duration): The initial backoff, after the first request fails. Defaults to `"10m"`. + +* `destination_retry_multiplier` (integer): How much we multiply the backoff by after each subsequent fail. Defaults to `2`. + +* `destination_max_retry_interval` (duration): A cap on the backoff. Defaults to `"1w"`. Example configuration: ```yaml @@ -1418,93 +1292,69 @@ Options related to caching. --- ### `event_cache_size` -The number of events to cache in memory. Defaults to 10K. Like other caches, -this is affected by `caches.global_factor` (see below). +*(size)* The number of events to cache in memory. Defaults to 10K. Like other caches, this is affected by `caches.global_factor` (see below). For example, the default is 10K and the global_factor default is 0.5. Since 10K * 0.5 is 5K then the event cache size will be 5K. -The cache affected by this configuration is named as "*getEvent*". +The cache affected by this configuration is named as "\*getEvent\*". Note that this option is not part of the `caches` section. +Defaults to `"10K"`. + Example configuration: ```yaml event_cache_size: 15K ``` --- -### `caches` and associated values +### `caches` -A cache 'factor' is a multiplier that can be applied to each of -Synapse's caches in order to increase or decrease the maximum -number of entries that can be stored. +*(object)* A cache "factor" is a multiplier that can be applied to each of Synapse's caches in order to increase or decrease the maximum number of entries that can be stored. -`caches` can be configured through the following sub-options: +This setting has the following sub-options: -* `global_factor`: Controls the global cache factor, which is the default cache factor - for all caches if a specific factor for that cache is not otherwise - set. +* `global_factor` (number): Controls the global cache factor, which is the default cache factor for all caches if a specific factor for that cache is not otherwise set. - This can also be set by the `SYNAPSE_CACHE_FACTOR` environment - variable. Setting by environment variable takes priority over - setting through the config file. + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment variable. Setting by environment variable takes priority over setting through the config file. Defaults to 0.5, which will halve the size of all caches. Note that changing this value also affects the HTTP connection pool. -* `per_cache_factors`: A dictionary of cache name to cache factor for that individual - cache. Overrides the global cache factor for a given cache. + Defaults to `0.5`. - These can also be set through environment variables comprised - of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital - letters and underscores. Setting by environment variable - takes priority over setting through the config file. - Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` +* `per_cache_factors` (object): A dictionary of cache name to cache factor for that individual cache. Overrides the global cache factor for a given cache. - Some caches have '*' and other characters that are not - alphanumeric or underscores. These caches can be named with or - without the special characters stripped. For example, to specify - the cache factor for `*stateGroupCache*` via an environment - variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + These can also be set through environment variables comprised of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital letters and underscores. Setting by environment variable takes priority over setting through the config file. Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` -* `expire_caches`: Controls whether cache entries are evicted after a specified time - period. Defaults to true. Set to false to disable this feature. Note that never expiring - caches may result in excessive memory usage. + Some caches have '*' and other characters that are not alphanumeric or underscores. These caches can be named with or without the special characters stripped. For example, to specify the cache factor for `*stateGroupCache*` via an environment variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. -* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can - be in a cache without having been accessed before being evicted. - Defaults to 30m. + Defaults to `{}`. -* `sync_response_cache_duration`: Controls how long the results of a /sync request are - cached for after a successful response is returned. A higher duration can help clients - with intermittent connections, at the cost of higher memory usage. - A value of zero means that sync responses are not cached. - Defaults to 2m. +* `expire_caches` (boolean): Controls whether cache entries are evicted after a specified time period. Set to false to disable this feature. Note that never expiring caches may result in excessive memory usage. Defaults to `true`. + +* `cache_entry_ttl` (duration): If `expire_caches` is enabled, this flag controls how long an entry can be in a cache without having been accessed before being evicted. Defaults to `"30m"`. + +* `sync_response_cache_duration` (duration): Controls how long the results of a /sync request are cached for after a successful response is returned. A higher duration can help clients with intermittent connections, at the cost of higher memory usage. A value of zero means that sync responses are not cached. *Changed in Synapse 1.62.0*: The default was changed from 0 to 2m. -* `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and - `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory - usage and cache entry availability. You must be using [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) - to utilize this option, and all three of the options must be specified for this feature to work. This option - defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work - and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. - Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry - durations. - * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. - They will continue to be evicted until the memory usage drops below the `target_cache_memory_usage`, set in - the setting below, or until the `min_cache_ttl` is hit. There is no default value for this option. - * `target_cache_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value - for this option. - * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when - caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches - from being emptied while Synapse is evicting due to memory. There is no default value for this option. + Defaults to `"2m"`. + +* `cache_autotuning` (object): `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory usage and cache entry availability. You must be using [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) to utilize this option, and all three of the options must be specified for this feature to work. This option defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry durations. + + This setting has the following sub-options: + + * `max_cache_memory_usage`: Sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. They will continue to be evicted until the memory usage drops below the `target_cache_memory_usage`, set in the setting below, or until the `min_cache_ttl` is hit. Defaults to `null`. + + * `target_cache_memory_usage`: Sets a rough target for the desired memory usage of the caches. Defaults to `null`. + + * `min_cache_ttl`: Sets a limit under which newer cache entries are not evicted and is only applied when caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches from being emptied while Synapse is evicting due to memory. Defaults to `null`. Example configuration: ```yaml -event_cache_size: 15K caches: global_factor: 1.0 per_cache_factors: @@ -1518,54 +1368,42 @@ caches: ### Reloading cache factors -The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a -[`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. +The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a [`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. ```commandline kill -HUP [PID_OF_SYNAPSE_PROCESS] ``` -If you are running multiple workers, you must individually update the worker -config file and send this signal to each worker process. +If you are running multiple workers, you must individually update the worker config file and send this signal to each worker process. -If you're using the [example systemd service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) -file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using -`systemctl reload matrix-synapse`. +If you're using the [example systemd service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using `systemctl reload matrix-synapse`. --- ## Database + Config options related to database settings. --- ### `database` -The `database` setting defines the database that synapse uses to store all of -its data. +*(object)* The `database` setting defines the database that synapse uses to store all of its data. -Associated sub-options: +For more information on using Synapse with Postgres, see [here](../../postgres.md). -* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) - or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. +This setting has the following sub-options: -* `txn_limit` gives the maximum number of transactions to run per connection - before reconnecting. Defaults to 0, which means no limit. +* `name` (string): This option specifies the database engine to use: either `sqlite3` (for SQLite) or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. Defaults to `"sqlite3"`. -* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to - start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) - by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information - [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). +* `txn_limit` (integer): Gives the maximum number of transactions to run per connection before reconnecting. 0 means no limit. Defaults to `0`. -* `args` gives options which are passed through to the database engine, - except for options starting with `cp_`, which are used to configure the Twisted - connection pool. For a reference to valid arguments, see: - * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) - * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) - * for [the connection pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) +* `allow_unsafe_locale` (boolean): This option is specific to Postgres. Under the default behavior, Synapse will refuse to start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). Defaults to `false`. -For more information on using Synapse with Postgres, -see [here](../../postgres.md). +* `args` (object): Gives options which are passed through to the database engine, except for options starting with `cp_`, which are used to configure the Twisted connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) -Example SQLite configuration: +Example configurations: ```yaml database: name: sqlite3 @@ -1573,7 +1411,6 @@ database: database: /path/to/homeserver.db ``` -Example Postgres configuration: ```yaml database: name: psycopg2 @@ -1590,19 +1427,11 @@ database: --- ### `databases` -The `databases` option allows specifying a mapping between certain database tables and -database host details, spreading the load of a single Synapse instance across multiple -database backends. This is often referred to as "database sharding". This option is only -supported for PostgreSQL database backends. +*(object)* The `databases` option allows specifying a mapping between certain database tables and database host details, spreading the load of a single Synapse instance across multiple database backends. This is often referred to as "database sharding". This option is only supported for PostgreSQL database backends. -**Important note:** This is a supported option, but is not currently used in production by the -Matrix.org Foundation. Proceed with caution and always make backups. +**Important note:** This is a supported option, but is not currently used in production by the Matrix.org Foundation. Proceed with caution and always make backups. -`databases` is a dictionary of arbitrarily-named database entries. Each entry is equivalent -to the value of the `database` homeserver config option (see above), with the addition of -a `data_stores` key. `data_stores` is an array of strings that specifies the data store(s) -(a defined label for a set of tables) that should be stored on the associated database -backend entry. +`databases` is a dictionary of arbitrarily-named database entries. Each entry is equivalent to the value of the `database` homeserver config option (see above), with the addition of a `data_stores` key. `data_stores` is an array of strings that specifies the data store(s) (a defined label for a set of tables) that should be stored on the associated database backend entry. The currently defined values for `data_stores` are: @@ -1618,30 +1447,22 @@ The currently defined values for `data_stores` are: * `"main"`: All other database tables and sequences. -All databases will end up with additional tables used for tracking database schema migrations -and any pending background updates. Synapse will create these automatically on startup when checking for -and/or performing database schema migrations. +All databases will end up with additional tables used for tracking database schema migrations and any pending background updates. Synapse will create these automatically on startup when checking for and/or performing database schema migrations. -To migrate an existing database configuration (e.g. all tables on a single database) to a different -configuration (e.g. the "main" data store on one database, and "state" on another), do the following: +To migrate an existing database configuration (e.g. all tables on a single database) to a different configuration (e.g. the "main" data store on one database, and "state" on another), do the following: 1. Take a backup of your existing database. Things can and do go wrong and database corruption is no joke! -2. Ensure all pending database migrations have been applied and background updates have run. The simplest - way to do this is to use the `update_synapse_database` script supplied with your Synapse installation. +2. Ensure all pending database migrations have been applied and background updates have run. The simplest way to do this is to use the `update_synapse_database` script supplied with your Synapse installation. ```sh update_synapse_database --database-config homeserver.yaml --run-background-updates ``` -3. Copy over the necessary tables and sequences from one database to the other. Tables relating to database - migrations, schemas, schema versions and background updates should **not** be copied. +3. Copy over the necessary tables and sequences from one database to the other. Tables relating to database migrations, schemas, schema versions and background updates should **not** be copied. - As an example, say that you'd like to split out the "state" data store from an existing database which - currently contains all data stores. + As an example, say that you'd like to split out the "state" data store from an existing database which currently contains all data stores. - Simply copy the tables and sequences defined above for the "state" datastore from the existing database - to the secondary database. As noted above, additional tables will be created in the secondary database - when Synapse is started. + Simply copy the tables and sequences defined above for the "state" datastore from the existing database to the secondary database. As noted above, additional tables will be created in the secondary database when Synapse is started. 4. Modify/create the `databases` option in your `homeserver.yaml` to match the desired database configuration. 5. Start Synapse. Check that it starts up successfully and that things generally seem to be working. @@ -1649,14 +1470,16 @@ configuration (e.g. the "main" data store on one database, and "state" on anothe Only one of the options `database` or `databases` may be specified in your config, but not both. -Example configuration: +Defaults to `{}`. +Example configuration: ```yaml databases: basement_box: name: psycopg2 txn_limit: 10000 - data_stores: ["main"] + data_stores: + - main args: user: synapse_user password: secretpassword @@ -1665,11 +1488,11 @@ databases: port: 5432 cp_min: 5 cp_max: 10 - my_other_database: name: psycopg2 txn_limit: 10000 - data_stores: ["state"] + data_stores: + - state args: user: synapse_user password: secretpassword @@ -1681,240 +1504,345 @@ databases: ``` --- ## Logging + Config options related to logging. --- ### `log_config` -This option specifies a yaml python logging config file as described -[here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). +*(string|null)* This option specifies a yaml python logging config file as described [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). Defaults to `null`. Example configuration: ```yaml -log_config: "CONFDIR/SERVERNAME.log.config" +log_config: CONFDIR/SERVERNAME.log.config ``` --- ## Ratelimiting + Options related to ratelimiting in Synapse. Each ratelimiting configuration is made of two parameters: - - `per_second`: number of requests a client can send per second. - - `burst_count`: number of requests a client can send before being throttled. +- `per_second`: number of requests a client can send per second. +- `burst_count`: number of requests a client can send before being throttled. + --- ### `rc_message` +*(object)* Ratelimiting settings for client messaging. -Ratelimiting settings for client messaging. +This is a ratelimiting option for messages that ratelimits sending based on the account the client is using. -This is a ratelimiting option for messages that ratelimits sending based on the account the client -is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_message: + per_second: 0.2 + burst_count: 10.0 +``` Example configuration: ```yaml rc_message: per_second: 0.5 - burst_count: 15 + burst_count: 15.0 ``` --- ### `rc_registration` -This option ratelimits registration requests based on the client's IP address. -It defaults to `per_second: 0.17`, `burst_count: 3`. +*(object)* This option ratelimits registration requests based on the client's IP address. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_registration: + per_second: 0.17 + burst_count: 3.0 +``` Example configuration: ```yaml rc_registration: per_second: 0.15 - burst_count: 2 + burst_count: 2.0 ``` --- ### `rc_registration_token_validity` -This option checks the validity of registration tokens that ratelimits requests based on -the client's IP address. -Defaults to `per_second: 0.1`, `burst_count: 5`. +*(object)* This option checks the validity of registration tokens that ratelimits requests based on the client's IP address. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_registration_token_validity: + per_second: 0.1 + burst_count: 5.0 +``` Example configuration: ```yaml rc_registration_token_validity: per_second: 0.3 - burst_count: 6 + burst_count: 6.0 ``` --- ### `rc_login` -This option specifies several limits for login: -* `address` ratelimits login requests based on the client's IP - address. Defaults to `per_second: 0.003`, `burst_count: 5`. +*(object)* This option specifies several limits for login. -* `account` ratelimits login requests based on the account the - client is attempting to log into. Defaults to `per_second: 0.003`, - `burst_count: 5`. +This setting has the following sub-options: -* `failed_attempts` ratelimits login requests based on the account the - client is attempting to log into, based on the amount of failed login - attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. +* `address` (object): Ratelimits login requests based on the client's IP address. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `account` (object): Ratelimits login requests based on the account the client is attempting to log into. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `failed_attempts` (object): Ratelimits login requests based on the account the client is attempting to log into, based on the amount of failed login attempts for this account. Defaults to `{"per_second": 0.17, "burst_count": 3.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_login: address: per_second: 0.15 - burst_count: 5 + burst_count: 5.0 account: per_second: 0.18 - burst_count: 4 + burst_count: 4.0 failed_attempts: per_second: 0.19 - burst_count: 7 + burst_count: 7.0 ``` --- ### `rc_admin_redaction` -This option sets ratelimiting redactions by room admins. If this is not explicitly -set then it uses the same ratelimiting as per `rc_message`. This is useful -to allow room admins to deal with abuse quickly. +*(object)* This option sets ratelimiting redactions by room admins. If this is not explicitly set then it uses the same ratelimiting as per `rc_message`. This is useful to allow room admins to deal with abuse quickly. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_admin_redaction: - per_second: 1 - burst_count: 50 + per_second: 1.0 + burst_count: 50.0 ``` --- ### `rc_joins` -This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: +*(object)* This option allows for ratelimiting number of rooms a user can join. -* `local`: ratelimits when users are joining rooms the server is already in. - Defaults to `per_second: 0.1`, `burst_count: 10`. +This setting has the following sub-options: -* `remote`: ratelimits when users are trying to join rooms not on the server (which - can be more computationally expensive than restricting locally). Defaults to - `per_second: 0.01`, `burst_count: 10` +* `local` (object): Ratelimits when users are joining rooms the server is already in. Defaults to `{"per_second": 0.1, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `remote` (object): Ratelimits when users are trying to join rooms not on the server (which can be more computationally expensive than restricting locally). Defaults to `{"per_second": 0.01, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. Example configuration: ```yaml rc_joins: local: per_second: 0.2 - burst_count: 15 + burst_count: 15.0 remote: per_second: 0.03 - burst_count: 12 + burst_count: 12.0 ``` --- ### `rc_joins_per_room` -This option allows admins to ratelimit joins to a room based on the number of recent -joins (local or remote) to that room. It is intended to mitigate mass-join spam -waves which target multiple homeservers. - -By default, one join is permitted to a room every second, with an accumulating -buffer of up to ten instantaneous joins. - -Example configuration (default values): -```yaml -rc_joins_per_room: - per_second: 1 - burst_count: 10 -``` +*(object)* This option allows admins to ratelimit joins to a room based on the number of recent joins (local or remote) to that room. It is intended to mitigate mass-join spam waves which target multiple homeservers. _Added in Synapse 1.64.0._ +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_joins_per_room: + per_second: 1.0 + burst_count: 10.0 +``` + +Example configuration: +```yaml +rc_joins_per_room: + per_second: 1.0 + burst_count: 10.0 +``` --- ### `rc_3pid_validation` -This option ratelimits how often a user or IP can attempt to validate a 3PID. -Defaults to `per_second: 0.003`, `burst_count: 5`. +*(object)* This option ratelimits how often a user or IP can attempt to validate a 3PID. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5.0 +``` Example configuration: ```yaml rc_3pid_validation: per_second: 0.003 - burst_count: 5 + burst_count: 5.0 ``` --- ### `rc_invites` -This option sets ratelimiting how often invites can be sent in a room or to a -specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10`, -`per_user` defaults to `per_second: 0.003`, `burst_count: 5`, and `per_issuer` -defaults to `per_second: 0.3`, `burst_count: 10`. +*(object)* This option sets ratelimiting how often invites can be sent in a room or to a specific user. -Client requests that invite user(s) when [creating a -room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) -will count against the `rc_invites.per_room` limit, whereas -client requests to [invite a single user to a -room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) -will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. +Client requests that invite user(s) when [creating a room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) will count against the `rc_invites.per_room` limit, whereas client requests to [invite a single user to a room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. -Federation requests to invite a user will count against the `rc_invites.per_user` -limit only, as Synapse presumes ratelimiting by room will be done by the sending server. - -The `rc_invites.per_user` limit applies to the *receiver* of the invite, rather than the -sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user -cannot *receive* more than a burst of 5 invites at a time. - -In contrast, the `rc_invites.per_issuer` limit applies to the *issuer* of the invite, meaning that a `rc_invite.per_issuer.burst_count` of 5 mandates that single user cannot *send* more than a burst of 5 invites at a time. +Federation requests to invite a user will count against the `rc_invites.per_user` limit only, as Synapse presumes ratelimiting by room will be done by the sending server. _Changed in version 1.63:_ added the `per_issuer` limit. +This setting has the following sub-options: + +* `per_room` (object): Applies to the room of the invitation. Defaults to `{"per_second": 0.3, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `per_user` (object): Applies to the *receiver* of the invite, rather than the sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user cannot *receive* more than a burst of 5 invites at a time. Defaults to `{"per_second": 0.003, "burst_count": 5.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +* `per_issuer` (object): Applies to the *issuer* of the invite, meaning that a `rc_invite.per_issuer.burst_count` of 5 mandates that single user cannot *send* more than a burst of 5 invites at a time. Defaults to `{"per_second": 0.3, "burst_count": 10.0}`. + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + Example configuration: ```yaml rc_invites: per_room: per_second: 0.5 - burst_count: 5 + burst_count: 5.0 per_user: per_second: 0.004 - burst_count: 3 + burst_count: 3.0 per_issuer: per_second: 0.5 - burst_count: 5 + burst_count: 5.0 ``` - --- ### `rc_third_party_invite` -This option ratelimits 3PID invites (i.e. invites sent to a third-party ID -such as an email address or a phone number) based on the account that's -sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. +*(object)* This option ratelimits 3PID invites (i.e. invites sent to a third-party ID such as an email address or a phone number) based on the account that's sending the invite. -Example configuration: +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: ```yaml rc_third_party_invite: per_second: 0.2 - burst_count: 10 + burst_count: 10.0 ``` --- ### `rc_media_create` -This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create` -endpoint based on the account that's creating the media. Defaults to -`per_second: 10`, `burst_count: 50`. +*(object)* This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create` endpoint based on the account that's creating the media. -Example configuration: +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: ```yaml rc_media_create: - per_second: 10 - burst_count: 50 + per_second: 10.0 + burst_count: 50.0 ``` --- ### `rc_federation` -Defines limits on federation requests. +*(object)* Defines limits on federation requests. -The `rc_federation` configuration has the following sub-options: -* `window_size`: window size in milliseconds. Defaults to 1000. -* `sleep_limit`: number of federation requests from a single server in - a window before the server will delay processing the request. Defaults to 10. -* `sleep_delay`: duration in milliseconds to delay processing events - from remote servers by if they go over the sleep limit. Defaults to 500. -* `reject_limit`: maximum number of concurrent federation requests - allowed from a single server. Defaults to 50. -* `concurrent`: number of federation requests to concurrently process - from a single server. Defaults to 3. +This setting has the following sub-options: + +* `window_size` (integer): Window size in milliseconds. Defaults to `1000`. + +* `sleep_limit` (integer): Number of federation requests from a single server in a window before the server will delay processing the request. Defaults to `10`. + +* `sleep_delay` (integer): Duration in milliseconds to delay processing events from remote servers by if they go over the sleep limit. Defaults to `500`. + +* `reject_limit` (integer): Maximum number of concurrent federation requests allowed from a single server. Defaults to `50`. + +* `concurrent` (integer): Number of federation requests to concurrently process from a single server. Defaults to `3`. Example configuration: ```yaml @@ -1928,55 +1856,75 @@ rc_federation: --- ### `rc_presence` -This option sets ratelimiting for presence. +*(object)* This option sets ratelimiting for presence. -The `rc_presence.per_user` option sets rate limits on how often a specific -users' presence updates are evaluated. Ratelimited presence updates sent via sync are -ignored, and no error is returned to the client. -This option also sets the rate limit for the -[`PUT /_matrix/client/v3/presence/{userId}/status`](https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3presenceuseridstatus) -endpoint. +This setting has the following sub-options: -`per_user` defaults to `per_second: 0.1`, `burst_count: 1`. +* `per_user` (object): Sets rate limits on how often a specific users' presence updates are evaluated. Ratelimited presence updates sent via sync are ignored, and no error is returned to the client. This option also sets the rate limit for the [`PUT /_matrix/client/v3/presence/{userId}/status`] endpoint. + + [`PUT /_matrix/client/v3/presence/{userId}/status`]: + + + This setting has the following sub-options: + + * `per_second` (number): Maximum number of requests a client can send per second. + + * `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_presence: + per_user: + per_second: 0.1 + burst_count: 1.0 +``` Example configuration: ```yaml rc_presence: per_user: per_second: 0.05 - burst_count: 1 + burst_count: 1.0 ``` --- ### `rc_delayed_event_mgmt` -Ratelimiting settings for delayed event management. +*(object)* Ratelimiting settings for delayed event management. -This is a ratelimiting option that ratelimits -attempts to restart, cancel, or view delayed events -based on the sending client's account and device ID. -It defaults to: `per_second: 1`, `burst_count: 5`. +This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account and device ID. Attempts to create or send delayed events are ratelimited not by this setting, but by `rc_message`. -Setting this to a high value allows clients to make delayed event management requests often -(such as repeatedly restarting a delayed event with a short timeout, -or restarting several different delayed events all at once) -without the risk of being ratelimited. +Setting this to a high value allows clients to make delayed event management requests often (such as repeatedly restarting a delayed event with a short timeout, or restarting several different delayed events all at once) without the risk of being ratelimited. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_delayed_event_mgmt: + per_user: + per_second: 1.0 + burst_count: 5.0 +``` Example configuration: ```yaml rc_delayed_event_mgmt: - per_second: 2 - burst_count: 20 + per_second: 2.0 + burst_count: 20.0 ``` --- ### `federation_rr_transactions_per_room_per_second` -Sets outgoing federation transaction frequency for sending read-receipts, -per-room. +*(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. -If we end up trying to send out more read-receipts, they will get buffered up -into fewer transactions. Defaults to 50. +If we end up trying to send out more read-receipts, they will get buffered up into fewer transactions. + +Defaults to `50`. Example configuration: ```yaml @@ -1984,37 +1932,29 @@ federation_rr_transactions_per_room_per_second: 40 ``` --- ## Media Store + Config options related to Synapse's media store. --- ### `enable_authenticated_media` -When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy -unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) - requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false -after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on. +*(boolean)* When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) – requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on. -In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this -case-by-case breakdown describes whether media downloads are permitted: +In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this case-by-case breakdown describes whether media downloads are permitted: * `enable_authenticated_media = False`: * unauthenticated client or homeserver requesting local media: allowed - * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, - or as long as the remote homeserver does not require authentication to retrieve the media + * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, or as long as the remote homeserver does not require authentication to retrieve the media * `enable_authenticated_media = True`: - * unauthenticated client or homeserver requesting local media: - allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); - otherwise denied. - * unauthenticated client or homeserver requesting remote media: the same as for local media; - allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); - otherwise denied. + * unauthenticated client or homeserver requesting local media: allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + * unauthenticated client or homeserver requesting remote media: the same as for local media; allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. -It is especially notable that media downloaded before this option existed (in older Synapse versions), or whilst this option was set to `False`, -will perpetually be available over the legacy, unauthenticated endpoint, even after this option is set to `True`. -This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media; -those older clients or homeservers will not be cut off from media they can already see. +It is especially notable that media downloaded before this option existed (in older Synapse versions), or whilst this option was set to `False`, will perpetually be available over the legacy, unauthenticated endpoint, even after this option is set to `True`. This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media; those older clients or homeservers will not be cut off from media they can already see. _Changed in Synapse 1.120:_ This option now defaults to `True` when not set, whereas before this version it defaulted to `False`. +Defaults to `true`. + Example configuration: ```yaml enable_authenticated_media: false @@ -2022,8 +1962,7 @@ enable_authenticated_media: false --- ### `enable_media_repo` -Enable the media store service in the Synapse master. Defaults to true. -Set to false if you are using a separate media store worker. +*(boolean)* Enable the media store service in the Synapse master. Set to false if you are using a separate media store worker. Defaults to `true`. Example configuration: ```yaml @@ -2032,18 +1971,16 @@ enable_media_repo: false --- ### `media_store_path` -Directory where uploaded images and attachments are stored. +*(string)* Directory where uploaded images and attachments are stored. Defaults to `"media_store"`. Example configuration: ```yaml -media_store_path: "DATADIR/media_store" +media_store_path: DATADIR/media_store ``` --- ### `max_pending_media_uploads` -How many *pending media uploads* can a given user have? A pending media upload -is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp -has not passed) and (b) the media has not yet been uploaded for. Defaults to 5. +*(integer)* How many *pending media uploads* can a given user have? A pending media upload is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp has not passed) and (b) the media has not yet been uploaded for. Defaults to `5`. Example configuration: ```yaml @@ -2052,42 +1989,51 @@ max_pending_media_uploads: 5 --- ### `unused_expiration_time` -How long to wait in milliseconds before expiring created media IDs. Defaults to -"24h" +*(duration)* How long to wait in milliseconds before expiring created media IDs. Defaults to `"24h"`. Example configuration: ```yaml -unused_expiration_time: "1h" +unused_expiration_time: 1h ``` --- ### `media_storage_providers` -Media storage providers allow media to be stored in different -locations. Defaults to none. Associated sub-options are: -* `module`: type of resource, e.g. `file_system`. -* `store_local`: whether to store newly uploaded local files -* `store_remote`: whether to store newly downloaded local files -* `store_synchronous`: whether to wait for successful storage for local uploads -* `config`: sets a path to the resource through the `directory` option +*(array)* Media storage providers allow media to be stored in different locations. Defaults to `[]`. + +Options for each entry include: + +* `module` (string): Type of resource, e.g. `file_system`. + +* `store_local` (boolean): Whether to store newly uploaded local files. + +* `store_remote` (boolean): Whether to store newly downloaded local files. + +* `store_synchronous` (boolean): Whether to wait for successful storage for local uploads. + +* `config` (object): Sets a path to the resource through the `directory` option. + + This setting has the following sub-options: + + * `directory` (string): Path to the resource. Example configuration: ```yaml media_storage_providers: - - module: file_system - store_local: false - store_remote: false - store_synchronous: false - config: - directory: /mnt/some/other/directory +- module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory ``` --- ### `max_upload_size` -The largest allowed upload size in bytes. +*(byte size)* The largest allowed upload size in bytes. -If you are using a reverse proxy you may also need to set this value in -your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. -See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. +If you are using a reverse proxy you may also need to set this value in your reverse proxy's config. Notably Nginx has a small max body size by default. See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Defaults to `"50M"`. Example configuration: ```yaml @@ -2096,7 +2042,7 @@ max_upload_size: 60M --- ### `max_image_pixels` -Maximum number of pixels that will be thumbnailed. Defaults to 32M. +*(byte size)* Maximum number of pixels that will be thumbnailed. Defaults to `"32M"`. Example configuration: ```yaml @@ -2105,7 +2051,7 @@ max_image_pixels: 35M --- ### `remote_media_download_burst_count` -Remote media downloads are ratelimited using a [leaky bucket algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given "bucket" is keyed to the IP address of the requester when requesting remote media downloads. This configuration option sets the size of the bucket against which the size in bytes of downloads are penalized - if the bucket is full, ie a given number of bytes have already been downloaded, further downloads will be denied until the bucket drains. Defaults to 500MiB. See also `remote_media_download_per_second` which determines the rate at which the "bucket" is emptied and thus has available space to authorize new requests. +*(byte size)* Remote media downloads are ratelimited using a [leaky bucket algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given "bucket" is keyed to the IP address of the requester when requesting remote media downloads. This configuration option sets the size of the bucket against which the size in bytes of downloads are penalized – if the bucket is full, i.e. a given number of bytes have already been downloaded, further downloads will be denied until the bucket drains. See also `remote_media_download_per_second` which determines the rate at which the "bucket" is emptied and thus has available space to authorize new requests. Defaults to `"500MiB"`. Example configuration: ```yaml @@ -2114,7 +2060,7 @@ remote_media_download_burst_count: 200M --- ### `remote_media_download_per_second` -Works in conjunction with `remote_media_download_burst_count` to ratelimit remote media downloads - this configuration option determines the rate at which the "bucket" (see above) leaks in bytes per second. As requests are made to download remote media, the size of those requests in bytes is added to the bucket, and once the bucket has reached it's capacity, no more requests will be allowed until a number of bytes has "drained" from the bucket. This setting determines the rate at which bytes drain from the bucket, with the practical effect that the larger the number, the faster the bucket leaks, allowing for more bytes downloaded over a shorter period of time. Defaults to 87KiB per second. See also `remote_media_download_burst_count`. +*(byte size)* Works in conjunction with `remote_media_download_burst_count` to ratelimit remote media downloads – this configuration option determines the rate at which the "bucket" (see above) leaks in bytes per second. As requests are made to download remote media, the size of those requests in bytes is added to the bucket, and once the bucket has reached it's capacity, no more requests will be allowed until a number of bytes has "drained" from the bucket. This setting determines the rate at which bytes drain from the bucket, with the practical effect that the larger the number, the faster the bucket leaks, allowing for more bytes downloaded over a shorter period of time. Defaults to 87KiB per second. See also `remote_media_download_burst_count`. Defaults to `"87KiB"`. Example configuration: ```yaml @@ -2123,36 +2069,24 @@ remote_media_download_per_second: 40K --- ### `prevent_media_downloads_from` -A list of domains to never download media from. Media from these -domains that is already downloaded will not be deleted, but will be -inaccessible to users. This option does not affect admin APIs trying -to download/operate on media. +*(array)* A list of domains to never download media from. Media from these domains that is already downloaded will not be deleted, but will be inaccessible to users. This option does not affect admin APIs trying to download/operate on media. -This will not prevent the listed domains from accessing media themselves. -It simply prevents users on this server from downloading media originating -from the listed servers. +This will not prevent the listed domains from accessing media themselves. It simply prevents users on this server from downloading media originating from the listed servers. -This will have no effect on media originating from the local server. This only -affects media downloaded from other Matrix servers, to control URL previews see -[`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or -[`url_preview_url_blacklist`](#url_preview_url_blacklist). +This will have no effect on media originating from the local server. This only affects media downloaded from other Matrix servers, to control URL previews see [`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or [`url_preview_url_blacklist`](#url_preview_url_blacklist). -Defaults to an empty list (nothing blocked). +Defaults to `[]`. Example configuration: ```yaml prevent_media_downloads_from: - - evil.example.org - - evil2.example.org +- evil.example.org +- evil2.example.org ``` --- ### `dynamic_thumbnails` -Whether to generate new thumbnails on the fly to precisely match -the resolution requested by the client. If true then whenever -a new resolution is requested by the client the server will -generate a new thumbnail. If false the server will pick a thumbnail -from a precalculated list. Defaults to false. +*(boolean)* Whether to generate new thumbnails on the fly to precisely match the resolution requested by the client. If true then whenever a new resolution is requested by the client the server will generate a new thumbnail. If false the server will pick a thumbnail from a precalculated list. Defaults to `false`. Example configuration: ```yaml @@ -2161,68 +2095,62 @@ dynamic_thumbnails: true --- ### `thumbnail_sizes` -List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: -* `width` -* `height` -* `method`: i.e. `crop`, `scale`, etc. +*(array)* List of thumbnails to precalculate when an image is uploaded. -Example configuration: +Options for each entry include: + +* `width` (integer): Width of the generated thumbnail. + +* `height` (integer): Height of the generated thumbnail. + +* `method` (string): Method to fit the thumbnail dimensions. Current options are `crop` and `scale`. + +Default configuration: ```yaml thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 320 - height: 240 - method: scale - - width: 640 - height: 480 - method: scale - - width: 800 - height: 600 - method: scale +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale ``` --- ### `media_retention` -Controls whether local media and entries in the remote media cache -(media that is downloaded from other homeservers) should be removed -under certain conditions, typically for the purpose of saving space. +*(object)* Controls whether local media and entries in the remote media cache (media that is downloaded from other homeservers) should be removed under certain conditions, typically for the purpose of saving space. -Purging media files will be the carried out by the media worker -(that is, the worker that has the `enable_media_repo` homeserver config -option set to 'true'). This may be the main process. +Purging media files will be the carried out by the media worker (that is, the worker that has the `enable_media_repo` homeserver config option set to `true`). This may be the main process. -The `media_retention.local_media_lifetime` and -`media_retention.remote_media_lifetime` config options control whether -media will be purged if it has not been accessed in a given amount of -time. Note that media is 'accessed' when loaded in a room in a client, or -otherwise downloaded by a local or remote user. If the media has never -been accessed, the media's creation time is used instead. Both thumbnails -and the original media will be removed. If either of these options are unset, -then media of that type will not be purged. +The `media_retention.local_media_lifetime` and `media_retention.remote_media_lifetime` config options control whether media will be purged if it has not been accessed in a given amount of time. Note that media is "accessed" when loaded in a room in a client, or otherwise downloaded by a local or remote user. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If either of these options are unset, then media of that type will not be purged. -Local or cached remote media that has been -[quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) -will not be deleted. Similarly, local media that has been marked as -[protected from quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) -will not be deleted. +Local or cached remote media that has been [quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) will not be deleted. Similarly, local media that has been marked as [protected from quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) will not be deleted. + +This setting has the following sub-options: + +* `local_media_lifetime`: Duration without access to a local media resource after which it will be purged. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If unset or null, local media will not be purged. Defaults to `null`. + +* `remote_media_lifetime`: Duration without access to a remote media resource after which it will be purged. If the media has never been accessed, the media's creation time is used instead. Both thumbnails and the original media will be removed. If unset or null, remote media will not be purged. Defaults to `null`. Example configuration: ```yaml media_retention: - local_media_lifetime: 90d - remote_media_lifetime: 14d + local_media_lifetime: 90d + remote_media_lifetime: 14d ``` --- ### `url_preview_enabled` -This setting determines whether the preview URL API is enabled. -It is disabled by default. Set to true to enable. If enabled you must specify a -`url_preview_ip_range_blacklist` blacklist. +*(boolean)* This setting determines whether the preview URL API is enabled. Set to true to enable. If enabled you must specify a `url_preview_ip_range_blacklist` blacklist. Defaults to `false`. Example configuration: ```yaml @@ -2231,111 +2159,80 @@ url_preview_enabled: true --- ### `url_preview_ip_range_blacklist` -List of IP address CIDR ranges that the URL preview spider is denied -from accessing. There are no defaults: you must explicitly -specify a list for URL previewing to work. You should specify any -internal services in your network that you do not want synapse to try -to connect to, otherwise anyone in any Matrix room could cause your -synapse to issue arbitrary GET requests to your internal services, -causing serious security issues. +*(array|null)* List of IP address CIDR ranges that the URL preview spider is denied from accessing. There are no defaults: you must explicitly specify a list for URL previewing to work. You should specify any internal services in your network that you do not want synapse to try to connect to, otherwise anyone in any Matrix room could cause your synapse to issue arbitrary GET requests to your internal services, causing serious security issues. -(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -listed here, since they correspond to unroutable addresses.) +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly listed here, since they correspond to unroutable addresses.) -This must be specified if `url_preview_enabled` is set. It is recommended that -you use the following example list as a starting point. +This must be specified if `url_preview_enabled` is set. It is recommended that you use the following example list as a starting point. Note: The value is ignored when an HTTP proxy is in use. +Defaults to `null`. + Example configuration: ```yaml url_preview_ip_range_blacklist: - - '127.0.0.0/8' - - '10.0.0.0/8' - - '172.16.0.0/12' - - '192.168.0.0/16' - - '100.64.0.0/10' - - '192.0.0.0/24' - - '169.254.0.0/16' - - '192.88.99.0/24' - - '198.18.0.0/15' - - '192.0.2.0/24' - - '198.51.100.0/24' - - '203.0.113.0/24' - - '224.0.0.0/4' - - '::1/128' - - 'fe80::/10' - - 'fc00::/7' - - '2001:db8::/32' - - 'ff00::/8' - - 'fec0::/10' +- 127.0.0.0/8 +- 10.0.0.0/8 +- 172.16.0.0/12 +- 192.168.0.0/16 +- 100.64.0.0/10 +- 192.0.0.0/24 +- 169.254.0.0/16 +- 192.88.99.0/24 +- 198.18.0.0/15 +- 192.0.2.0/24 +- 198.51.100.0/24 +- 203.0.113.0/24 +- 224.0.0.0/4 +- ::1/128 +- fe80::/10 +- fc00::/7 +- 2001:db8::/32 +- ff00::/8 +- fec0::/10 ``` --- ### `url_preview_ip_range_whitelist` -This option sets a list of IP address CIDR ranges that the URL preview spider is allowed -to access even if they are specified in `url_preview_ip_range_blacklist`. -This is useful for specifying exceptions to wide-ranging blacklisted -target IP ranges - e.g. for enabling URL previews for a specific private -website only visible in your network. Defaults to none. +*(array)* This option sets a list of IP address CIDR ranges that the URL preview spider is allowed to access even if they are specified in `url_preview_ip_range_blacklist`. This is useful for specifying exceptions to wide-ranging blacklisted target IP ranges – e.g. for enabling URL previews for a specific private website only visible in your network. Defaults to `[]`. Example configuration: ```yaml url_preview_ip_range_whitelist: - - '192.168.1.1' +- 192.168.1.1 ``` --- ### `url_preview_url_blacklist` -Optional list of URL matches that the URL preview spider is denied from -accessing. This is a usability feature, not a security one. You should use -`url_preview_ip_range_blacklist` in preference to this, otherwise someone could -define a public DNS entry that points to a private IP address and circumvent -the blacklist. Applications that perform redirects or serve different content -when detecting that Synapse is accessing them can also bypass the blacklist. -This is more useful if you know there is an entire shape of URL that you know -that you do not want Synapse to preview. +*(array)* Optional list of URL matches that the URL preview spider is denied from accessing. This is a usability feature, not a security one. You should use `url_preview_ip_range_blacklist` in preference to this, otherwise someone could define a public DNS entry that points to a private IP address and circumvent the blacklist. Applications that perform redirects or serve different content when detecting that Synapse is accessing them can also bypass the blacklist. This is more useful if you know there is an entire shape of URL that you know that you do not want Synapse to preview. -Each list entry is a dictionary of url component attributes as returned -by urlparse.urlsplit as applied to the absolute form of the URL. See -[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more -information. Some examples are: +Each list entry is a dictionary of url component attributes as returned by urlparse.urlsplit as applied to the absolute form of the URL. See [here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more information. Some examples are: * `username` * `netloc` * `scheme` * `path` -The values of the dictionary are treated as a filename match pattern -applied to that component of URLs, unless they start with a ^ in which -case they are treated as a regular expression match. If all the -specified component matches for a given list item succeed, the URL is -blacklisted. +The values of the dictionary are treated as a filename match pattern applied to that component of URLs, unless they start with a ^ in which case they are treated as a regular expression match. If all the specified component matches for a given list item succeed, the URL is blacklisted. + +Defaults to `[]`. Example configuration: ```yaml url_preview_url_blacklist: - # blacklist any URL with a username in its URI - - username: '*' - - # blacklist all *.google.com URLs - - netloc: 'google.com' - - netloc: '*.google.com' - - # blacklist all plain HTTP URLs - - scheme: 'http' - - # blacklist http(s)://www.acme.com/foo - - netloc: 'www.acme.com' - path: '/foo' - - # blacklist any URL with a literal IPv4 address - - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +- username: '*' +- netloc: google.com +- netloc: '*.google.com' +- scheme: http +- netloc: www.acme.com + path: /foo +- netloc: ^[0-9]+.[0-9]+.[0-9]+.[0-9]+$ ``` --- ### `max_spider_size` -The largest allowed URL preview spidering size in bytes. Defaults to 10M. +*(byte size)* The largest allowed URL preview spidering size in bytes. Defaults to `"10M"`. Example configuration: ```yaml @@ -2344,43 +2241,43 @@ max_spider_size: 8M --- ### `url_preview_accept_language` -A list of values for the Accept-Language HTTP header used when -downloading webpages during URL preview generation. This allows -Synapse to specify the preferred languages that URL previews should -be in when communicating with remote servers. +*(array)* A list of values for the Accept-Language HTTP header used when downloading webpages during URL preview generation. This allows Synapse to specify the preferred languages that URL previews should be in when communicating with remote servers. -Each value is a IETF language tag; a 2-3 letter identifier for a -language, optionally followed by subtags separated by '-', specifying -a country or region variant. +Each value is a IETF language tag; a 2-3 letter identifier for a language, optionally followed by subtags separated by `-`, specifying a country or region variant. -Multiple values can be provided, and a weight can be added to each by -using quality value syntax (;q=). '*' translates to any language. +Multiple values can be provided, and a weight can be added to each by using quality value syntax (;q=). `*` translates to any language. -Defaults to "en". +Default configuration: +```yaml +url_preview_accept_language: +- en +``` Example configuration: ```yaml - url_preview_accept_language: - - 'en-UK' - - 'en-US;q=0.9' - - 'fr;q=0.8' - - '*;q=0.7' +url_preview_accept_language: +- en-UK +- en-US;q=0.9 +- fr;q=0.8 +- '*;q=0.7' ``` --- ### `oembed` -oEmbed allows for easier embedding content from a website. It can be -used for generating URLs previews of services which support it. A default list of oEmbed providers -is included with Synapse. Set `disable_default_providers` to true to disable using -these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each -should be in the form of providers.json). By default this list is empty. +*(object)* oEmbed allows for easier embedding content from a website. It can be used for generating URLs previews of services which support it. A default list of oEmbed providers is included with Synapse. + +This setting has the following sub-options: + +* `disable_default_providers` (boolean): Do not use Synapse's default list of oEmbed providers. Defaults to `false`. + +* `additional_providers` (array): Additional files with oEmbed configuration (each should be in the form of providers.json). Defaults to `[]`. Example configuration: ```yaml oembed: disable_default_providers: true additional_providers: - - oembed/my_providers.json + - oembed/my_providers.json ``` --- ## Captcha @@ -2390,33 +2287,30 @@ See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. --- ### `recaptcha_public_key` -This homeserver's ReCAPTCHA public key. Must be specified if -[`enable_registration_captcha`](#enable_registration_captcha) is enabled. +*(string|null)* This homeserver's ReCAPTCHA public key. Must be specified if [`enable_registration_captcha`](#enable_registration_captcha) is enabled. Defaults to `null`. Example configuration: ```yaml -recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_public_key: YOUR_PUBLIC_KEY ``` --- ### `recaptcha_private_key` -This homeserver's ReCAPTCHA private key. Must be specified if -[`enable_registration_captcha`](#enable_registration_captcha) is -enabled. +*(string|null)* This homeserver's ReCAPTCHA private key. Must be specified if [`enable_registration_captcha`](#enable_registration_captcha) is enabled. Defaults to `null`. Example configuration: ```yaml -recaptcha_private_key: "YOUR_PRIVATE_KEY" +recaptcha_private_key: YOUR_PRIVATE_KEY ``` --- ### `enable_registration_captcha` -Set to `true` to require users to complete a CAPTCHA test when registering an account. -Requires a valid ReCaptcha public/private key. -Defaults to `false`. +*(boolean)* Set to `true` to require users to complete a CAPTCHA test when registering an account. Requires a valid ReCaptcha public/private key. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `false`. + Example configuration: ```yaml enable_registration_captcha: true @@ -2424,65 +2318,73 @@ enable_registration_captcha: true --- ### `recaptcha_siteverify_api` -The API endpoint to use for verifying `m.login.recaptcha` responses. -Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. +*(string)* The API endpoint to use for verifying `m.login.recaptcha` responses. Defaults to `"https://www.recaptcha.net/recaptcha/api/siteverify"`. Example configuration: ```yaml -recaptcha_siteverify_api: "https://my.recaptcha.site" +recaptcha_siteverify_api: https://my.recaptcha.site ``` --- ## TURN + Options related to adding a TURN server to Synapse. --- ### `turn_uris` -The public URIs of the TURN server to give to clients. +*(array)* The public URIs of the TURN server to give to clients. Defaults to `[]`. Example configuration: ```yaml -turn_uris: [turn:example.org] +turn_uris: +- turn:example.org ``` --- ### `turn_shared_secret` -The shared secret used to compute passwords for the TURN server. +*(string|null)* The shared secret used to compute passwords for the TURN server. Defaults to `null`. Example configuration: ```yaml -turn_shared_secret: "YOUR_SHARED_SECRET" +turn_shared_secret: YOUR_SHARED_SECRET ``` --- ### `turn_shared_secret_path` -An alternative to [`turn_shared_secret`](#turn_shared_secret): -allows the shared secret to be specified in an external file. +*(string|null)* An alternative to [`turn_shared_secret`](#turn_shared_secret): allows the shared secret to be specified in an external file. -The file should be a plain text file, containing only the shared secret. -Synapse reads the shared secret from the given file once at startup. +The file should be a plain text file, containing only the shared secret. Synapse reads the shared secret from the given file once at startup. + +_Added in Synapse 1.116.0._ + +Defaults to `null`. Example configuration: ```yaml turn_shared_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.116.0._ - --- -### `turn_username` and `turn_password` +### `turn_username` -The Username and password if the TURN server needs them and does not use a token. +*(string|null)* TURN server username if not using a token. Defaults to `null`. Example configuration: ```yaml -turn_username: "TURNSERVER_USERNAME" -turn_password: "TURNSERVER_PASSWORD" +turn_username: TURNSERVER_USERNAME +``` +--- +### `turn_password` + +*(string|null)* TURN server password if not using a token. Defaults to `null`. + +Example configuration: +```yaml +turn_password: TURNSERVER_PASSWORD ``` --- ### `turn_user_lifetime` -How long generated TURN credentials last. Defaults to 1h. +*(duration)* How long generated TURN credentials last. Defaults to `"1h"`. Example configuration: ```yaml @@ -2491,37 +2393,33 @@ turn_user_lifetime: 2h --- ### `turn_allow_guests` -Whether guests should be allowed to use the TURN server. This defaults to true, otherwise -VoIP will be unreliable for guests. However, it does introduce a slight security risk as -it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). +*(boolean)* Whether guests should be allowed to use the TURN server. If false, VoIP will be unreliable for guests. However, it does introduce a slight security risk as it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). Defaults to `true`. Example configuration: ```yaml turn_allow_guests: false ``` --- -## Registration ## +## Registration Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. --- ### `enable_registration` -Enable registration for new users. Defaults to `false`. +*(boolean)* Enable registration for new users. -It is highly recommended that if you enable registration, you set one or more -or the following options, to avoid abuse of your server by "bots": +It is highly recommended that if you enable registration, you set one or more or the following options, to avoid abuse of your server by "bots": - * [`enable_registration_captcha`](#enable_registration_captcha) - * [`registrations_require_3pid`](#registrations_require_3pid) - * [`registration_requires_token`](#registration_requires_token) +* [`enable_registration_captcha`](#enable_registration_captcha) +* [`registrations_require_3pid`](#registrations_require_3pid) +* [`registration_requires_token`](#registration_requires_token) -(In order to enable registration without any verification, you must also set -[`enable_registration_without_verification`](#enable_registration_without_verification).) +(In order to enable registration without any verification, you must also set [`enable_registration_without_verification`](#enable_registration_without_verification).) -Note that even if this setting is disabled, new accounts can still be created -via the admin API if -[`registration_shared_secret`](#registration_shared_secret) is set. +Note that even if this setting is disabled, new accounts can still be created via the admin API if [`registration_shared_secret`](#registration_shared_secret) is set. + +Defaults to `false`. Example configuration: ```yaml @@ -2530,9 +2428,7 @@ enable_registration: true --- ### `enable_registration_without_verification` -Enable registration without email or captcha verification. Note: this option is *not* recommended, -as registration without verification is a known vector for spam and abuse. Defaults to `false`. Has no effect -unless [`enable_registration`](#enable_registration) is also enabled. +*(boolean)* Enable registration without email or captcha verification. Note: this option is *not* recommended, as registration without verification is a known vector for spam and abuse. Has no effect unless [`enable_registration`](#enable_registration) is also enabled. Defaults to `false`. Example configuration: ```yaml @@ -2541,21 +2437,22 @@ enable_registration_without_verification: true --- ### `registrations_require_3pid` -If this is set, users must provide all of the specified types of [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when registering an account. +*(array)* If this is set, users must provide all of the specified types of [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when registering an account. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `[]`. + Example configuration: ```yaml registrations_require_3pid: - - email - - msisdn +- email +- msisdn ``` --- ### `disable_msisdn_registration` -Explicitly disable asking for MSISDNs from the registration -flow (overrides `registrations_require_3pid` if MSISDNs are set as required). +*(boolean)* Explicitly disable asking for MSISDNs from the registration flow (overrides `registrations_require_3pid` if MSISDNs are set as required). Defaults to `false`. Example configuration: ```yaml @@ -2564,26 +2461,32 @@ disable_msisdn_registration: true --- ### `allowed_local_3pids` -Mandate that users are only allowed to associate certain formats of -3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. -`pattern` is a [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re). +*(array|null)* Mandate that users are only allowed to associate certain formats of 3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. `pattern` is a [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re). More information about 3PIDs, allowed `medium` types and their `address` syntax can be found [in the Matrix spec](https://spec.matrix.org/latest/appendices/#3pid-types). +Defaults to `null`. + +Options for each entry include: + +* `medium` (string): The medium for which to allow 3PID association. + +* `pattern` (string): A [Perl-like regular expression](https://docs.python.org/3/library/re.html#module-re) allowing association of a 3PID to a local account if it matches the given format. + Example configuration: ```yaml allowed_local_3pids: - - medium: email - pattern: '^[^@]+@matrix\.org$' - - medium: email - pattern: '^[^@]+@vector\.im$' - - medium: msisdn - pattern: '^44\d{10}$' +- medium: email + pattern: ^[^@]+@matrix\.org$ +- medium: email + pattern: ^[^@]+@vector\.im$ +- medium: msisdn + pattern: ^44\d{10}$ ``` --- ### `enable_3pid_lookup` -Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. +*(boolean)* Enable 3PIDs lookup requests to identity servers from this server. Defaults to `true`. Example configuration: ```yaml @@ -2592,14 +2495,12 @@ enable_3pid_lookup: false --- ### `registration_requires_token` -Require users to submit a token during registration. -Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). -Disabling this option will not delete any tokens previously generated. -Defaults to `false`. Set to `true` to enable. - +*(boolean)* Require users to submit a token during registration. Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). Disabling this option will not delete any tokens previously generated. Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. +Defaults to `false`. + Example configuration: ```yaml registration_requires_token: true @@ -2607,55 +2508,44 @@ registration_requires_token: true --- ### `registration_shared_secret` -If set, allows registration of standard or admin accounts by anyone who has the -shared secret, even if [`enable_registration`](#enable_registration) is not -set. +*(string|null)* If set, allows registration of standard or admin accounts by anyone who has the shared secret, even if [`enable_registration`](#enable_registration) is not set. -This is primarily intended for use with the `register_new_matrix_user` script -(see [Registering a user](../../setup/installation.md#registering-a-user)); -however, the interface is [documented](../../admin_api/register_api.html). +This is primarily intended for use with the `register_new_matrix_user` script (see [Registering a user](../../setup/installation.md#registering-a-user)); however, the interface is [documented](../../admin_api/register_api.html). -Replacing an existing `registration_shared_secret` with a new one requires users -of the [Shared-Secret Registration API](../../admin_api/register_api.html) to -start using the new secret for requesting any further one-time nonces. +Replacing an existing `registration_shared_secret` with a new one requires users of the [Shared-Secret Registration API](../../admin_api/register_api.html) to start using the new secret for requesting any further one-time nonces. -> ⚠️ **Warning** – The additional consequences of replacing -> [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it delegates -> to `registration_shared_secret`. +> ⚠️ **Warning** – The additional consequences of replacing [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it delegates to `registration_shared_secret`. See also [`registration_shared_secret_path`](#registration_shared_secret_path). +Defaults to `null`. + Example configuration: ```yaml registration_shared_secret: ``` - --- ### `registration_shared_secret_path` -An alternative to [`registration_shared_secret`](#registration_shared_secret): -allows the shared secret to be specified in an external file. +*(string|null)* An alternative to [`registration_shared_secret`](#registration_shared_secret): allows the shared secret to be specified in an external file. The file should be a plain text file, containing only the shared secret. -If this file does not exist, Synapse will create a new shared -secret on startup and store it in this file. +If this file does not exist, Synapse will create a new shared secret on startup and store it in this file. + +_Added in Synapse 1.67.0._ + +Defaults to `null`. Example configuration: ```yaml registration_shared_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.67.0._ - --- ### `bcrypt_rounds` -Set the number of bcrypt rounds used to generate password hash. -Larger numbers increase the work factor needed to generate the hash. -The default number is 12 (which equates to 2^12 rounds). -N.B. that increasing this will exponentially increase the time required -to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +*(integer)* Set the number of bcrypt rounds used to generate password hash. Larger numbers increase the work factor needed to generate the hash. The default number is 12 (which equates to 2^12 rounds). N.B. that increasing this will exponentially increase the time required to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. Defaults to `12`. + Example configuration: ```yaml bcrypt_rounds: 14 @@ -2663,9 +2553,7 @@ bcrypt_rounds: 14 --- ### `allow_guest_access` -Allows users to register as guests without a password/email/etc, and -participate in rooms hosted on this server which have been made -accessible to anonymous users. Defaults to false. +*(boolean)* Allows users to register as guests without a password/email/etc, and participate in rooms hosted on this server which have been made accessible to anonymous users. Defaults to `false`. Example configuration: ```yaml @@ -2674,11 +2562,11 @@ allow_guest_access: true --- ### `default_identity_server` -The identity server which we suggest that clients should use when users log -in on this server. +*(string|null)* The identity server which we suggest that clients should use when users log in on this server. -(By default, no suggestion is made, so it is left up to the client. -This setting is ignored unless `public_baseurl` is also explicitly set.) +(By default, no suggestion is made, so it is left up to the client. This setting is ignored unless `public_baseurl` is also explicitly set.) + +Defaults to `null`. Example configuration: ```yaml @@ -2687,39 +2575,37 @@ default_identity_server: https://matrix.org --- ### `account_threepid_delegates` -Delegate verification of phone numbers to an identity server. +*(object)* Delegate verification of phone numbers to an identity server. -When a user wishes to add a phone number to their account, we need to verify that they -actually own that phone number, which requires sending them a text message (SMS). -Currently Synapse does not support sending those texts itself and instead delegates the -task to an identity server. The base URI for the identity server to be used is -specified by the `account_threepid_delegates.msisdn` option. +When a user wishes to add a phone number to their account, we need to verify that they actually own that phone number, which requires sending them a text message (SMS). Currently Synapse does not support sending those texts itself and instead delegates the task to an identity server. The base URI for the identity server to be used is specified by the `account_threepid_delegates.msisdn` option. -If this is left unspecified, Synapse will not allow users to add phone numbers to -their account. +If this is left unspecified, Synapse will not allow users to add phone numbers to their account. -(Servers handling the these requests must answer the `/requestToken` endpoints defined -by the Matrix Identity Service API -[specification](https://matrix.org/docs/spec/identity_service/latest).) +(Servers handling the these requests must answer the `/requestToken` endpoints defined by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest).) *Deprecated in Synapse 1.64.0*: The `email` option is deprecated. -*Removed in Synapse 1.66.0*: The `email` option has been removed. -If present, Synapse will report a configuration error on startup. +*Removed in Synapse 1.66.0*: The `email` option has been removed. If present, Synapse will report a configuration error on startup. + +Defaults to `{}`. + +This setting has the following sub-options: + +* `msisdn` (string|null): Identity server base URI for MSISDN (phone numbers). See above. Example configuration: ```yaml account_threepid_delegates: - msisdn: http://localhost:8090 # Delegate SMS sending to this local process + msisdn: http://localhost:8090 ``` --- ### `enable_set_displayname` -Whether users are allowed to change their displayname after it has -been initially set. Useful when provisioning users based on the -contents of a third-party directory. +*(boolean)* Whether users are allowed to change their displayname after it has been initially set. Useful when provisioning users based on the contents of a third-party directory. -Does not apply to server administrators. Defaults to true. +Does not apply to server administrators. + +Defaults to `true`. Example configuration: ```yaml @@ -2728,11 +2614,11 @@ enable_set_displayname: false --- ### `enable_set_avatar_url` -Whether users are allowed to change their avatar after it has been -initially set. Useful when provisioning users based on the contents -of a third-party directory. +*(boolean)* Whether users are allowed to change their avatar after it has been initially set. Useful when provisioning users based on the contents of a third-party directory. -Does not apply to server administrators. Defaults to true. +Does not apply to server administrators. + +Defaults to `true`. Example configuration: ```yaml @@ -2741,10 +2627,7 @@ enable_set_avatar_url: false --- ### `enable_3pid_changes` -Whether users can change the third-party IDs associated with their accounts -(email address and msisdn). - -Defaults to true. +*(boolean)* Whether users can change the third-party IDs associated with their accounts (email address and msisdn). Defaults to `true`. Example configuration: ```yaml @@ -2753,39 +2636,30 @@ enable_3pid_changes: false --- ### `auto_join_rooms` -Users who register on this homeserver will automatically be joined -to the rooms listed under this option. +*(array)* Users who register on this homeserver will automatically be joined to the rooms listed under this option. -By default, any room aliases included in this list will be created -as a publicly joinable room when the first user registers for the -homeserver. If the room already exists, make certain it is a publicly joinable -room, i.e. the join rule of the room must be set to 'public'. You can find more options -relating to auto-joining rooms below. +By default, any room aliases included in this list will be created as a publicly joinable room when the first user registers for the homeserver. If the room already exists, make certain it is a publicly joinable room, i.e. the join rule of the room must be set to `public`. You can find more options relating to auto-joining rooms below. -As Spaces are just rooms under the hood, Space aliases may also be -used. +As Spaces are just rooms under the hood, Space aliases may also be used. + +Defaults to `[]`. Example configuration: ```yaml auto_join_rooms: - - "#exampleroom:example.com" - - "#anotherexampleroom:example.com" +- '#exampleroom:example.com' +- '#anotherexampleroom:example.com' ``` --- ### `autocreate_auto_join_rooms` -Where `auto_join_rooms` are specified, setting this flag ensures that -the rooms exist by creating them when the first user on the -homeserver registers. This option will not create Spaces. +*(boolean)* Where `auto_join_rooms` are specified, setting this flag ensures that the rooms exist by creating them when the first user on the homeserver registers. This option will not create Spaces. -By default the auto-created rooms are publicly joinable from any federated -server. Use the `autocreate_auto_join_rooms_federated` and -`autocreate_auto_join_room_preset` settings to customise this behaviour. +By default the auto-created rooms are publicly joinable from any federated server. Use the `autocreate_auto_join_rooms_federated` and `autocreate_auto_join_room_preset` settings to customise this behaviour. -Setting to false means that if the rooms are not manually created, -users cannot be auto-joined since they do not exist. +Setting to false means that if the rooms are not manually created, users cannot be auto-joined since they do not exist. -Defaults to true. +Defaults to `true`. Example configuration: ```yaml @@ -2794,15 +2668,13 @@ autocreate_auto_join_rooms: false --- ### `autocreate_auto_join_rooms_federated` -Whether the rooms listed in `auto_join_rooms` that are auto-created are available -via federation. Only has an effect if `autocreate_auto_join_rooms` is true. +*(boolean)* Whether the rooms listed in `auto_join_rooms` that are auto-created are available via federation. Only has an effect if `autocreate_auto_join_rooms` is true. -Note that whether a room is federated cannot be modified after -creation. +Note that whether a room is federated cannot be modified after creation. -Defaults to true: the room will be joinable from other servers. -Set to false to prevent users from other homeservers from -joining these rooms. +If true, the room will be joinable from other servers. If false, users from other homeservers are prevented from joining these rooms. + +Defaults to `true`. Example configuration: ```yaml @@ -2811,25 +2683,18 @@ autocreate_auto_join_rooms_federated: false --- ### `autocreate_auto_join_room_preset` -The room preset to use when auto-creating one of `auto_join_rooms`. Only has an -effect if `autocreate_auto_join_rooms` is true. +*(string)* The room preset to use when auto-creating one of `auto_join_rooms`. Only has an effect if `autocreate_auto_join_rooms` is true. Possible values for this option are: -* "public_chat": the room is joinable by anyone, including - federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "public_chat": the room is joinable by anyone, including federated servers if `autocreate_auto_join_rooms_federated` is true (the default). * "private_chat": an invitation is required to join these rooms. -* "trusted_private_chat": an invitation is required to join this room and the invitee is - assigned a power level of 100 upon joining the room. +* "trusted_private_chat": an invitation is required to join this room and the invitee is assigned a power level of 100 upon joining the room. -Each preset will set up a room in the same manner as if it were provided as the `preset` parameter when -calling the -[`POST /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) -Client-Server API endpoint. +Each preset will set up a room in the same manner as if it were provided as the `preset` parameter when calling the [`POST /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) Client-Server API endpoint. -If a value of "private_chat" or "trusted_private_chat" is used then -`auto_join_mxid_localpart` must also be configured. +If a value of "private_chat" or "trusted_private_chat" is used then `auto_join_mxid_localpart` must also be configured. -Defaults to "public_chat". +Defaults to `"public_chat"`. Example configuration: ```yaml @@ -2838,22 +2703,17 @@ autocreate_auto_join_room_preset: private_chat --- ### `auto_join_mxid_localpart` -The local part of the user id which is used to create `auto_join_rooms` if -`autocreate_auto_join_rooms` is true. If this is not provided then the -initial user account that registers will be used to create the rooms. +*(string|null)* The local part of the user id which is used to create `auto_join_rooms` if `autocreate_auto_join_rooms` is true. If this is not provided then the initial user account that registers will be used to create the rooms. -The user id is also used to invite new users to any auto-join rooms which -are set to invite-only. +The user id is also used to invite new users to any auto-join rooms which are set to invite-only. -It *must* be configured if `autocreate_auto_join_room_preset` is set to -"private_chat" or "trusted_private_chat". +It *must* be configured if `autocreate_auto_join_room_preset` is set to "private_chat" or "trusted_private_chat". -Note that this must be specified in order for new users to be correctly -invited to any auto-join rooms which have been set to invite-only (either -at the time of creation or subsequently). +Note that this must be specified in order for new users to be correctly invited to any auto-join rooms which have been set to invite-only (either at the time of creation or subsequently). -Note that, if the room already exists, this user must be joined and -have the appropriate permissions to invite new members. +Note that, if the room already exists, this user must be joined and have the appropriate permissions to invite new members. + +Defaults to `null`. Example configuration: ```yaml @@ -2862,10 +2722,7 @@ auto_join_mxid_localpart: system --- ### `auto_join_rooms_for_guests` -When `auto_join_rooms` is specified, setting this flag to false prevents -guest accounts from being automatically joined to the rooms. - -Defaults to true. +*(boolean)* When `auto_join_rooms` is specified, setting this flag to false prevents guest accounts from being automatically joined to the rooms. Defaults to `true`. Example configuration: ```yaml @@ -2874,13 +2731,7 @@ auto_join_rooms_for_guests: false --- ### `inhibit_user_in_use_error` -Whether to inhibit errors raised when registering a new account if the user ID -already exists. If turned on, requests to `/register/available` will always -show a user ID as available, and Synapse won't raise an error when starting -a registration with a user ID that already exists. However, Synapse will still -raise an error if the registration completes and the username conflicts. - -Defaults to false. +*(boolean)* Whether to inhibit errors raised when registering a new account if the user ID already exists. If turned on, requests to `/register/available` will always show a user ID as available, and Synapse won't raise an error when starting a registration with a user ID that already exists. However, Synapse will still raise an error if the registration completes and the username conflicts. Defaults to `false`. Example configuration: ```yaml @@ -2889,30 +2740,27 @@ inhibit_user_in_use_error: true --- ### `allow_underscore_prefixed_registration` -Whether users are allowed to register with a underscore-prefixed localpart. -By default, AppServices use prefixes like `_example` to namespace their -associated ghost users. If turned on, this may result in clashes or confusion. -Useful when provisioning users from an external identity provider. - -Defaults to false. +*(boolean)* Whether users are allowed to register with a underscore-prefixed localpart. By default, AppServices use prefixes like `_example` to namespace their associated ghost users. If turned on, this may result in clashes or confusion. Useful when provisioning users from an external identity provider. Defaults to `false`. Example configuration: ```yaml -allow_underscore_prefixed_registration: false +allow_underscore_prefixed_registration: true ``` --- ## User session management + +Config options related to user session management. + --- ### `session_lifetime` -Time that a user's session remains valid for, after they log in. +*(duration)* Time that a user's session remains valid for, after they log in. Note that this is not currently compatible with guest logins. -Note also that this is calculated at login time: changes are not applied retrospectively to users who have already -logged in. +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already logged in. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2921,16 +2769,15 @@ session_lifetime: 24h --- ### `refreshable_access_token_lifetime` -Time that an access token remains valid for, if the session is using refresh tokens. +*(duration)* Time that an access token remains valid for, if the session is using refresh tokens. For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). Note that this only applies to clients which advertise support for refresh tokens. -Note also that this is calculated at login time and refresh time: changes are not applied to -existing sessions until they are refreshed. +Note also that this is calculated at login time and refresh time: changes are not applied to existing sessions until they are refreshed. -By default, this is 5 minutes. +Defaults to `"5m"`. Example configuration: ```yaml @@ -2939,15 +2786,11 @@ refreshable_access_token_lifetime: 10m --- ### `refresh_token_lifetime` -Time that a refresh token remains valid for (provided that it is not -exchanged for another one first). -This option can be used to automatically log-out inactive sessions. -Please see the manual for more information. +*(duration)* Time that a refresh token remains valid for (provided that it is not exchanged for another one first). This option can be used to automatically log-out inactive sessions. Please see the manual for more information. -Note also that this is calculated at login time and refresh time: -changes are not applied to existing sessions until they are refreshed. +Note also that this is calculated at login time and refresh time: changes are not applied to existing sessions until they are refreshed. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2956,17 +2799,13 @@ refresh_token_lifetime: 24h --- ### `nonrefreshable_access_token_lifetime` -Time that an access token remains valid for, if the session is NOT -using refresh tokens. +*(duration)* Time that an access token remains valid for, if the session is NOT using refresh tokens. -Please note that not all clients support refresh tokens, so setting -this to a short value may be inconvenient for some users who will -then be logged out frequently. +Please note that not all clients support refresh tokens, so setting this to a short value may be inconvenient for some users who will then be logged out frequently. -Note also that this is calculated at login time: changes are not applied -retrospectively to existing sessions for users that have already logged in. +Note also that this is calculated at login time: changes are not applied retrospectively to existing sessions for users that have already logged in. -By default, this is infinite. +Defaults to `"infinity"`. Example configuration: ```yaml @@ -2977,54 +2816,50 @@ nonrefreshable_access_token_lifetime: 24h The amount of time to allow a user-interactive authentication session to be active. -This defaults to 0, meaning the user is queried for their credentials -before every action, but this can be overridden to allow a single -validation to be re-used. This weakens the protections afforded by -the user-interactive authentication process, by allowing for multiple -(and potentially different) operations to use the same validation session. +This defaults to 0, meaning the user is queried for their credentials before every action, but this can be overridden to allow a single validation to be re-used. This weakens the protections afforded by the user-interactive authentication process, by allowing for multiple (and potentially different) operations to use the same validation session. -This is ignored for potentially "dangerous" operations (including -deactivating an account, modifying an account password, adding a 3PID, -and minting additional login tokens). +This is ignored for potentially "dangerous" operations (including deactivating an account, modifying an account password, adding a 3PID, and minting additional login tokens). Use the `session_timeout` sub-option here to change the time allowed for credential validation. +Defaults to `0`. + Example configuration: ```yaml ui_auth: - session_timeout: "15s" + session_timeout: 15s ``` --- ### `login_via_existing_session` -Matrix supports the ability of an existing session to mint a login token for -another client. +*(object)* Matrix supports the ability of an existing session to mint a login token for another client. -Synapse disables this by default as it has security ramifications -- a malicious -client could use the mechanism to spawn more than one session. +Synapse disables this by default as it has security ramifications – a malicious client could use the mechanism to spawn more than one session. -The duration of time the generated token is valid for can be configured with the -`token_timeout` sub-option. +This setting has the following sub-options: -User-interactive authentication is required when this is enabled unless the -`require_ui_auth` sub-option is set to `False`. +* `enabled` (boolean): Enable login via existing session. Defaults to `false`. + +* `require_ui_auth` (boolean): Require user-interactive authentication. Defaults to `true`. + +* `token_timeout` (duration): Duration of time the generated token is valid. Defaults to `"5m"`. Example configuration: ```yaml login_via_existing_session: - enabled: true - require_ui_auth: false - token_timeout: "5m" + enabled: true + require_ui_auth: false + token_timeout: 5m ``` --- ## Metrics + Config options related to metrics. --- ### `enable_metrics` -Set to true to enable collection and rendering of performance metrics. -Defaults to false. +*(boolean)* Set to true to enable collection and rendering of performance metrics. Defaults to `false`. Example configuration: ```yaml @@ -3033,51 +2868,46 @@ enable_metrics: true --- ### `sentry` -Use this option to enable sentry integration. Provide the DSN assigned to you by sentry -with the `dsn` setting. +*(object)* Use this option to enable sentry integration. Provide the DSN assigned to you by sentry with the `dsn` setting. - An optional `environment` field can be used to specify an environment. This allows - for log maintenance based on different environments, ensuring better organization - and analysis.. +An optional `environment` field can be used to specify an environment. This allows for log maintenance based on different environments, ensuring better organization and analysis. -NOTE: While attempts are made to ensure that the logs don't contain -any sensitive information, this cannot be guaranteed. By enabling -this option the sentry server may therefore receive sensitive -information, and it in turn may then disseminate sensitive information -through insecure notification channels if so configured. +NOTE: While attempts are made to ensure that the logs don't contain any sensitive information, this cannot be guaranteed. By enabling this option the sentry server may therefore receive sensitive information, and it in turn may then disseminate sensitive information through insecure notification channels if so configured. + +This setting has the following sub-options: + +* `dsn` (string|null): The DSN assigned by sentry. If unset or null, sentry integration is disabled. Defaults to `null`. + +* `environment` (string|null): Sentry environment. Defaults to `null`. Example configuration: ```yaml sentry: - environment: "production" - dsn: "..." + environment: production + dsn: '...' ``` --- ### `metrics_flags` -Flags to enable Prometheus metrics which are not suitable to be -enabled by default, either for performance reasons or limited use. -Currently the only option is `known_servers`, which publishes -`synapse_federation_known_servers`, a gauge of the number of -servers this homeserver knows about, including itself. May cause -performance problems on large homeservers. +*(object)* Flags to enable Prometheus metrics which are not suitable to be enabled by default, either for performance reasons or limited use. Currently the only option is `known_servers`. + +This setting has the following sub-options: + +* `known_servers` (boolean): Publishes `synapse_federation_known_servers`, a gauge of the number of servers this homeserver knows about, including itself. May cause performance problems on large homeservers. Defaults to `false`. Example configuration: ```yaml metrics_flags: - known_servers: true + known_servers: true ``` --- ### `report_stats` -Whether or not to report homeserver usage statistics. This is originally -set when generating the config. Set this option to true or false to change the current -behavior. See -[Reporting Homeserver Usage Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) -for information on what data is reported. +*(boolean)* Whether or not to report homeserver usage statistics. This is originally set when generating the config. Set this option to true or false to change the current behavior. See [Reporting Homeserver Usage Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) for information on what data is reported. -Statistics will be reported 5 minutes after Synapse starts, and then every 3 hours -after that. +Statistics will be reported 5 minutes after Synapse starts, and then every 3 hours after that. + +Defaults to `false`. Example configuration: ```yaml @@ -3086,8 +2916,7 @@ report_stats: true --- ### `report_stats_endpoint` -The endpoint to report homeserver usage statistics to. -Defaults to https://matrix.org/report-usage-stats/push +*(string)* The endpoint to report homeserver usage statistics to. Defaults to `"https://matrix.org/report-usage-stats/push"`. Example configuration: ```yaml @@ -3095,14 +2924,13 @@ report_stats_endpoint: https://example.com/report-usage-stats/push ``` --- ## API Configuration -Config settings related to the client/server API + +Config settings related to the client/server API. --- ### `room_prejoin_state` -This setting controls the state that is shared with users upon receiving an -invite to a room, or in reply to a knock on a room. By default, the following -state events are shared with users: +*(object)* This setting controls the state that is shared with users upon receiving an invite to a room, or in reply to a knock on a room. By default, the following state events are shared with users: - `m.room.join_rules` - `m.room.canonical_alias` @@ -3112,56 +2940,43 @@ state events are shared with users: - `m.room.create` - `m.room.topic` -To change the default behavior, use the following sub-options: -* `disable_default_event_types`: boolean. Set to `true` to disable the above - defaults. If this is enabled, only the event types listed in - `additional_event_types` are shared. Defaults to `false`. -* `additional_event_types`: A list of additional state events to include in the - events to be shared. By default, this list is empty (so only the default event - types are shared). +*Changed in Synapse 1.74:* admins can filter the events in prejoin state based on their state key. - Each entry in this list should be either a single string or a list of two - strings. - * A standalone string `t` represents all events with type `t` (i.e. - with no restrictions on state keys). - * A pair of strings `[t, s]` represents a single event with type `t` and - state key `s`. The same type can appear in two entries with different state - keys: in this situation, both state keys are included in prejoin state. +This setting has the following sub-options: + +* `disable_default_event_types` (boolean): Set to `true` to disable the above defaults. If this is enabled, only the event types listed in `additional_event_types` are shared. Defaults to `false`. + +* `additional_event_types` (array): A list of additional state events to include in the events to be shared. By default, this list is empty (so only the default event types are shared). + + Each entry in this list should be either a single string or a list of two strings. + * A standalone string `t` represents all events with type `t` (i.e. with no restrictions on state keys). + * A pair of strings `[t, s]` represents a single event with type `t` and state key `s`. The same type can appear in two entries with different state keys: in this situation, both state keys are included in prejoin state. + + Defaults to `[]`. Example configuration: ```yaml room_prejoin_state: - disable_default_event_types: false - additional_event_types: - # Share all events of type `org.example.custom.event.typeA` - - org.example.custom.event.typeA - # Share only events of type `org.example.custom.event.typeB` whose - # state_key is "foo" - - ["org.example.custom.event.typeB", "foo"] - # Share only events of type `org.example.custom.event.typeC` whose - # state_key is "bar" or "baz" - - ["org.example.custom.event.typeC", "bar"] - - ["org.example.custom.event.typeC", "baz"] + disable_default_event_types: false + additional_event_types: + - org.example.custom.event.typeA + - - org.example.custom.event.typeB + - foo + - - org.example.custom.event.typeC + - bar + - - org.example.custom.event.typeC + - baz ``` - -*Changed in Synapse 1.74:* admins can filter the events in prejoin state based -on their state key. - --- ### `track_puppeted_user_ips` -We record the IP address of clients used to access the API for various -reasons, including displaying it to the user in the "Where you're signed in" -dialog. +*(boolean)* We record the IP address of clients used to access the API for various reasons, including displaying it to the user in the "Where you're signed in" dialog. -By default, when puppeting another user via the admin API, the client IP -address is recorded against the user who created the access token (ie, the -admin user), and *not* the puppeted user. +By default, when puppeting another user via the admin API, the client IP address is recorded against the user who created the access token (ie, the admin user), and *not* the puppeted user. -Set this option to true to also record the IP address against the puppeted -user. (This also means that the puppeted user will count as an "active" user -for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc -above.) +Set this option to true to also record the IP address against the puppeted user. (This also means that the puppeted user will count as an "active" user for the purpose of monthly active user tracking – see `limit_usage_by_mau` etc above.) + +Defaults to `false`. Example configuration: ```yaml @@ -3170,19 +2985,18 @@ track_puppeted_user_ips: true --- ### `app_service_config_files` -A list of application service config files to use. +*(array)* A list of application service config files to use. Defaults to `[]`. Example configuration: ```yaml app_service_config_files: - - app_service_1.yaml - - app_service_2.yaml +- app_service_1.yaml +- app_service_2.yaml ``` --- ### `track_appservice_user_ips` -Defaults to false. Set to true to enable tracking of application service IP addresses. -Implicitly enables MAU tracking for application service users. +*(boolean)* Set to true to enable tracking of application service IP addresses. Implicitly enables MAU tracking for application service users. Defaults to `false`. Example configuration: ```yaml @@ -3191,32 +3005,29 @@ track_appservice_user_ips: true --- ### `use_appservice_legacy_authorization` -Whether to send the application service access tokens via the `access_token` query parameter -per older versions of the Matrix specification. Defaults to false. Set to true to enable sending -access tokens via a query parameter. +*(boolean)* Whether to send the application service access tokens via the `access_token` query parameter per older versions of the Matrix specification. Defaults to false. Set to true to enable sending access tokens via a query parameter. -**Enabling this option is considered insecure and is not recommended. ** +**Enabling this option is considered insecure and is not recommended.** + +Defaults to `false`. Example configuration: ```yaml use_appservice_legacy_authorization: true ``` - --- ### `macaroon_secret_key` -A secret which is used to sign +*(string|null)* A secret which is used to sign - access token for guest users, - short-term login token used during SSO logins (OIDC or SAML2) and - token used for unsubscribing from email notifications. -If none is specified, the `registration_shared_secret` is used, if one is given; -otherwise, a secret key is derived from the signing key. +If none is specified, the `registration_shared_secret` is used, if one is given; otherwise, a secret key is derived from the signing key. -> ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new one -> will lead to invalidation of access tokens for all guest users. It will also -> break unsubscribe links in emails sent before the change. An unlucky user -> might encounter a broken SSO login flow and would have to start again. +> ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new one will lead to invalidation of access tokens for all guest users. It will also break unsubscribe links in emails sent before the change. An unlucky user might encounter a broken SSO login flow and would have to start again. + +Defaults to `null`. Example configuration: ```yaml @@ -3225,28 +3036,26 @@ macaroon_secret_key: --- ### `macaroon_secret_key_path` -An alternative to [`macaroon_secret_key`](#macaroon_secret_key): -allows the secret key to be specified in an external file. +*(string|null)* An alternative to [`macaroon_secret_key`](#macaroon_secret_key): allows the secret key to be specified in an external file. -The file should be a plain text file, containing only the secret key. -Synapse reads the secret key from the given file once at startup. +The file should be a plain text file, containing only the secret key. Synapse reads the secret key from the given file once at startup. + +_Added in Synapse 1.121.0._ + +Defaults to `null`. Example configuration: ```yaml macaroon_secret_key_path: /path/to/secrets/file ``` - -_Added in Synapse 1.121.0._ - --- ### `form_secret` -A secret which is used to calculate HMACs for form values, to stop -falsification of values. Must be specified for the User Consent -forms to work. +*(string|null)* A secret which is used to calculate HMACs for form values, to stop falsification of values. Must be specified for the User Consent forms to work. -Replacing an existing `form_secret` with a new one might break the user consent -page for an unlucky user and require them to reopen the page from a new link. +Replacing an existing `form_secret` with a new one might break the user consent page for an unlucky user and require them to reopen the page from a new link. + +Defaults to `null`. Example configuration: ```yaml @@ -3255,67 +3064,63 @@ form_secret: --- ### `form_secret_path` -An alternative to [`form_secret`](#form_secret): -allows the secret to be specified in an external file. +*(string|null)* An alternative to [`form_secret`](#form_secret): allows the secret to be specified in an external file. -The file should be a plain text file, containing only the secret. -Synapse reads the secret from the given file once at startup. +The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. + +_Added in Synapse 1.126.0._ + +Defaults to `null`. Example configuration: ```yaml form_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.126.0._ - --- ## Signing Keys -Config options relating to signing keys + +Config options relating to signing keys. --- ### `signing_key_path` -Path to the signing key to sign events and federation requests with. +*(string|null)* Path to the signing key to sign events and federation requests with. -*New in Synapse 1.67*: If this file does not exist, Synapse will create a new signing -key on startup and store it in this file. +*New in Synapse 1.67*: If this file does not exist, Synapse will create a new signing key on startup and store it in this file. + +Defaults to `null`. Example configuration: ```yaml -signing_key_path: "CONFDIR/SERVERNAME.signing.key" +signing_key_path: CONFDIR/SERVERNAME.signing.key ``` --- ### `old_signing_keys` -The keys that the server used to sign messages with but won't use -to sign new messages. For each key, `key` should be the base64-encoded public key, and -`expired_ts`should be the time (in milliseconds since the unix epoch) that -it was last used. +*(object)* The keys that the server used to sign messages with but won't use to sign new messages. -It is possible to build an entry from an old `signing.key` file using the -`export_signing_key` script which is provided with synapse. +It is possible to build an entry from an old `signing.key` file using the `export_signing_key` script which is provided with synapse. -If you have lost the private key file, you can ask another server you trust to -tell you the public keys it has seen from your server. To fetch the keys from -`matrix.org`, try something like: +If you have lost the private key file, you can ask another server you trust to tell you the public keys it has seen from your server. To fetch the keys from `matrix.org`, try something like: ``` curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com | jq '.server_keys | map(.verify_keys) | add' ``` +Defaults to `{}`. + Example configuration: ```yaml old_signing_keys: - "ed25519:id": { key: "base64string", expired_ts: 123456789123 } + ed25519:id: + key: base64string + expired_ts: 123456789123 ``` --- ### `key_refresh_interval` -How long key response published by this server is valid for. -Used to set the `valid_until_ts` in `/key/v2` APIs. -Determines how quickly servers will query to check which keys -are still valid. Defaults to 1d. +*(duration)* How long key response published by this server is valid for. Used to set the `valid_until_ts` in `/key/v2` APIs. Determines how quickly servers will query to check which keys are still valid. Defaults to `"1d"`. Example configuration: ```yaml @@ -3324,57 +3129,41 @@ key_refresh_interval: 2d --- ### `trusted_key_servers` -The trusted servers to download signing keys from. +*(array)* The trusted servers to download signing keys from. When we need to fetch a signing key, each server is tried in parallel. -Normally, the connection to the key server is validated via TLS certificates. -Additional security can be provided by configuring a `verify key`, which -will make synapse check that the response is signed by that key. +Normally, the connection to the key server is validated via TLS certificates. Additional security can be provided by configuring a `verify key`, which will make synapse check that the response is signed by that key. -This setting supersedes an older setting named `perspectives`. The old format -is still supported for backwards-compatibility, but it is deprecated. +This setting supersedes an older setting named `perspectives`. The old format is still supported for backwards-compatibility, but it is deprecated. -`trusted_key_servers` defaults to matrix.org, but using it will generate a -warning on start-up. To suppress this warning, set -`suppress_key_server_warning` to true. +`trusted_key_servers` defaults to matrix.org, but using it will generate a warning on start-up. To suppress this warning, set `suppress_key_server_warning` to true. -If the use of a trusted key server has to be deactivated, e.g. in a private -federation or for privacy reasons, this can be realised by setting -an empty array (`trusted_key_servers: []`). Then Synapse will request the keys -directly from the server that owns the keys. If Synapse does not get keys directly -from the server, the events of this server will be rejected. +If the use of a trusted key server has to be deactivated, e.g. in a private federation or for privacy reasons, this can be realised by setting an empty array (`trusted_key_servers: []`). Then Synapse will request the keys directly from the server that owns the keys. If Synapse does not get keys directly from the server, the events of this server will be rejected. -Options for each entry in the list include: -* `server_name`: the name of the server. Required. -* `verify_keys`: an optional map from key id to base64-encoded public key. - If specified, we will check that the response is signed by at least - one of the given keys. -* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, - and `federation_verify_certificates` is not `true`, synapse will refuse - to start, because this would allow anyone who can spoof DNS responses - to masquerade as the trusted key server. If you know what you are doing - and are sure that your network environment provides a secure connection - to the key server, you can set this to `true` to override this behaviour. - -Example configuration #1: +Default configuration: ```yaml trusted_key_servers: - - server_name: "my_trusted_server.example.com" - verify_keys: - "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" - - server_name: "my_other_trusted_server.example.com" +- server_name: matrix.org ``` -Example configuration #2: + +Example configurations: ```yaml trusted_key_servers: - - server_name: "matrix.org" +- server_name: my_trusted_server.example.com + verify_keys: + ed25519:auto: abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr +- server_name: my_other_trusted_server.example.com +``` + +```yaml +trusted_key_servers: +- server_name: matrix.org ``` --- ### `suppress_key_server_warning` -Set the following to true to disable the warning that is emitted when the -`trusted_key_servers` include 'matrix.org'. See above. +*(boolean)* Set the following to true to disable the warning that is emitted when the `trusted_key_servers` include "matrix.org". See above. Defaults to `false`. Example configuration: ```yaml @@ -3383,677 +3172,509 @@ suppress_key_server_warning: true --- ### `key_server_signing_keys_path` -The signing keys to use when acting as a trusted key server. If not specified -defaults to the server signing key. +*(string|null)* The signing keys to use when acting as a trusted key server. If not specified defaults to the server signing key. Can contain multiple keys, one per line. +Defaults to `null`. + Example configuration: ```yaml -key_server_signing_keys_path: "key_server_signing_keys.key" +key_server_signing_keys_path: key_server_signing_keys.key ``` --- ## Single sign-on integration -The following settings can be used to make Synapse use a single sign-on -provider for authentication, instead of its internal password database. +The following settings can be used to make Synapse use a single sign-on provider for authentication, instead of its internal password database. -You will probably also want to set the following options to `false` to -disable the regular login/registration flows: - * [`enable_registration`](#enable_registration) - * [`password_config.enabled`](#password_config) +You will probably also want to set the following options to `false` to disable the regular login/registration flows: +* [`enable_registration`](#enable_registration) +* [`password_config.enabled`](#password_config) --- ### `saml2_config` -Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and -to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). +*(object)* Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to enable SAML login. You can either put your entire pysaml config inline using the `sp_config` option, or you can specify a path to a psyaml config file with the sub-option `config_path`. + +Once SAML support is enabled, a metadata file will be exposed at `https://:/_synapse/client/saml2/metadata.xml`, which you may be able to use to configure your SAML IdP with. Alternatively, you can manually configure the IdP to use an ACS location of `https://:/_synapse/client/saml2/authn_response`. -At least one of `sp_config` or `config_path` must be set in this section to -enable SAML login. You can either put your entire pysaml config inline using the `sp_config` -option, or you can specify a path to a psyaml config file with the sub-option `config_path`. This setting has the following sub-options: -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. -* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. - Default values will be used for the `entityid` and `service` settings, - so it is not normally necessary to specify them unless you need to - override them. Here are a few useful sub-options for configuring pysaml: - * `metadata`: Point this to the IdP's metadata. You must provide either a local - file via the `local` attribute or (preferably) a URL via the - `remote` attribute. - * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. - Defaults to 0. - * `service`: By default, the user has to go to our login page first. If you'd like - to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` - section. -* `config_path`: specify a separate pysaml2 configuration file thusly: - `config_path: "CONFDIR/sp_conf.py"` -* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to - complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. -* `user_mapping_provider`: Using this option, an external module can be provided as a - custom solution to mapping attributes returned from a saml provider onto a matrix user. The - `user_mapping_provider` has the following attributes: - * `module`: The custom module's class. - * `config`: Custom configuration values for the module. Use the values provided in the - example if you are using the built-in user_mapping_provider, or provide your own - config values for a custom class if you are using one. This section will be passed as a Python - dictionary to the module's `parse_config` method. The built-in provider takes the following two - options: - * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use - to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the - `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. - * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a - matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') - and `dotreplace` (which replaces unpermitted characters with '.'). - The default is `hexencode`. Note: This used to be configured by the - `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. -* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to - MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` - matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this - backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. - The default is 'uid'. -* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes - match particular values. The requirements can be listed under - `attribute_requirements` as shown in the example. All of the listed attributes must - match for the login to be permitted. Values can be specified in a `one_of` list to allow - multiple values for an attribute. -* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` - option must be set to the entity to redirect users to. - Most deployments only have a single IdP entity and so should omit this option. +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. +* `idp_icon` (string|null): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) -Once SAML support is enabled, a metadata file will be exposed at -`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to -use to configure your SAML IdP with. Alternatively, you can manually configure -the IdP to use an ACS location of -`https://:/_synapse/client/saml2/authn_response`. +* `idp_brand`: An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `sp_config` (object|null): Configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. Default values will be used for the `entityid` and `service` settings, so it is not normally necessary to specify them unless you need to override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local file via the `local` attribute or (preferably) a URL via the `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` section. Defaults to `null`. + +* `config_path` (string|null): Specify a separate pysaml2 configuration file. Defaults to `null`. + +* `saml_session_lifetime` (duration): The lifetime of a SAML session. This defines how long a user has to complete the authentication process, if `allow_unsolicited` is unset. Defaults to `"15m"`. + +* `user_mapping_provider` (object): Using this option, an external module can be provided as a custom solution to mapping attributes returned from a saml provider onto a matrix user. + + This setting has the following sub-options: + + * `module` (string): The custom module's class. + + * `config` (object): Custom configuration values for the module. Use the values provided in the example if you are using the built-in user_mapping_provider, or provide your own config values for a custom class if you are using one. This section will be passed as a Python dictionary to the module's `parse_config` method. The built-in provider takes the following two options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use to derive the Matrix ID from. It is "uid" by default. Note: This used to be configured by the `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a matrix ID. Options include: `hexencode` (which maps unpermitted characters to `=xx`) and `dotreplace` (which replaces unpermitted characters with `.`). The default is `hexencode`. Note: This used to be configured by the `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. + +* `grandfathered_mxid_source_attribute` (string): In previous versions of synapse, the mapping from SAML attribute to MXID was always calculated dynamically rather than stored in a table. For backwards-compatibility, we will look for `user_ids` matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this backwards-compatibility lookup. Typically it should be "uid", but if the attribute maps are changed, it may be necessary to change it. Defaults to `"uid"`. + +* `attribute_requirements` (array): It is possible to configure Synapse to only allow logins if SAML attributes match particular values. The requirements can be listed under `attribute_requirements` as shown in the example. All of the listed attributes must match for the login to be permitted. Values can be specified in a `one_of` list to allow multiple values for an attribute. + + Options for each entry include: + + * `attribute` (string): SAML attribute for which to allow logins. + + * `value` (string): Value the SAML attribute must match. + + * `one_of` (array): List of values the SAML attribute must all match. + +* `idp_entityid` (string|null): If the metadata XML contains multiple IdP entities then the `idp_entityid` option must be set to the entity to redirect users to. Most deployments only have a single IdP entity and so should omit this option. Defaults to `null`. Example configuration: ```yaml saml2_config: sp_config: metadata: - local: ["saml2/idp.xml"] + local: + - saml2/idp.xml remote: - - url: https://our_idp/metadata.xml + - url: https://our_idp/metadata.xml accepted_time_diff: 3 - service: sp: allow_unsolicited: true - - # The examples below are just used to generate our metadata xml, and you - # may well not need them, depending on your setup. Alternatively you - # may need a whole lot more detail - see the pysaml2 docs! - description: ["My awesome SP", "en"] - name: ["Test SP", "en"] - + description: + - My awesome SP + - en + name: + - Test SP + - en ui_info: display_name: - - lang: en - text: "Display Name is the descriptive name of your service." + - lang: en + text: Display Name is the descriptive name of your service. description: - - lang: en - text: "Description should be a short paragraph explaining the purpose of the service." + - lang: en + text: Description should be a short paragraph explaining the purpose of the + service. information_url: - - lang: en - text: "https://example.com/terms-of-service" + - lang: en + text: https://example.com/terms-of-service privacy_statement_url: - - lang: en - text: "https://example.com/privacy-policy" + - lang: en + text: https://example.com/privacy-policy keywords: - - lang: en - text: ["Matrix", "Element"] + - lang: en + text: + - Matrix + - Element logo: - - lang: en - text: "https://example.com/logo.svg" - width: "200" - height: "80" - + - lang: en + text: https://example.com/logo.svg + width: '200' + height: '80' organization: name: Example com display_name: - - ["Example co", "en"] - url: "http://example.com" - + - - Example co + - en + url: http://example.com contact_person: - - given_name: Bob - sur_name: "the Sysadmin" - email_address: ["admin@example.com"] - contact_type: technical - + - given_name: Bob + sur_name: the Sysadmin + email_address: + - admin@example.com + contact_type: technical saml_session_lifetime: 5m - user_mapping_provider: - # Below options are intended for the built-in provider, they should be - # changed if using a custom module. config: mxid_source_attribute: displayName mxid_mapping: dotreplace - grandfathered_mxid_source_attribute: upn - attribute_requirements: - - attribute: userGroup - value: "staff" - - attribute: department - one_of: - - "sales" - - "admins" - - idp_entityid: 'https://our_idp/entityid' + - attribute: userGroup + value: staff + - attribute: department + one_of: + - sales + - admins + idp_entityid: https://our_idp/entityid ``` --- ### `oidc_providers` -List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration -and login. See [here](../../openid.md) -for information on how to configure these options. +*(array)* List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration and login. See [here](../../openid.md) for information on how to configure these options. -For backwards compatibility, it is also possible to configure a single OIDC -provider via an `oidc_config` setting. This is now deprecated and admins are -advised to migrate to the `oidc_providers` format. (When doing that migration, -use `oidc` for the `idp_id` to ensure that existing users continue to be -recognised.) +For backwards compatibility, it is also possible to configure a single OIDC provider via an `oidc_config` setting. This is now deprecated and admins are advised to migrate to the `oidc_providers` format. (When doing that migration, use `oidc` for the `idp_id` to ensure that existing users continue to be recognised.) -Options for each entry include: -* `idp_id`: a unique identifier for this identity provider. Used internally - by Synapse; should be a single word such as 'github'. - Note that, if this is changed, users authenticating via that provider - will no longer be recognised as the same user! - (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) - -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. - -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) - -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. - -* `discover`: set to false to disable the use of the OIDC discovery mechanism - to discover endpoints. Defaults to true. - -* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery - is enabled) to discover the provider's endpoints. - -* `client_id`: Required. oauth2 client id to use. - -* `client_secret`: oauth2 client secret to use. May be omitted if - `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. - Must be omitted if `client_secret_path` is specified. - -* `client_secret_path`: path to the oauth2 client secret to use. With that - it's not necessary to leak secrets into the config file itself. - Mutually exclusive with `client_secret`. Can be omitted if - `client_secret_jwt_key` is specified. - - *Added in Synapse 1.91.0.* - -* `client_secret_jwt_key`: Alternative to client_secret: details of a key used - to create a JSON Web Token to be used as an OAuth2 client secret. If - given, must be a dictionary with the following properties: - - * `key`: a pem-encoded signing key. Must be a suitable key for the - algorithm specified. Required unless `key_file` is given. - - * `key_file`: the path to file containing a pem-encoded signing key file. - Required unless `key` is given. - - * `jwt_header`: a dictionary giving properties to include in the JWT - header. Must include the key `alg`, giving the algorithm used to - sign the JWT, such as "ES256", using the JWA identifiers in - RFC7518. - - * `jwt_payload`: an optional dictionary giving properties to include in - the JWT payload. Normally this should include an `iss` key. - -* `client_auth_method`: auth method to use when exchanging the token. Valid - values are `client_secret_basic` (default), `client_secret_post` and - `none`. - -* `pkce_method`: Whether to use proof key for code exchange when requesting - and exchanging the token. Valid values are: `auto`, `always`, or `never`. Defaults - to `auto`, which uses PKCE if supported during metadata discovery. Set to `always` - to force enable PKCE or `never` to force disable PKCE. - -* `id_token_signing_alg_values_supported`: List of the JWS signing algorithms (`alg` - values) that are supported for signing the `id_token`. - - This is *not* required if `discovery` is disabled. We default to supporting `RS256` in - the downstream usage if no algorithms are configured here or in the discovery - document. - - According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid - approach would be to reject this provider as non-compliant if it's not included but we - simply allow whatever and see what happens (you're the one that configured the value - and cooperating with the identity provider). - - The `alg` value `"none"` MAY be supported but can only be used if the Authorization - Endpoint does not include `id_token` in the `response_type` (ex. - `/authorize?response_type=code` where `none` can apply, - `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when - using the Authorization Code Flow). - -* `scopes`: list of scopes to request. This should normally include the "openid" - scope. Defaults to `["openid"]`. - -* `authorization_endpoint`: the oauth2 authorization endpoint. Required if - provider discovery is disabled. - -* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is - disabled. - -* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is - disabled and the 'openid' scope is not requested. - -* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and - the 'openid' scope is used. - -* `skip_verification`: set to 'true' to skip metadata verification. Use this if - you are connecting to a provider that is not OpenID Connect compliant. - Defaults to false. Avoid this in production. - -* `user_profile_method`: Whether to fetch the user profile from the userinfo - endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. - Valid values are: `auto` or `userinfo_endpoint`. - Defaults to `auto`, which uses the userinfo endpoint if `openid` is - not included in `scopes`. Set to `userinfo_endpoint` to always use the - userinfo endpoint. - -* `redirect_uri`: An optional string, that if set will override the `redirect_uri` - parameter sent in the requests to the authorization and token endpoints. - Useful if you want to redirect the client to another endpoint as part of the - OIDC login. Be aware that the client must then call Synapse's OIDC callback - URL (`/_synapse/client/oidc/callback`) manually afterwards. - Must be a valid URL including scheme and path. - -* `additional_authorization_parameters`: String to string dictionary that will be passed as - additional parameters to the authorization grant URL. - -* `passthrough_authorization_parameters`: List of parameters that will be passed through from the redirect endpoint - to the authorization grant URL. - -* `allow_existing_users`: set to true to allow a user logging in via OIDC to - match a pre-existing account instead of failing. This could be used if - switching from password logins to OIDC. Defaults to false. - -* `enable_registration`: set to 'false' to disable automatic registration of new - users. This allows the OIDC SSO flow to be limited to sign in only, rather than - automatically registering users that have a valid SSO login but do not have - a pre-registered account. Defaults to true. - -* `user_mapping_provider`: Configuration for how attributes returned from a OIDC - provider are mapped onto a matrix user. This setting has the following - sub-properties: - - * `module`: The class name of a custom mapping module. Default is - `synapse.handlers.oidc.JinjaOidcMappingProvider`. - See [OpenID Mapping Providers](../../sso_mapping_providers.md#openid-mapping-providers) - for information on implementing a custom mapping provider. - - * `config`: Configuration for the mapping provider module. This section will - be passed as a Python dictionary to the user mapping provider - module's `parse_config` method. - - For the default provider, the following settings are available: - - * `subject_template`: Jinja2 template for a unique identifier for the user. - Defaults to `{{ user.sub }}`, which OpenID Connect compliant providers should provide. - - This replaces and overrides `subject_claim`. - - * `subject_claim`: name of the claim containing a unique identifier - for the user. Defaults to 'sub', which OpenID Connect - compliant providers should provide. - - *Deprecated in Synapse v1.75.0.* - - * `picture_template`: Jinja2 template for an url for the user's profile picture. - Defaults to `{{ user.picture }}`, which OpenID Connect compliant providers should - provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. - - This replaces and overrides `picture_claim`. - - Currently only supported in monolithic (single-process) server configurations - where the media repository runs within the Synapse process. - - * `picture_claim`: name of the claim containing an url for the user's profile picture. - Defaults to 'picture', which OpenID Connect compliant providers should provide - and has to refer to a direct image file such as PNG, JPEG, or GIF image file. - - Currently only supported in monolithic (single-process) server configurations - where the media repository runs within the Synapse process. - - *Deprecated in Synapse v1.75.0.* - - * `localpart_template`: Jinja2 template for the localpart of the MXID. - If this is not set, the user will be prompted to choose their - own username (see the documentation for the `sso_auth_account_details.html` - template). This template can use the `localpart_from_email` filter. - - * `confirm_localpart`: Whether to prompt the user to validate (or - change) the generated localpart (see the documentation for the - 'sso_auth_account_details.html' template), instead of - registering the account right away. - - * `display_name_template`: Jinja2 template for the display name to set - on first login. If unset, no displayname will be set. - - * `email_template`: Jinja2 template for the email address of the user. - If unset, no email address will be added to the account. - - * `extra_attributes`: a map of Jinja2 templates for extra attributes - to send back to the client during login. Note that these are non-standard and clients will ignore them - without modifications. - - When rendering, the Jinja2 templates are given a 'user' variable, - which is set to the claims returned by the UserInfo Endpoint and/or - in the ID Token. - -* `backchannel_logout_enabled`: set to `true` to process OIDC Back-Channel Logout notifications. - Those notifications are expected to be received on `/_synapse/client/oidc/backchannel_logout`. - Defaults to `false`. - -* `backchannel_logout_ignore_sub`: by default, the OIDC Back-Channel Logout feature checks that the - `sub` claim matches the subject claim received during login. This check can be disabled by setting - this to `true`. Defaults to `false`. - - You might want to disable this if the `subject_claim` returned by the mapping provider is not `sub`. - -It is possible to configure Synapse to only allow logins if certain attributes -match particular values in the OIDC userinfo. The requirements can be listed under -`attribute_requirements` as shown here: +It is possible to configure Synapse to only allow logins if certain attributes match particular values in the OIDC userinfo. The requirements can be listed under `attribute_requirements` as shown here: ```yaml attribute_requirements: - - attribute: family_name - one_of: ["Stephensson", "Smith"] - - attribute: groups - value: "admin" - # If `value` or `one_of` are not specified, the attribute only needs - # to exist, regardless of value. - - attribute: picture + - attribute: family_name + one_of: ["Stephensson", "Smith"] + - attribute: groups + value: "admin" + # If `value` or `one_of` are not specified, the attribute only needs + # to exist, regardless of value. + - attribute: picture ``` `attribute` is a required field, while `value` and `one_of` are optional. -All of the listed attributes must match for the login to be permitted. Additional attributes can be added to -userinfo by expanding the `scopes` section of the OIDC config to retrieve -additional information from the OIDC provider. +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to userinfo by expanding the `scopes` section of the OIDC config to retrieve additional information from the OIDC provider. -If the OIDC claim is a list, then the attribute must match any value in the list. -Otherwise, it must exactly match the value of the claim. Using the example -above, the `family_name` claim MUST be either "Stephensson" or "Smith", but the `groups` -claim MUST contain "admin". +If the OIDC claim is a list, then the attribute must match any value in the list. Otherwise, it must exactly match the value of the claim. Using the example above, the `family_name` claim MUST be either "Stephensson" or "Smith", but the `groups` claim MUST contain "admin". + +Defaults to `[]`. + +Options for each entry include: + +* `idp_id` (string): A unique identifier for this identity provider. Used internally by Synapse; should be a single word such as "github". Note that, if this is changed, users authenticating via that provider will no longer be recognised as the same user! (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. + +* `idp_icon` (string): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) + +* `idp_brand` (string): An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover` (boolean): Set to false to disable the use of the OIDC discovery mechanism to discover endpoints. Defaults to true. + +* `issuer` (string): Required. The OIDC issuer. Used to validate tokens and (if discovery is enabled) to discover the provider's endpoints. + +* `client_id` (string): Required. OAuth2 client id to use. + +* `client_secret` (string|null): OAuth2 client secret to use. May be omitted if `client_secret_jwt_key` is given, or if `client_auth_method` is `none`. Must be omitted if `client_secret_path` is specified. + +* `client_secret_path` (string|null): Path to the OAuth2 client secret to use. With that it's not necessary to leak secrets into the config file itself. Mutually exclusive with `client_secret`. Can be omitted if `client_secret_jwt_key` is specified. + + *Added in Synapse 1.91.0.* + +* `client_secret_jwt_key` (object|null): Alternative to client_secret: details of a key used to create a JSON Web Token to be used as an OAuth2 client secret. + + This setting has the following sub-options: + + * `key` (string|null): A pem-encoded signing key. Must be a suitable key for the algorithm specified. Required unless `key_file` is given. + + * `key_file` (string|null): Path to the file containing a pem-encoded signing key. Required unless `key` is given. + + * `jwt_header` (object): Dictionary giving properties to include in the JWT header. Must include the key `alg`. + + This setting has the following sub-options: + + * `alg` (string): Algorithm used to sign the JWT, such as ES256, using the JWA identifiers in RFC7518. + + * `jwt_payload` (object): Optional dictionary giving properties to include in the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method` (string|null): Auth method to use when exchanging the token. Valid values are `client_secret_basic` (default), `client_secret_post` and `none`. + +* `pkce_method` (string|null): Whether to use proof key for code exchange when requesting and exchanging the token. Valid values are: `auto`, `always`, or `never`. Defaults to `auto`, which uses PKCE if supported during metadata discovery. Set to `always` to force enable PKCE or `never` to force disable PKCE. + +* `id_token_signing_alg_values_supported` (array): List of the JWS signing algorithms (`alg` values) that are supported for signing the `id_token`. + + This is *not* required if `discovery` is disabled. We default to supporting `RS256` in the downstream usage if no algorithms are configured here or in the discovery document. + + According to the spec, the algorithm `"RS256"` MUST be included. The absolute rigid approach would be to reject this provider as non-compliant if it's not included but we simply allow whatever and see what happens (you're the one that configured the value and cooperating with the identity provider). + + The `alg` value `"none"` MAY be supported but can only be used if the Authorization Endpoint does not include `id_token` in the `response_type` (ex. `/authorize?response_type=code` where `none` can apply, `/authorize?response_type=code%20id_token` where `none` can't apply) (such as when using the Authorization Code Flow). + +* `scopes` (array|null): List of scopes to request. This should normally include the "openid" scope. Defaults to `["openid"]`. + +* `authorization_endpoint` (string): The OAuth2 authorization endpoint. Required if provider discovery is disabled. + +* `token_endpoint` (string): The OAuth2 token endpoint. Required if provider discovery is disabled. + +* `userinfo_endpoint` (string): The OIDC userinfo endpoint. Required if discovery is disabled and the "openid" scope is not requested. + +* `jwks_uri` (string): URI where to fetch the JWKS. Required if discovery is disabled and the "openid" scope is used. + +* `skip_verification` (boolean): Set to `true` to skip metadata verification. Use this if you are connecting to a provider that is not OpenID Connect compliant. Defaults to false. Avoid this in production. + +* `user_profile_method` (string|null): Whether to fetch the user profile from the userinfo endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. Valid values are: `auto` or `userinfo_endpoint`. Defaults to `auto`, which uses the userinfo endpoint if `openid` is not included in `scopes`. Set to `userinfo_endpoint` to always use the userinfo endpoint. + +* `redirect_uri` (string|null): An optional string, that if set will override the `redirect_uri` parameter sent in the requests to the authorization and token endpoints. Useful if you want to redirect the client to another endpoint as part of the OIDC login. Be aware that the client must then call Synapse's OIDC callback URL (`/_synapse/client/oidc/callback`) manually afterwards. Must be a valid URL including scheme and path. + +* `additional_authorization_parameters` (object): String to string dictionary that will be passed as additional parameters to the authorization grant URL. + +* `passthrough_authorization_parameters` (array): List of parameters that will be passed through from the redirect endpoint to the authorization grant URL. + +* `allow_existing_users` (boolean): Set to true to allow a user logging in via OIDC to match a pre-existing account instead of failing. This could be used if switching from password logins to OIDC. Defaults to false. + +* `enable_registration` (boolean): Set to `false` to disable automatic registration of new users. This allows the OIDC SSO flow to be limited to sign in only, rather than automatically registering users that have a valid SSO login but do not have a pre-registered account. Defaults to true. + +* `user_mapping_provider` (object): Configuration for how attributes returned from a OIDC provider are mapped onto a matrix user. + + When rendering, the Jinja2 templates are given a `user` variable, which is set to the claims returned by the UserInfo Endpoint and/or in the ID Token. + + This setting has the following sub-options: + + * `module` (string): The class name of a custom mapping module. Default is `synapse.handlers.oidc.JinjaOidcMappingProvider`. See [OpenID Mapping Providers](../../sso_mapping_providers.md#openid-mapping-providers) for information on implementing a custom mapping provider. + + * `config` (object): Configuration for the mapping provider module. This section will be passed as a Python dictionary to the user mapping provider module's `parse_config` method. + + For the default provider, the following settings are available: + + * `subject_template`: Jinja2 template for a unique identifier for the user. Defaults to `{{ user.sub }}`, which OpenID Connect compliant providers should provide. + + This replaces and overrides `subject_claim`. + + * `subject_claim`: name of the claim containing a unique identifier for the user. Defaults to `sub`, which OpenID Connect compliant providers should provide. + + *Deprecated in Synapse v1.75.0.* + + * `picture_template`: Jinja2 template for an url for the user's profile picture. Defaults to `{{ user.picture }}`, which OpenID Connect compliant providers should provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. + + This replaces and overrides `picture_claim`. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + * `picture_claim`: name of the claim containing an url for the user's profile picture. Defaults to "picture", which OpenID Connect compliant providers should provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + *Deprecated in Synapse v1.75.0.* + + * `localpart_template`: Jinja2 template for the localpart of the MXID. If this is not set, the user will be prompted to choose their own username (see the documentation for the `sso_auth_account_details.html` template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or change) the generated localpart (see the documentation for the "sso_auth_account_details.html" template), instead of registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes to send back to the client during login. Note that these are non-standard and clients will ignore them without modifications. + +* `backchannel_logout_enabled` (boolean): Set to `true` to process OIDC Back-Channel Logout notifications. Those notifications are expected to be received on `/_synapse/client/oidc/backchannel_logout`. Defaults to `false`. + +* `backchannel_logout_ignore_sub` (boolean): By default, the OIDC Back-Channel Logout feature checks that the `sub` claim matches the subject claim received during login. This check can be disabled by setting this to `true`. Defaults to `false`. + + You might want to disable this if the `subject_claim` returned by the mapping provider is not `sub`. Example configuration: ```yaml oidc_providers: - # Generic example - # - - idp_id: my_idp - idp_name: "My OpenID provider" - idp_icon: "mxc://example.com/mediaid" - discover: false - issuer: "https://accounts.example.com/" - client_id: "provided-by-your-issuer" - client_secret: "provided-by-your-issuer" - client_auth_method: client_secret_post - scopes: ["openid", "profile"] - authorization_endpoint: "https://accounts.example.com/oauth2/auth" - token_endpoint: "https://accounts.example.com/oauth2/token" - userinfo_endpoint: "https://accounts.example.com/userinfo" - jwks_uri: "https://accounts.example.com/.well-known/jwks.json" - additional_authorization_parameters: - acr_values: 2fa - passthrough_authorization_parameters: ["login_hint"] - skip_verification: true - enable_registration: true - user_mapping_provider: - config: - subject_claim: "id" - localpart_template: "{{ user.login }}" - display_name_template: "{{ user.name }}" - email_template: "{{ user.email }}" - attribute_requirements: - - attribute: userGroup - value: "synapseUsers" +- idp_id: my_idp + idp_name: My OpenID provider + idp_icon: mxc://example.com/mediaid + discover: false + issuer: https://accounts.example.com/ + client_id: provided-by-your-issuer + client_secret: provided-by-your-issuer + client_auth_method: client_secret_post + scopes: + - openid + - profile + authorization_endpoint: https://accounts.example.com/oauth2/auth + token_endpoint: https://accounts.example.com/oauth2/token + userinfo_endpoint: https://accounts.example.com/userinfo + jwks_uri: https://accounts.example.com/.well-known/jwks.json + additional_authorization_parameters: + acr_values: 2fa + passthrough_authorization_parameters: + - login_hint + skip_verification: true + enable_registration: true + user_mapping_provider: + config: + subject_claim: id + localpart_template: '{{ user.login }}' + display_name_template: '{{ user.name }}' + email_template: '{{ user.email }}' + attribute_requirements: + - attribute: userGroup + value: synapseUsers ``` --- ### `cas_config` -Enable Central Authentication Service (CAS) for registration and login. -Has the following sub-options: -* `enabled`: Set this to true to enable authorization against a CAS server. - Defaults to false. -* `idp_name`: A user-facing name for this identity provider, which is used to - offer the user a choice of login mechanisms. -* `idp_icon`: An optional icon for this identity provider, which is presented - by clients and Synapse's own IdP picker page. If given, must be an - MXC URI of the format `mxc:///`. (An easy way to - obtain such an MXC URI is to upload an image to an (unencrypted) room - and then copy the "url" from the source of the event.) -* `idp_brand`: An optional brand for this identity provider, allowing clients - to style the login flow according to the identity provider in question. - See the [spec](https://spec.matrix.org/latest/) for possible options here. -* `server_url`: The URL of the CAS authorization endpoint. -* `protocol_version`: The CAS protocol version, defaults to none (version 3 is required if you want to use "required_attributes"). -* `displayname_attribute`: The attribute of the CAS response to use as the display name. - If no name is given here, no displayname will be set. -* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes - match particular values. All of the keys given below must exist - and the values must match the given value. Alternately if the given value - is `None` then any value is allowed (the attribute just must exist). - All of the listed attributes must match for the login to be permitted. -* `enable_registration`: set to 'false' to disable automatic registration of new - users. This allows the CAS SSO flow to be limited to sign in only, rather than - automatically registering users that have a valid SSO login but do not have - a pre-registered account. Defaults to true. -* `allow_numeric_ids`: set to 'true' allow numeric user IDs (default false). - This allows CAS SSO flow to provide user IDs composed of numbers only. - These identifiers will be prefixed by the letter "u" by default. - The prefix can be configured using the "numeric_ids_prefix" option. - Be careful to choose the prefix correctly to avoid any possible conflicts - (e.g. user 1234 becomes u1234 when a user u1234 already exists). -* `numeric_ids_prefix`: the prefix you wish to add in front of a numeric user ID - when the "allow_numeric_ids" option is set to "true". - By default, the prefix is the letter "u" and only alphanumeric characters are allowed. +*(object)* Enable Central Authentication Service (CAS) for registration and login. - *Added in Synapse 1.93.0.* +This setting has the following sub-options: + +* `enabled` (boolean): Set this to true to enable authorization against a CAS server. Defaults to `false`. + +* `idp_name` (string): A user-facing name for this identity provider, which is used to offer the user a choice of login mechanisms. + +* `idp_icon` (string|null): An optional icon for this identity provider, which is presented by clients and Synapse's own IdP picker page. If given, must be an MXC URI of the format `mxc:///`. (An easy way to obtain such an MXC URI is to upload an image to an (unencrypted) room and then copy the URL from the source of the event.) Defaults to `null`. + +* `idp_brand` (string|null): An optional brand for this identity provider, allowing clients to style the login flow according to the identity provider in question. See the [spec](https://spec.matrix.org/latest/) for possible options here. Defaults to `null`. + +* `server_url` (string): The URL of the CAS authorization endpoint. + +* `protocol_version` (integer|null): The CAS protocol version. (Version 3 is required if you want to use `required_attributes`). Defaults to `null`. + +* `displayname_attribute` (string|null): The attribute of the CAS response to use as the display name. If no name is given here, no displayname will be set. Defaults to `null`. + +* `required_attributes` (object): It is possible to configure Synapse to only allow logins if CAS attributes match particular values. All of the keys given below must exist and the values must match the given value. Alternately if the given value is `None` then any value is allowed (the attribute just must exist). All of the listed attributes must match for the login to be permitted. Defaults to `{}`. + +* `enable_registration` (boolean): Set to `false` to disable automatic registration of new users. This allows the CAS SSO flow to be limited to sign in only, rather than automatically registering users that have a valid SSO login but do not have a pre-registered account. Defaults to `true`. + +* `allow_numeric_ids` (boolean): Set to `true` allow numeric user IDs. This allows CAS SSO flow to provide user IDs composed of numbers only. These identifiers will be prefixed by the letter "u" by default. The prefix can be configured using the `numeric_ids_prefix` option. Be careful to choose the prefix correctly to avoid any possible conflicts (e.g. user 1234 becomes u1234 when a user u1234 already exists). Defaults to `false`. + +* `numeric_ids_prefix` (string): The prefix you wish to add in front of a numeric user ID when the `allow_numeric_ids` option is set to `true`. Only alphanumeric characters are allowed. + + *Added in Synapse 1.93.0.* + + Defaults to `"u"`. Example configuration: ```yaml cas_config: enabled: true - server_url: "https://cas-server.com" + server_url: https://cas-server.com protocol_version: 3 displayname_attribute: name required_attributes: - userGroup: "staff" + userGroup: staff department: None enable_registration: true allow_numeric_ids: true - numeric_ids_prefix: "numericuser" + numeric_ids_prefix: numericuser ``` --- ### `sso` -Additional settings to use with single-sign on systems such as OpenID Connect, -SAML2 and CAS. +*(object)* Additional settings to use with single-sign on systems such as OpenID Connect, SAML2 and CAS. -Server admins can configure custom templates for pages related to SSO. See -[here](../../templates.md) for more information. +Server admins can configure custom templates for pages related to SSO. See [here](../../templates.md) for more information. -Options include: -* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not - have to confirm giving access to their account to the URL. Any client - whose URL starts with an entry in the following list will not be subject - to an additional confirmation step after the SSO login is completed. - WARNING: An entry such as "https://my.client" is insecure, because it - will also match "https://my.client.evil.site", exposing your users to - phishing attacks from evil.site. To avoid this, include a slash after the - hostname: "https://my.client/". - The login fallback page (used by clients that don't natively support the - required login flows) is whitelisted in addition to any URLs in this list. - By default, this list contains only the login fallback page. -* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from - the identity provider. Currently only syncing the displayname is supported. Fields - are checked on every SSO login, and are updated if necessary. - Note that enabling this option will override user profile information, - regardless of whether users have opted-out of syncing that - information when first signing in. Defaults to false. +This setting has the following sub-options: +* `client_whitelist` (array|null): A list of client URLs which are whitelisted so that the user does not have to confirm giving access to their account to the URL. Any client whose URL starts with an entry in the following list will not be subject to an additional confirmation step after the SSO login is completed. + + WARNING: An entry such as "https://my.client" is insecure, because it will also match "https://my.client.evil.site", exposing your users to phishing attacks from evil.site. To avoid this, include a slash after the hostname: "https://my.client/". + + The login fallback page (used by clients that don't natively support the required login flows) is whitelisted in addition to any URLs in this list. By default, this list contains only the login fallback page. + + Defaults to `null`. + +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Currently only syncing the displayname is supported. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Defaults to `false`. Example configuration: ```yaml sso: - client_whitelist: - - https://riot.im/develop - - https://my.custom.client/ - update_profile_information: true + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true ``` --- ### `jwt_config` -JSON web token integration. The following settings can be used to make -Synapse JSON web tokens for authentication, instead of its internal -password database. +*(object)* JSON web token integration. The following settings can be used to make Synapse JSON web tokens for authentication, instead of its internal password database. -Each JSON Web Token needs to contain a "sub" (subject) claim, which is -used as the localpart of the mxid. +Each JSON Web Token needs to contain a "sub" (subject) claim, which is used as the localpart of the mxid. -Additionally, the expiration time ("exp"), not before time ("nbf"), -and issued at ("iat") claims are validated if present. +Additionally, the expiration time ("exp"), not before time ("nbf"), and issued at ("iat") claims are validated if present. -Note that this is a non-standard login type and client support is -expected to be non-existent. +Note that this is a non-standard login type and client support is expected to be non-existent. See [here](../../jwt.md) for more. -Additional sub-options for this setting include: -* `enabled`: Set to true to enable authorization using JSON web - tokens. Defaults to false. -* `secret`: This is either the private shared secret or the public key used to - decode the contents of the JSON web token. Required if `enabled` is set to true. -* `algorithm`: The algorithm used to sign (or HMAC) the JSON web token. - Supported algorithms are listed - [here (section JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). - Required if `enabled` is set to true. -* `subject_claim`: Name of the claim containing a unique identifier for the user. - Optional, defaults to `sub`. -* `display_name_claim`: Name of the claim containing the display name for the user. Optional. - If provided, the display name will be set to the value of this claim upon first login. -* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the - "iss" claim will be required and validated for all JSON web tokens. -* `audiences`: A list of audiences to validate the "aud" claim against. Optional. - If provided the "aud" claim will be required and validated for all JSON web tokens. - Note that if the "aud" claim is included in a JSON web token then - validation will fail without configuring audiences. +This setting has the following sub-options: + +* `enabled` (boolean): Set to true to enable authorization using JSON web tokens. Defaults to `false`. + +* `secret` (string): This is either the private shared secret or the public key used to decode the contents of the JSON web token. Required if `enabled` is set to true. + +* `algorithm` (string): The algorithm used to sign (or HMAC) the JSON web token. Supported algorithms are listed [here (section JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). Required if `enabled` is set to true. + +* `subject_claim` (string|null): Name of the claim containing a unique identifier for the user. Defaults to `"sub"`. + +* `display_name_claim` (string|null): Name of the claim containing the display name for the user. If provided, the display name will be set to the value of this claim upon first login. Defaults to `null`. + +* `issuer` (string|null): The issuer to validate the "iss" claim against. If provided the "iss" claim will be required and validated for all JSON web tokens. Defaults to `null`. + +* `audiences` (array|null): A list of audiences to validate the "aud" claim against. If provided the "aud" claim will be required and validated for all JSON web tokens. Note that if the "aud" claim is included in a JSON web token then validation will fail without configuring audiences. Defaults to `null`. Example configuration: ```yaml jwt_config: - enabled: true - secret: "provided-by-your-issuer" - algorithm: "provided-by-your-issuer" - subject_claim: "name_of_claim" - display_name_claim: "name_of_claim" - issuer: "provided-by-your-issuer" - audiences: - - "provided-by-your-issuer" + enabled: true + secret: provided-by-your-issuer + algorithm: provided-by-your-issuer + subject_claim: name_of_claim + display_name_claim: name_of_claim + issuer: provided-by-your-issuer + audiences: + - provided-by-your-issuer ``` --- ### `password_config` -Use this setting to enable password-based logins. +*(object)* Use this setting to enable password-based logins. This setting has the following sub-options: -* `enabled`: Defaults to true. - Set to false to disable password authentication. - Set to `only_for_reauth` to allow users with existing passwords to use them - to reauthenticate (not log in), whilst preventing new users from setting passwords. -* `localdb_enabled`: Set to false to disable authentication against the local password - database. This is ignored if `enabled` is false, and is only useful - if you have other `password_providers`. Defaults to true. -* `pepper`: Set the value here to a secret random string for extra security. - DO NOT CHANGE THIS AFTER INITIAL SETUP! -* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. - Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: - * `enabled`: Defaults to false. Set to true to enable. - * `minimum_length`: Minimum accepted length for a password. Defaults to 0. - * `require_digit`: Whether a password must contain at least one digit. - Defaults to false. - * `require_symbol`: Whether a password must contain at least one symbol. - A symbol is any character that's not a number or a letter. Defaults to false. - * `require_lowercase`: Whether a password must contain at least one lowercase letter. - Defaults to false. - * `require_uppercase`: Whether a password must contain at least one uppercase letter. - Defaults to false. +* `enabled` (boolean|string): Set to false to disable password authentication. Set to `only_for_reauth` to allow users with existing passwords to use them to reauthenticate (not log in), whilst preventing new users from setting passwords. Defaults to `true`. + +* `localdb_enabled` (boolean): Set to false to disable authentication against the local password database. This is ignored if `enabled` is false, and is only useful if you have other `password_providers`. Defaults to `true`. + +* `pepper` (string|null): Set the value here to a secret random string for extra security. DO NOT CHANGE THIS AFTER INITIAL SETUP! Defaults to `null`. + +* `policy` (object): Define and enforce a password policy, such as minimum lengths for passwords, etc. This is an implementation of MSC2000. + + This setting has the following sub-options: + + * `enabled` (boolean): Set to true to enable. Defaults to `false`. + + * `minimum_length` (integer): Minimum accepted length for a password. Defaults to `0`. + + * `require_digit` (boolean): Whether a password must contain at least one digit. Defaults to `false`. + + * `require_symbol` (boolean): Whether a password must contain at least one symbol. A symbol is any character that's not a number or a letter. Defaults to `false`. + + * `require_lowercase` (boolean): Whether a password must contain at least one lowercase letter. Defaults to `false`. + + * `require_uppercase` (boolean): Whether a password must contain at least one uppercase letter. Defaults to `false`. Example configuration: ```yaml password_config: - enabled: false - localdb_enabled: false - pepper: "EVEN_MORE_SECRET" - - policy: - enabled: true - minimum_length: 15 - require_digit: true - require_symbol: true - require_lowercase: true - require_uppercase: true + enabled: false + localdb_enabled: false + pepper: EVEN_MORE_SECRET + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true ``` --- ## Push -Configuration settings related to push notifications + +Configuration settings related to push notifications. --- ### `push` -This setting defines options for push notifications. +*(object)* This setting defines options for push notifications. -This option has a number of sub-options. They are as follows: -* `enabled`: Enables or disables push notification calculation. Note, disabling this will also - stop unread counts being calculated for rooms. This mode of operation is intended - for homeservers which may only have bots or appservice users connected, or are otherwise - not interested in push/unread counters. This is enabled by default. -* `include_content`: Clients requesting push notifications can either have the body of - the message sent in the notification poke along with other details - like the sender, or just the event ID and room ID (`event_id_only`). - If clients choose to have the body sent, this option controls whether the - notification request includes the content of the event (other details - like the sender are still included). If `event_id_only` is enabled, it - has no effect. - For modern android devices the notification content will still appear - because it is loaded by the app. iPhone, however will send a - notification saying only that a message arrived and who it came from. - Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. -* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. - This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the - user has unread messages in. Defaults to true, meaning push clients will see the number of - rooms with unread messages in them. Set to false to instead send the number - of unread messages. -* `jitter_delay`: Delays push notifications by a random amount up to the given - duration. Useful for mitigating timing attacks. Optional, defaults to no - delay. _Added in Synapse 1.84.0._ +This setting has the following sub-options: + +* `enabled` (boolean): Enables or disables push notification calculation. Note, disabling this will also stop unread counts being calculated for rooms. This mode of operation is intended for homeservers which may only have bots or appservice users connected, or are otherwise not interested in push/unread counters. Defaults to `true`. + +* `include_content` (boolean): Clients requesting push notifications can either have the body of the message sent in the notification poke along with other details like the sender, or just the event ID and room ID (`event_id_only`). If clients choose to have the body sent, this option controls whether the notification request includes the content of the event (other details like the sender are still included). If `event_id_only` is enabled, it has no effect. For modern Android devices the notification content will still appear because it is loaded by the app. iPhone, however will send a notification saying only that a message arrived and who it came from. Set to false to only include the event ID and room ID in push notification payloads. Defaults to `true`. + +* `group_unread_count_by_room` (boolean): When a push notification is received, an unread count is also sent. This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the user has unread messages in. If true, push clients will see the number of rooms with unread messages in them. Set to false to instead send the number of unread messages. Defaults to `true`. + +* `jitter_delay` (duration): Delays push notifications by a random amount up to the given duration. Useful for mitigating timing attacks. Optional. + + _Added in Synapse 1.84.0._ + + Defaults to `"0s"`. Example configuration: ```yaml @@ -4061,29 +3682,27 @@ push: enabled: true include_content: false group_unread_count_by_room: false - jitter_delay: "10s" + jitter_delay: 10s ``` --- ## Rooms + Config options relating to rooms. --- ### `encryption_enabled_by_default_for_room_type` -Controls whether locally-created rooms should be end-to-end encrypted by -default. +*(string)* Controls whether locally-created rooms should be end-to-end encrypted by default. Possible options are "all", "invite", and "off". They are defined as: * "all": any locally-created room -* "invite": any room created with the `private_chat` or `trusted_private_chat` - room creation presets +* "invite": any room created with the `private_chat` or `trusted_private_chat` room creation presets * "off": this option will take no effect -The default value is "off". +Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. -Note that this option will only affect rooms created after it is set. It -will also not affect rooms created by other servers. +Defaults to `"off"`. Example configuration: ```yaml @@ -4092,74 +3711,63 @@ encryption_enabled_by_default_for_room_type: invite --- ### `user_directory` -This setting defines options related to the user directory. +*(object)* This setting defines options related to the user directory. -This option has the following sub-options: -* `enabled`: Defines whether users can search the user directory. If false then - empty responses are returned to all queries. Defaults to true. -* `search_all_users`: Defines whether to search all users visible to your homeserver at the time the search is performed. - If set to true, will return all users known to the homeserver matching the search query. - If false, search results will only contain users - visible in public rooms and users sharing a room with the requester. - Defaults to false. +This setting has the following sub-options: - NB. If you set this to true, and the last time the user_directory search - indexes were (re)built was before Synapse 1.44, you'll have to - rebuild the indexes in order to search through all known users. +* `enabled` (boolean): Defines whether users can search the user directory. If false then empty responses are returned to all queries. Defaults to `true`. - These indexes are built the first time Synapse starts; admins can - manually trigger a rebuild via the API following the instructions - [for running background updates](../administration/admin_api/background_updates.md#run), - set to true to return search results containing all known users, even if that - user does not share a room with the requester. -* `prefer_local_users`: Defines whether to prefer local users in search query results. - If set to true, local users are more likely to appear above remote users when searching the - user directory. Defaults to false. -* `exclude_remote_users`: If set to true, the search will only return local users. Defaults to false. -* `show_locked_users`: Defines whether to show locked users in search query results. Defaults to false. +* `search_all_users` (boolean): Defines whether to search all users visible to your homeserver at the time the search is performed. If set to true, will return all users known to the homeserver matching the search query. If false, search results will only contain users visible in public rooms and users sharing a room with the requester. + + NB. If you set this to true, and the last time the user_directory search indexes were (re)built was before Synapse 1.44, you'll have to rebuild the indexes in order to search through all known users. + + These indexes are built the first time Synapse starts; admins can manually trigger a rebuild via the API following the instructions [for running background updates](../administration/admin_api/background_updates.md#run), set to true to return search results containing all known users, even if that user does not share a room with the requester. + + Defaults to `false`. + +* `prefer_local_users` (boolean): Defines whether to prefer local users in search query results. If set to true, local users are more likely to appear above remote users when searching the user directory. Defaults to `false`. + +* `exclude_remote_users` (boolean): If set to true, the search will only return local users. Defaults to `false`. + +* `show_locked_users` (boolean): Defines whether to show locked users in search query results. Defaults to `false`. Example configuration: ```yaml user_directory: - enabled: false - search_all_users: true - prefer_local_users: true - exclude_remote_users: false - show_locked_users: true + enabled: false + search_all_users: true + prefer_local_users: true + exclude_remote_users: false + show_locked_users: true ``` --- ### `user_consent` -For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). +*(object)* For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). -Parts of this section are required if enabling the `consent` resource under -[`listeners`](#listeners), in particular `template_dir` and `version`. +Parts of this section are required if enabling the `consent` resource under [`listeners`](#listeners), in particular `template_dir` and `version`. -* `template_dir`: gives the location of the templates for the HTML forms. - This directory should contain one subdirectory per language (eg, `en`, `fr`), - and each language directory should contain the policy document (named as - .html) and a success page (success.html). +This setting has the following sub-options: -* `version`: specifies the 'current' version of the policy document. It defines - the version to be served by the consent resource if there is no 'v' - parameter. +* `template_dir` (string): Gives the location of the templates for the HTML forms. This directory should contain one subdirectory per language (eg, `en`, `fr`), and each language directory should contain the policy document (named as .html) and a success page (success.html). -* `server_notice_content`: if enabled, will send a user a "Server Notice" - asking them to consent to the privacy policy. The [`server_notices` section](#server_notices) - must also be configured for this to work. Notices will *not* be sent to - guest users unless `send_server_notice_to_guests` is set to true. +* `version` (number): Specifies the "current" version of the policy document. It defines the version to be served by the consent resource if there is no `v` parameter. -* `block_events_error`, if set, will block any attempts to send events - until the user consents to the privacy policy. The value of the setting is - used as the text of the error. +* `server_notice_content` (object): If enabled, will send a user a "Server Notice" asking them to consent to the privacy policy. The [`server_notices` section](#server_notices) must also be configured for this to work. Notices will *not* be sent to guest users unless `send_server_notice_to_guests` is set to true. -* `require_at_registration`, if enabled, will add a step to the registration - process, similar to how captcha works. Users will be required to accept the - policy before their account is created. + This setting has the following sub-options: -* `policy_name` is the display name of the policy users will see when registering - for an account. Has no effect unless `require_at_registration` is enabled. - Defaults to "Privacy Policy". + * `msgtype` (string): Message type of the notice event. + + * `body` (string): Message template for the server notice event body. + +* `send_server_notice_to_guests` (boolean): Send server notices to guest users, too. Defaults to `false`. + +* `block_events_error` (string|null): If set, will block any attempts to send events until the user consents to the privacy policy. The value of the setting is used as the text of the error. Defaults to `null`. + +* `require_at_registration` (boolean): If enabled, will add a step to the registration process, similar to how captcha works. Users will be required to accept the policy before their account is created. + +* `policy_name` (string): Human-readable name of the privacy policy. Defaults to `"Privacy Policy"`. Example configuration: ```yaml @@ -4168,25 +3776,22 @@ user_consent: version: 1.0 server_notice_content: msgtype: m.text - body: >- - To continue using this homeserver you must review and agree to the - terms and conditions at %(consent_uri)s + body: To continue using this homeserver you must review and agree to the terms + and conditions at %(consent_uri)s send_server_notice_to_guests: true - block_events_error: >- - To continue using this homeserver you must review and agree to the - terms and conditions at %(consent_uri)s + block_events_error: To continue using this homeserver you must review and agree + to the terms and conditions at %(consent_uri)s require_at_registration: false policy_name: Privacy Policy ``` --- ### `stats` -Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) -for more. +*(object)* Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) for more. -* `enabled`: Set to false to disable room and user statistics. Note that doing - so may cause certain features (such as the room directory) not to work - correctly. Defaults to true. +This setting has the following sub-options: + +* `enabled` (boolean): Set to false to disable room and user statistics. Note that doing so may cause certain features (such as the room directory) not to work correctly. Defaults to `true`. Example configuration: ```yaml @@ -4196,42 +3801,53 @@ stats: --- ### `server_notices` -Use this setting to enable a room which can be used to send notices -from the server to users. It is a special room which users cannot leave; notices -in the room come from a special "notices" user id. +*(object)* Use this setting to enable a room which can be used to send notices from the server to users. It is a special room which users cannot leave; notices in the room come from a special "notices" user id. -If you use this setting, you *must* define the `system_mxid_localpart` -sub-setting, which defines the id of the user which will be used to send the -notices. - -Sub-options for this setting include: -* `system_mxid_display_name`: set the display name of the "notices" user -* `system_mxid_avatar_url`: set the avatar for the "notices" user -* `room_name`: set the room name of the server notices room -* `room_avatar_url`: optional string. The room avatar to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given an avatar. Defaults to the empty string. _Added in Synapse 1.99.0._ -* `room_topic`: optional string. The topic to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given a topic. Defaults to the empty string. _Added in Synapse 1.99.0._ -* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited. - Defaults to false. _Added in Synapse 1.98.0._ +If you use this setting, you *must* define the `system_mxid_localpart` sub-setting, which defines the id of the user which will be used to send the notices. Note that the name, topic and avatar of existing server notice rooms will only be updated when a new notice event is sent. +This setting has the following sub-options: + +* `system_mxid_display_name` (string): Display name of the "notices" user. Defaults to `"Notices"`. + +* `system_mxid_avatar_url` (string|null): Avatar for the "notices" user. Defaults to `null`. + +* `room_name` (string): Room name of the server notices room. Defaults to `"Server Notices"`. + +* `room_avatar_url` (string|null): Room avatar to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given an avatar. + + _Added in Synapse 1.99.0._ + + Defaults to `null`. + +* `room_topic` (string|null): Topic to use for server notice rooms. If set to the empty string `""`, notice rooms will not be given a topic. Defaults to the empty string. + + _Added in Synapse 1.99.0._ + + Defaults to `null`. + +* `auto_join` (boolean): If true, the user will be automatically joined to the room instead of being invited. + + _Added in Synapse 1.98.0._ + + Defaults to `false`. + Example configuration: ```yaml server_notices: system_mxid_localpart: notices - system_mxid_display_name: "Server Notices" - system_mxid_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" - room_name: "Server Notices" - room_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" - room_topic: "Room used by your server admin to notice you of important information" + system_mxid_display_name: Server Notices + system_mxid_avatar_url: mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ + room_name: Server Notices + room_avatar_url: mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ + room_topic: Room used by your server admin to notice you of important information auto_join: true ``` --- ### `enable_room_list_search` -Set to false to disable searching the public room list. When disabled -blocks searching local and remote room lists for local and remote -users by always returning an empty list for all queries. Defaults to true. +*(boolean)* Set to false to disable searching the public room list. When disabled blocks searching local and remote room lists for local and remote users by always returning an empty list for all queries. Defaults to `true`. Example configuration: ```yaml @@ -4240,196 +3856,142 @@ enable_room_list_search: false --- ### `alias_creation_rules` -The `alias_creation_rules` option allows server admins to prevent unwanted -alias creation on this server. +*(array|null)* The `alias_creation_rules` option allows server admins to prevent unwanted alias creation on this server. -This setting is an optional list of 0 or more rules. By default, no list is -provided, meaning that all alias creations are permitted. +This setting is an optional list of 0 or more rules. By default, no list is provided, meaning that all alias creations are permitted. -Otherwise, requests to create aliases are matched against each rule in order. -The first rule that matches decides if the request is allowed or denied. If no -rule matches, the request is denied. In particular, this means that configuring -an empty list of rules will deny every alias creation request. +Otherwise, requests to create aliases are matched against each rule in order. The first rule that matches decides if the request is allowed or denied. If no rule matches, the request is denied. In particular, this means that configuring an empty list of rules will deny every alias creation request. -Each rule is a YAML object containing four fields, each of which is an optional string: +Each of the glob patterns is optional, defaulting to `*` ("match anything"). Note that the patterns match against fully qualified IDs, e.g. against `@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. -* `user_id`: a glob pattern that matches against the creator of the alias. -* `alias`: a glob pattern that matches against the alias being created. -* `room_id`: a glob pattern that matches against the room ID the alias is being pointed at. -* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. +Each rule is a YAML object containing four fields, each of which is an optional string -Each of the glob patterns is optional, defaulting to `*` ("match anything"). -Note that the patterns match against fully qualified IDs, e.g. against -`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead -of `alice`, `room` and `abcedgghijk`. +Defaults to `null`. -Example configuration: +Options for each entry include: +* `user_id` (string|null): Glob pattern that matches against the creator of the alias. + +* `alias` (string|null): Glob pattern that matches against the alias being created. + +* `room_id` (string|null): Glob pattern that matches against the room ID the alias is being pointed at. + +* `action` (string): Either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. + +Example configurations: ```yaml -# No rule list specified. All alias creations are allowed. -# This is the default behaviour. -alias_creation_rules: +alias_creation_rules: null ``` ```yaml -# A list of one rule which allows everything. -# This has the same effect as the previous example. alias_creation_rules: - - "action": "allow" +- action: allow ``` ```yaml -# An empty list of rules. All alias creations are denied. alias_creation_rules: [] ``` ```yaml -# A list of one rule which denies everything. -# This has the same effect as the previous example. alias_creation_rules: - - "action": "deny" +- action: deny ``` ```yaml -# Prevent a specific user from creating aliases. -# Allow other users to create any alias alias_creation_rules: - - user_id: "@bad_user:example.com" - action: deny - - - action: allow +- user_id: '@bad_user:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent aliases being created which point to a specific room. alias_creation_rules: - - room_id: "!forbiddenRoom:example.com" - action: deny - - - action: allow +- room_id: '!forbiddenRoom:example.com' + action: deny +- action: allow ``` - --- ### `room_list_publication_rules` -The `room_list_publication_rules` option allows server admins to prevent -unwanted entries from being published in the public room list. +*(array|null)* The `room_list_publication_rules` option allows server admins to prevent unwanted entries from being published in the public room list. -The format of this option is the same as that for -[`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more -rules. By default, no list is provided, meaning that no one may publish to the -room list (except server admins). +The format of this option is the same as that for [`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more rules. By default, no list is provided, meaning that no one may publish to the room list (except server admins). -Otherwise, requests to publish a room are matched against each rule in order. -The first rule that matches decides if the request is allowed or denied. If no -rule matches, the request is denied. In particular, this means that configuring -an empty list of rules will deny every alias creation request. +Otherwise, requests to publish a room are matched against each rule in order. The first rule that matches decides if the request is allowed or denied. If no rule matches, the request is denied. In particular, this means that configuring an empty list of rules will deny every alias creation request. -Requests to create a public (public as in published to the room directory) room which violates -the configured rules will result in the room being created but not published to the room directory. +Requests to create a public (public as in published to the room directory) room which violates the configured rules will result in the room being created but not published to the room directory. -Each rule is a YAML object containing four fields, each of which is an optional string: +Each of the glob patterns is optional, defaulting to `*` ("match anything"). Note that the patterns match against fully qualified IDs, e.g. against `@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. -* `user_id`: a glob pattern that matches against the user publishing the room. -* `alias`: a glob pattern that matches against one of published room's aliases. +Each rule is a YAML object containing four fields, each of which is an optional string. + +_Changed in Synapse 1.126.0: The default was changed to deny publishing to the room list by default_ + +Defaults to `null`. + +Options for each entry include: + +* `user_id` (string|null): Glob pattern that matches against the user publishing the room. + +* `alias` (string|null): Glob pattern that matches against one of published room's aliases. - If the room has no aliases, the alias match fails unless `alias` is unspecified or `*`. - If the room has exactly one alias, the alias match succeeds if the `alias` pattern matches that alias. - If the room has two or more aliases, the alias match succeeds if the pattern matches at least one of the aliases. -* `room_id`: a glob pattern that matches against the room ID of the room being published. -* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. -Each of the glob patterns is optional, defaulting to `*` ("match anything"). -Note that the patterns match against fully qualified IDs, e.g. against -`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead -of `alice`, `room` and `abcedgghijk`. +* `room_id` (string|null): Glob pattern that matches against the room ID of the room being published. +* `action` (string): Either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. -_Changed in Synapse 1.126.0: The default was changed to deny publishing to the -room list by default_ - - -Example configuration: - +Example configurations: ```yaml -# No rule list specified. No one may publish any room to the public list, except server admins. -# This is the default behaviour. -room_list_publication_rules: +room_list_publication_rules: null ``` ```yaml -# A list of one rule which denies everything. -# This has the same effect as the previous example. room_list_publication_rules: - - "action": "deny" +- action: deny ``` ```yaml -# An empty list of rules. -# This has the same effect as the previous example. room_list_publication_rules: [] ``` ```yaml -# A list of one rule which allows everything. -# This was the default behaviour pre v1.126.0. room_list_publication_rules: - - "action": "allow" +- action: allow ``` ```yaml -# Prevent a specific user from publishing rooms. -# Allow other users to publish anything. room_list_publication_rules: - - user_id: "@bad_user:example.com" - action: deny - - - action: allow +- user_id: '@bad_user:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent publication of a specific room. room_list_publication_rules: - - room_id: "!forbiddenRoom:example.com" - action: deny - - - action: allow +- room_id: '!forbiddenRoom:example.com' + action: deny +- action: allow ``` ```yaml -# Prevent publication of rooms with at least one alias containing the word "potato". room_list_publication_rules: - - alias: "#*potato*:example.com" - action: deny - - - action: allow +- alias: '#*potato*:example.com' + action: deny +- action: allow ``` - --- ### `default_power_level_content_override` -The `default_power_level_content_override` option controls the default power -levels for rooms. +*(object)* The `default_power_level_content_override` option controls the default power levels for rooms. -Useful if you know that your users need special permissions in rooms -that they create (e.g. to send particular types of state events without -needing an elevated power level). This takes the same shape as the -`power_level_content_override` parameter in the /createRoom API, but -is applied before that parameter. +Useful if you know that your users need special permissions in rooms that they create (e.g. to send particular types of state events without needing an elevated power level). This takes the same shape as the `power_level_content_override` parameter in the /createRoom API, but is applied before that parameter. -Note that each key provided inside a preset (for example `events` in the example -below) will overwrite all existing defaults inside that key. So in the example -below, newly-created private_chat rooms will have no rules for any event types -except `com.example.foo`. - -Example configuration: -```yaml -default_power_level_content_override: - private_chat: { "events": { "com.example.foo" : 0 } } - trusted_private_chat: null - public_chat: null -``` +Note that each key provided inside a preset (for example `events` in the example below) will overwrite all existing defaults inside that key. So in Example #1, newly-created private_chat rooms will have no rules for any event types except `com.example.foo`. The default power levels for each preset are: + ```yaml "m.room.name": 50 "m.room.power_levels": 100 @@ -4441,149 +4003,140 @@ The default power levels for each preset are: "m.room.encryption": 100 ``` -So a complete example where the default power-levels for a preset are maintained -but the power level for a new key is set is: +In Example #2 the default power-levels for a preset are maintained, but the power level for a new key is set. + +Defaults to `{}`. + +Example configurations: ```yaml default_power_level_content_override: - private_chat: + private_chat: events: - "com.example.foo": 0 - "m.room.name": 50 - "m.room.power_levels": 100 - "m.room.history_visibility": 100 - "m.room.canonical_alias": 50 - "m.room.avatar": 50 - "m.room.tombstone": 100 - "m.room.server_acl": 100 - "m.room.encryption": 100 - trusted_private_chat: null - public_chat: null + com.example.foo: 0 + trusted_private_chat: null + public_chat: null ``` +```yaml +default_power_level_content_override: + private_chat: + events: + com.example.foo: 0 + m.room.name: 50 + m.room.power_levels: 100 + m.room.history_visibility: 100 + m.room.canonical_alias: 50 + m.room.avatar: 50 + m.room.tombstone: 100 + m.room.server_acl: 100 + m.room.encryption: 100 + trusted_private_chat: null + public_chat: null +``` --- ### `forget_rooms_on_leave` -Set to true to automatically forget rooms for users when they leave them, either -normally or via a kick or ban. Defaults to false. +*(boolean)* Set to true to automatically forget rooms for users when they leave them, either normally or via a kick or ban. Defaults to `false`. Example configuration: ```yaml -forget_rooms_on_leave: false +forget_rooms_on_leave: true ``` --- ### `exclude_rooms_from_sync` -A list of rooms to exclude from sync responses. This is useful for server -administrators wishing to group users into a room without these users being able -to see it from their client. -By default, no room is excluded. +*(array)* A list of rooms to exclude from sync responses. This is useful for server administrators wishing to group users into a room without these users being able to see it from their client. Defaults to `[]`. Example configuration: ```yaml exclude_rooms_from_sync: - - "!foo:example.com" +- '!foo:example.com' ``` - --- ## Opentracing + Configuration options related to Opentracing support. --- ### `opentracing` -These settings enable and configure opentracing, which implements distributed tracing. -This allows you to observe the causal chains of events across servers -including requests, key lookups etc., across any server running -synapse or any other services which support opentracing -(specifically those implemented with Jaeger). +*(object)* These settings enable and configure opentracing, which implements distributed tracing. This allows you to observe the causal chains of events across servers including requests, key lookups etc., across any server running synapse or any other services which support opentracing (specifically those implemented with Jaeger). -Sub-options include: -* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. -* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. - See [here](../../opentracing.md) for more. - This is a list of regexes which are matched against the `server_name` of the homeserver. - By default, it is empty, so no servers are matched. -* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, - even if the tracing system would otherwise drop the traces due to probabilistic sampling. - By default, the list is empty. -* `jaeger_config`: Jaeger can be configured to sample traces at different rates. - All configuration options provided by Jaeger can be set here. Jaeger's configuration is - mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). +This setting has the following sub-options: + +* `enabled` (boolean): Whether tracing is enabled. Set to true to enable. Defaults to `false`. + +* `homeserver_whitelist` (array): The list of homeservers we wish to send and receive span contexts and span baggage. See [here](../../opentracing.md) for more. This is a list of regexes which are matched against the `server_name` of the homeserver. If the list is empty, no servers are matched. Defaults to `[]`. + +* `force_tracing_for_users` (array): A list of the matrix IDs of users whose requests will always be traced, even if the tracing system would otherwise drop the traces due to probabilistic sampling. Defaults to `[]`. + +* `jaeger_config` (object): Jaeger can be configured to sample traces at different rates. All configuration options provided by Jaeger can be set here. Jaeger's configuration is mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). Defaults to `{}`. Example configuration: ```yaml opentracing: - enabled: true - homeserver_whitelist: - - ".*" - force_tracing_for_users: - - "@user1:server_name" - - "@user2:server_name" - - jaeger_config: - sampler: - type: const - param: 1 - logging: - false + enabled: true + homeserver_whitelist: + - .* + force_tracing_for_users: + - '@user1:server_name' + - '@user2:server_name' + jaeger_config: + sampler: + type: const + param: 1 + logging: false ``` --- ## Coordinating workers -Configuration options related to workers which belong in the main config file -(usually called `homeserver.yaml`). -A Synapse deployment can scale horizontally by running multiple Synapse processes -called _workers_. Incoming requests are distributed between workers to handle higher -loads. Some workers are privileged and can accept requests from other workers. + +Configuration options related to workers which belong in the main config file (usually called `homeserver.yaml`). A Synapse deployment can scale horizontally by running multiple Synapse processes called _workers_. Incoming requests are distributed between workers to handle higher loads. Some workers are privileged and can accept requests from other workers. As a result, the worker configuration is divided into two parts. -1. The first part (in this section of the manual) defines which shardable tasks - are delegated to privileged workers. This allows unprivileged workers to make - requests to a privileged worker to act on their behalf. -1. [The second part](#individual-worker-configuration) - controls the behaviour of individual workers in isolation. +1. The first part (in this section of the manual) defines which shardable tasks are delegated to privileged workers. This allows unprivileged workers to make requests to a privileged worker to act on their behalf. +2. [The second part](#individual-worker-configuration) controls the behaviour of individual workers in isolation. For guidance on setting up workers, see the [worker documentation](../../workers.md). --- ### `worker_replication_secret` -A shared secret used by the replication APIs on the main process to authenticate -HTTP requests from workers. +*(string|null)* A shared secret used by the replication APIs on the main process to authenticate HTTP requests from workers. -The default, this value is omitted (equivalently `null`), which means that -traffic between the workers and the main process is not authenticated. +If unset or null, traffic between the workers and the main process is not authenticated. -Replacing an existing `worker_replication_secret` with a new one will break -communication with all workers that have not yet updated their secret. +Replacing an existing `worker_replication_secret` with a new one will break communication with all workers that have not yet updated their secret. + +Defaults to `null`. Example configuration: ```yaml -worker_replication_secret: "secret_secret" +worker_replication_secret: secret_secret ``` --- ### `worker_replication_secret_path` -An alternative to [`worker_replication_secret`](#worker_replication_secret): -allows the secret to be specified in an external file. +*(string|null)* An alternative to [`worker_replication_secret`](#worker_replication_secret): allows the secret to be specified in an external file. -The file should be a plain text file, containing only the secret. -Synapse reads the secret from the given file once at startup. +The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. + +_Added in Synapse 1.126.0._ + +Defaults to `null`. Example configuration: ```yaml worker_replication_secret_path: /path/to/secrets/file ``` - -_Added in Synapse 1.126.0._ - --- ### `start_pushers` -Unnecessary to set if using [`pusher_instances`](#pusher_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). +*(boolean)* Unnecessary to set if using [`pusher_instances`](#pusher_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). -Controls sending of push notifications on the main process. Set to `false` -if using a [pusher worker](../../workers.md#synapseapppusher). Defaults to `true`. +Controls sending of push notifications on the main process. Set to `false` if using a [pusher worker](../../workers.md#synapseapppusher). + +Defaults to `true`. Example configuration: ```yaml @@ -4592,32 +4145,26 @@ start_pushers: false --- ### `pusher_instances` -It is possible to scale the processes that handle sending push notifications to [sygnal](https://github.com/matrix-org/sygnal) -and email by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to -a `pusher_instances` map. Doing so will remove handling of this function from the main -process. Multiple workers can be added to this map, in which case the work is balanced -across them. Ensure the main process and all pusher workers are restarted after changing -this option. +*(array)* It is possible to scale the processes that handle sending push notifications to [sygnal](https://github.com/matrix-org/sygnal) and email by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to a `pusher_instances` map. Doing so will remove handling of this function from the main process. Multiple workers can be added to this map, in which case the work is balanced across them. Ensure the main process and all pusher workers are restarted after changing this option. Defaults to `[]`. -Example configuration for a single worker: +Example configurations: ```yaml pusher_instances: - - pusher_worker1 -``` -And for multiple workers: -```yaml -pusher_instances: - - pusher_worker1 - - pusher_worker2 +- pusher_worker1 ``` +```yaml +pusher_instances: +- pusher_worker1 +- pusher_worker2 +``` --- ### `send_federation` -Unnecessary to set if using [`federation_sender_instances`](#federation_sender_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). +*(boolean)* Unnecessary to set if using [`federation_sender_instances`](#federation_sender_instances) with [`generic_workers`](../../workers.md#synapseappgeneric_worker). + +Controls sending of outbound federation transactions on the main process. Set to `false` if using a [federation sender worker](../../workers.md#synapseappfederation_sender). -Controls sending of outbound federation transactions on the main process. -Set to `false` if using a [federation sender worker](../../workers.md#synapseappfederation_sender). Defaults to `true`. Example configuration: @@ -4627,48 +4174,31 @@ send_federation: false --- ### `federation_sender_instances` -It is possible to scale the processes that handle sending outbound federation requests -by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to -a `federation_sender_instances` map. Doing so will remove handling of this function from -the main process. Multiple workers can be added to this map, in which case the work is -balanced across them. +*(array)* It is possible to scale the processes that handle sending outbound federation requests by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to a `federation_sender_instances` map. Doing so will remove handling of this function from the main process. Multiple workers can be added to this map, in which case the work is balanced across them. -The way that the load balancing works is any outbound federation request will be assigned -to a federation sender worker based on the hash of the destination server name. This -means that all requests being sent to the same destination will be processed by the same -worker instance. Multiple `federation_sender_instances` are useful if there is a federation -with multiple servers. +The way that the load balancing works is any outbound federation request will be assigned to a federation sender worker based on the hash of the destination server name. This means that all requests being sent to the same destination will be processed by the same worker instance. Multiple `federation_sender_instances` are useful if there is a federation with multiple servers. -This configuration setting must be shared between all workers handling federation -sending, and if changed all federation sender workers must be stopped at the same time -and then started, to ensure that all instances are running with the same config (otherwise -events may be dropped). +This configuration setting must be shared between all workers handling federation sending, and if changed all federation sender workers must be stopped at the same time and then started, to ensure that all instances are running with the same config (otherwise events may be dropped). -Example configuration for a single worker: +Defaults to `[]`. + +Example configurations: ```yaml federation_sender_instances: - - federation_sender1 +- federation_sender1 ``` -And for multiple workers: + ```yaml federation_sender_instances: - - federation_sender1 - - federation_sender2 +- federation_sender1 +- federation_sender2 ``` --- ### `instance_map` -When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP -replication listener of the worker, if configured, and to the main process. Each worker -declared under [`stream_writers`](../../workers.md#stream-writers) and -[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP -replication listener, and that listener should be included in the `instance_map`. The -main process also needs an entry on the `instance_map`, and it should be listed under -`main` **if even one other worker exists**. Ensure the port matches with what is -declared inside the `listener` block for a `replication` listener. +*(object)* When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP replication listener of the worker, if configured, and to the main process. Each worker declared under [`stream_writers`](../../workers.md#stream-writers) and [`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP replication listener, and that listener should be included in the `instance_map`. The main process also needs an entry on the `instance_map`, and it should be listed under `main` **if even one other worker exists**. Ensure the port matches with what is declared inside the `listener` block for a `replication` listener. Defaults to `{}`. - -Example configuration: +Example configurations: ```yaml instance_map: main: @@ -4682,7 +4212,7 @@ instance_map: port: 8035 tls: true ``` -Example configuration(#2, for UNIX sockets): + ```yaml instance_map: main: @@ -4693,12 +4223,27 @@ instance_map: --- ### `stream_writers` -Experimental: When using workers you can define which workers should -handle writing to streams such as event persistence and typing notifications. -Any worker specified here must also be in the [`instance_map`](#instance_map). +*(object)* Experimental: When using workers you can define which workers should handle writing to streams such as event persistence and typing notifications. Any worker specified here must also be in the [`instance_map`](#instance_map). -See the list of available streams in the -[worker documentation](../../workers.md#stream-writers). +See the list of available streams in the [worker documentation](../../workers.md#stream-writers). + +Defaults to `{}`. + +This setting has the following sub-options: + +* `events` (string): Name of a worker assigned to the `events` stream. + +* `typing` (string): Name of a worker assigned to the `typing` stream. + +* `to_device` (string): Name of a worker assigned to the `to_device` stream. + +* `account_data` (string): Name of a worker assigned to the `account_data` stream. + +* `receipts` (string): Name of a worker assigned to the `receipts` stream. + +* `presence` (string): Name of a worker assigned to the `presence` stream. + +* `push_rules` (string): Name of a worker assigned to the `push_rules` stream. Example configuration: ```yaml @@ -4709,30 +4254,24 @@ stream_writers: --- ### `outbound_federation_restricted_to` -When using workers, you can restrict outbound federation traffic to only go through a -specific subset of workers. Any worker specified here must also be in the -[`instance_map`](#instance_map). -[`worker_replication_secret`](#worker_replication_secret) must also be configured to -authorize inter-worker communication. +*(array)* When using workers, you can restrict outbound federation traffic to only go through a specific subset of workers. Any worker specified here must also be in the [`instance_map`](#instance_map). [`worker_replication_secret`](#worker_replication_secret) must also be configured to authorize inter-worker communication. -```yaml -outbound_federation_restricted_to: - - federation_sender1 - - federation_sender2 -``` - -Also see the [worker -documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) -for more info. +Also see the [worker documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) for more info. _Added in Synapse 1.89.0._ +Defaults to `[]`. + +Example configuration: +```yaml +outbound_federation_restricted_to: +- federation_sender1 +- federation_sender2 +``` --- ### `run_background_tasks_on` -The [worker](../../workers.md#background-tasks) that is used to run -background tasks (e.g. cleaning up expired data). If not provided this -defaults to the main process. +*(string|null)* The [worker](../../workers.md#background-tasks) that is used to run background tasks (e.g. cleaning up expired data). If not provided this defaults to the main process. Defaults to `null`. Example configuration: ```yaml @@ -4741,73 +4280,80 @@ run_background_tasks_on: worker1 --- ### `update_user_directory_from_worker` -The [worker](../../workers.md#updating-the-user-directory) that is used to -update the user directory. If not provided this defaults to the main process. +*(string|null)* The [worker](../../workers.md#updating-the-user-directory) that is used to update the user directory. If not provided this defaults to the main process. + +_Added in Synapse 1.59.0._ + +Defaults to `null`. Example configuration: ```yaml update_user_directory_from_worker: worker1 ``` - -_Added in Synapse 1.59.0._ - --- ### `notify_appservices_from_worker` -The [worker](../../workers.md#notifying-application-services) that is used to -send output traffic to Application Services. If not provided this defaults -to the main process. +*(string|null)* The [worker](../../workers.md#notifying-application-services) that is used to send output traffic to Application Services. If not provided this defaults to the main process. + +_Added in Synapse 1.59.0._ + +Defaults to `null`. Example configuration: ```yaml notify_appservices_from_worker: worker1 ``` - -_Added in Synapse 1.59.0._ - --- ### `media_instance_running_background_jobs` -The [worker](../../workers.md#synapseappmedia_repository) that is used to run -background tasks for media repository. If running multiple media repositories -you must configure a single instance to run the background tasks. If not provided -this defaults to the main process or your single `media_repository` worker. +*(string|null)* The [worker](../../workers.md#synapseappmedia_repository) that is used to run background tasks for media repository. If running multiple media repositories you must configure a single instance to run the background tasks. If not provided this defaults to the main process or your single `media_repository` worker. + +_Added in Synapse 1.16.0._ + +Defaults to `null`. Example configuration: ```yaml media_instance_running_background_jobs: worker1 ``` - -_Added in Synapse 1.16.0._ - --- ### `redis` -Configuration for Redis when using workers. This *must* be enabled when using workers. +*(object)* Configuration for Redis when using workers. This *must* be enabled when using workers. + +_Added in Synapse 1.78.0._ + +_Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_ + +_Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ + +_Changed in Synapse 1.116.0: Added password\_path_ + This setting has the following sub-options: -* `enabled`: whether to use Redis support. Defaults to false. -* `host` and `port`: Optional host and port to use to connect to redis. Defaults to - localhost and 6379 -* `path`: The full path to a local Unix socket file. **If this is used, `host` and - `port` are ignored.** Defaults to `/tmp/redis.sock' -* `password`: Optional password if configured on the Redis instance. -* `password_path`: Alternative to `password`, reading the password from an - external file. The file should be a plain text file, containing only the - password. Synapse reads the password from the given file once at startup. -* `dbid`: Optional redis dbid if needs to connect to specific redis logical db. -* `use_tls`: Whether to use tls connection. Defaults to false. -* `certificate_file`: Optional path to the certificate file -* `private_key_file`: Optional path to the private key file -* `ca_file`: Optional path to the CA certificate file. Use this one or: -* `ca_path`: Optional path to the folder containing the CA certificate file - _Added in Synapse 1.78.0._ +* `enabled` (boolean): Whether to use Redis support. Defaults to `false`. - _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_ +* `host` (string): Optional host to use to connect to Redis. Defaults to `"localhost"`. - _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ +* `port` (integer): Optional port to use to connect to Redis. Defaults to `6379`. - _Changed in Synapse 1.116.0: Added password\_path_ +* `path` (string): The full path to a local Unix socket file. **If this is used, `host` and `port` are ignored.** Defaults to `"/tmp/redis.sock"`. + +* `password` (string|null): Optional password if configured on the Redis instance. Defaults to `null`. + +* `password_path` (string|null): Alternative to `password`, reading the password from an external file. The file should be a plain text file, containing only the password. Synapse reads the password from the given file once at startup. Defaults to `null`. + +* `dbid` (string|null): Optional redis dbid if needs to connect to specific redis logical db. Defaults to `null`. + +* `use_tls` (boolean): Whether to use a TLS connection. Defaults to `false`. + +* `certificate_file` (string|null): Optional path to the certificate file. Defaults to `null`. + +* `private_key_file` (string|null): Optional path to the private key file. Defaults to `null`. + +* `ca_file` (string|null): Optional path to the CA certificate file. Use this one or `ca_path` Defaults to `null`. + +* `ca_path` (string|null): Optional path to the folder containing the CA certificate file. Use this one or `ca_file` Defaults to `null`. Example configuration: ```yaml @@ -4816,31 +4362,25 @@ redis: host: localhost port: 6379 password_path: - # OR password: dbid: - #use_tls: True - #certificate_file: - #private_key_file: - #ca_file: ``` --- ## Individual worker configuration -These options configure an individual worker, in its worker configuration file. -They should be not be provided when configuring the main process. -Note also the configuration above for -[coordinating a cluster of workers](#coordinating-workers). +These options configure an individual worker, in its worker configuration file. They should be not be provided when configuring the main process. + +Note also the configuration above for [coordinating a cluster of workers](#coordinating-workers). For guidance on setting up workers, see the [worker documentation](../../workers.md). --- ### `worker_app` -The type of worker. The currently available worker applications are listed -in [worker documentation](../../workers.md#available-worker-applications). +*(string)* The type of worker. The currently available worker applications are listed in [worker documentation](../../workers.md#available-worker-applications). -The most common worker is the -[`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). +The most common worker is the [`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). + +There is no default for this option. Example configuration: ```yaml @@ -4849,9 +4389,7 @@ worker_app: synapse.app.generic_worker --- ### `worker_name` -A unique name for the worker. The worker needs a name to be addressed in -further parameters and identification in log files. We strongly recommend -giving each worker a unique `worker_name`. +*(string)* A unique name for the worker. The worker needs a name to be addressed in further parameters and identification in log files. We strongly recommend giving each worker a unique `worker_name`. There is no default for this option. Example configuration: ```yaml @@ -4860,46 +4398,45 @@ worker_name: generic_worker1 --- ### `worker_listeners` -A worker can handle HTTP requests. To do so, a `worker_listeners` option -must be declared, in the same way as the [`listeners` option](#listeners) -in the shared config. +*(array)* A worker can handle HTTP requests. To do so, a `worker_listeners` option must be declared, in the same way as the [`listeners` option](#listeners) in the shared config. -Workers declared in [`stream_writers`](#stream_writers) and [`instance_map`](#instance_map) - will need to include a `replication` listener here, in order to accept internal HTTP -requests from other workers. +Workers declared in [`stream_writers`](#stream_writers) and [`instance_map`](#instance_map) will need to include a `replication` listener here, in order to accept internal HTTP requests from other workers. -Example configuration: +Example #2 is using UNIX sockets with a `replication` listener. + +Defaults to `[]`. + +Example configurations: ```yaml worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] +- type: http + port: 8083 + resources: + - names: + - client + - federation ``` -Example configuration(#2, using UNIX sockets with a `replication` listener): + ```yaml worker_listeners: - - type: http - path: /run/synapse/worker_replication.sock - resources: - - names: [replication] - - type: http - path: /run/synapse/worker_public.sock - resources: - - names: [client, federation] +- type: http + path: /run/synapse/worker_replication.sock + resources: + - names: + - replication +- type: http + path: /run/synapse/worker_public.sock + resources: + - names: + - client + - federation ``` --- ### `worker_manhole` -A worker may have a listener for [`manhole`](../../manhole.md). -It allows server administrators to access a Python shell on the worker. +*(integer|null)* A worker may have a listener for [`manhole`](../../manhole.md). It allows server administrators to access a Python shell on the worker. -Example configuration: -```yaml -worker_manhole: 9000 -``` - -This is a short form for: +The example below is a short form for ```yaml worker_listeners: - port: 9000 @@ -4909,14 +4446,16 @@ worker_listeners: It needs also an additional [`manhole_settings`](#manhole_settings) configuration. +Defaults to `null`. + +Example configuration: +```yaml +worker_manhole: 9000 +``` --- ### `worker_daemonize` -Specifies whether the worker should be started as a daemon process. -If Synapse is being managed by [systemd](../../systemd-with-workers/), this option -must be omitted or set to `false`. - -Defaults to `false`. +*(boolean)* Specifies whether the worker should be started as a daemon process. If Synapse is being managed by [systemd](../../systemd-with-workers/), this option must be omitted or set to `false`. Defaults to `false`. Example configuration: ```yaml @@ -4925,15 +4464,14 @@ worker_daemonize: true --- ### `worker_pid_file` -When running a worker as a daemon, we need a place to store the -[PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. -This option defines the location of that "pid file". +*(string|null)* When running a worker as a daemon, we need a place to store the [PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. This option defines the location of that "pid file". -This option is required if `worker_daemonize` is `true` and ignored -otherwise. It has no default. +This option is required if `worker_daemonize` is `true` and ignored otherwise. See also the [`pid_file` option](#pid_file) option for the main Synapse process. +Defaults to `null`. + Example configuration: ```yaml worker_pid_file: DATADIR/generic_worker1.pid @@ -4941,9 +4479,7 @@ worker_pid_file: DATADIR/generic_worker1.pid --- ### `worker_log_config` -This option specifies a yaml python logging config file as described -[here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). -See also the [`log_config` option](#log_config) option for the main Synapse process. +*(string|null)* This option specifies a yaml python logging config file as described [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). See also the [`log_config` option](#log_config) option for the main Synapse process. Defaults to `null`. Example configuration: ```yaml @@ -4951,62 +4487,62 @@ worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml ``` --- ## Background Updates + Configuration settings related to background updates. --- ### `background_updates` -Background updates are database updates that are run in the background in batches. -The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to -sleep can all be configured. This is helpful to speed up or slow down the updates. +*(object)* Background updates are database updates that are run in the background in batches. The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to sleep can all be configured. This is helpful to speed up or slow down the updates. + This setting has the following sub-options: -* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. - Set a different time to change the default. -* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. -* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. - Set a duration to change the default. -* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. - Set a size to change the default. -* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. - Set a size to change the default. + +* `background_update_duration_ms` (integer): How long in milliseconds to run a batch of background updates for. Defaults to `100`. + +* `sleep_enabled` (boolean): Whether to sleep between updates. Defaults to `true`. + +* `sleep_duration_ms` (integer): If sleeping between updates, how long in milliseconds to sleep for. Defaults to `1000`. + +* `min_batch_size` (integer): Minimum size a batch of background updates can be. Must be greater than 0. Defaults to `1`. + +* `default_batch_size` (integer): The batch size to use for the first iteration of a new background update. Defaults to `100`. Example configuration: ```yaml background_updates: - background_update_duration_ms: 500 - sleep_enabled: false - sleep_duration_ms: 300 - min_batch_size: 10 - default_batch_size: 50 + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 ``` --- ## Auto Accept Invites + Configuration settings related to automatically accepting invites. --- ### `auto_accept_invites` -Automatically accepting invites controls whether users are presented with an invite request or if they -are instead automatically joined to a room when receiving an invite. Set the `enabled` sub-option to true to -enable auto-accepting invites. Defaults to false. -This setting has the following sub-options: -* `enabled`: Whether to run the auto-accept invites logic. Defaults to false. -* `only_for_direct_messages`: Whether invites should be automatically accepted for all room types, or only - for direct messages. Defaults to false. -* `only_from_local_users`: Whether to only automatically accept invites from users on this homeserver. Defaults to false. -* `worker_to_run_on`: Which worker to run this module on. This must match - the "worker_name". If not set or `null`, invites will be accepted on the - main process. +*(object)* Automatically accepting invites controls whether users are presented with an invite request or if they are instead automatically joined to a room when receiving an invite. Set the `enabled` sub-option to true to enable auto-accepting invites. -NOTE: Care should be taken not to enable this setting if the `synapse_auto_accept_invite` module is enabled and installed. -The two modules will compete to perform the same task and may result in undesired behaviour. For example, multiple join -events could be generated from a single invite. +NOTE: Care should be taken not to enable this setting if the `synapse_auto_accept_invite` module is enabled and installed. The two modules will compete to perform the same task and may result in undesired behaviour. For example, multiple join events could be generated from a single invite. + +This setting has the following sub-options: + +* `enabled` (boolean): Whether to run the auto-accept invites logic. Defaults to `false`. + +* `only_for_direct_messages` (boolean): Whether invites should be automatically accepted for all room types, or only for direct messages. Defaults to `false`. + +* `only_from_local_users` (boolean): Whether to only automatically accept invites from users on this homeserver. Defaults to `false`. + +* `worker_to_run_on` (string|null): Which worker to run this module on. This must match the "worker_name". If not set or `null`, invites will be accepted on the main process. Defaults to `null`. Example configuration: ```yaml auto_accept_invites: - enabled: true - only_for_direct_messages: true - only_from_local_users: true - worker_to_run_on: "worker_1" + enabled: true + only_for_direct_messages: true + only_from_local_users: true + worker_to_run_on: worker_1 ``` diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml new file mode 100644 index 0000000000..5ebe80f51c --- /dev/null +++ b/schema/synapse-config.schema.yaml @@ -0,0 +1,5711 @@ +$schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json +$id: https://element-hq.github.io/synapse/v1.131/schema/synapse-config.schema.json +type: object +properties: + modules: + type: array + description: >- + Use the `module` sub-option to add modules under this option to extend + functionality. The `module` setting then has a sub-option, `config`, which + can be used to define some configuration for the `module`. + items: + type: object + properties: + module: + type: string + description: Path to the Python class of the module. + config: + type: object + description: Configuration options for the module. + default: [] + examples: + - - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} + server_name: + type: string + description: >- + This sets the public-facing domain of the server. + + + The `server_name` name will appear at the end of usernames and room + addresses created on your server. For example if the `server_name` was + example.com, usernames on your server would be in the format + `@user:example.com`. + + + In most cases you should avoid using a matrix specific subdomain such as + matrix.example.com or synapse.example.com as the `server_name` for the + same reasons you wouldn't use user@email.example.com as your email + address. See [here](../../delegate.md) for information on how to host + Synapse on a subdomain while preserving a clean `server_name`. + + + The `server_name` cannot be changed later so it is important to configure + this correctly before you start Synapse. It should be all lowercase and + may contain an explicit port. + examples: + - matrix.org + - localhost:8080 + pid_file: + type: ["string", "null"] + description: When running Synapse as a daemon, the file to store the pid in. + default: null + examples: + - DATADIR/homeserver.pid + daemonize: + type: boolean + description: >- + Specifies whether Synapse should be started as a daemon process. If + Synapse is being managed by [systemd](../../systemd-with-workers/), this + option must be omitted or set to `false`. + + + This can also be set by the `--daemonize` (`-D`) argument when starting + Synapse. + + + See `worker_daemonize` for more information on daemonizing workers. + default: false + examples: + - true + print_pidfile: + type: boolean + description: >- + Print the path to the pidfile just before daemonizing. + + + This can also be set by the `--print-pidfile` argument when starting Synapse. + default: false + examples: + - true + user_agent_suffix: + type: ["string", "null"] + description: >- + A suffix that is appended to the Synapse user-agent (ex. `Synapse/v1.123.0`). + default: null + examples: + - " (I'm a teapot; Linux x86_64)" + use_frozen_dicts: + type: boolean + description: >- + Determines whether we should freeze the internal dict object in + `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. + signature dicts. However, freezing a dict is expensive. + default: false + examples: + - true + web_client_location: + type: ["string", "null"] + description: The absolute URL to the web client which `/` will redirect to. + default: null + examples: + - "https://riot.example.com/" + public_baseurl: + type: ["string", "null"] + description: >- + The public-facing base URL that clients use to access this Homeserver (not + including _matrix/...). This is the same URL a user might enter into the + "Custom Homeserver URL" field on their client. If you use Synapse with a + reverse proxy, this should be the URL to reach Synapse via the proxy. + Otherwise, it should be the URL to reach Synapse's client HTTP listener + (see [`listeners`](#listeners) below). + + + If unset or null, `https:///` is used. + default: null + examples: + - "https://example.com/" + serve_server_wellknown: + type: boolean + description: >- + By default, other servers will try to reach our server on port 8448, which + can be inconvenient in some environments. + + + Provided `https:///` on port 443 is routed to Synapse, this + option configures Synapse to serve a file at + `https:///.well-known/matrix/server`. This will tell other + servers to send traffic to port 443 instead. + + + This option currently defaults to false. + + + See [Delegation of incoming federation traffic](../../delegate.md) for + more information. + default: false + examples: + - true + extra_well_known_client_content: + type: object + description: >- + This option allows server runners to add arbitrary key-value pairs to the + [client-facing `.well-known` + response](https://spec.matrix.org/latest/client-server-api/#well-known-uri). + Note that the `public_baseurl` config option must be provided for Synapse + to serve a response to `/.well-known/matrix/client` at all. + + + If this option is provided, it parses the given yaml to json and serves it + on `/.well-known/matrix/client` endpoint alongside the standard + properties. + + + *Added in Synapse 1.62.0.* + examples: + - option1: value1 + option2: value2 + soft_file_limit: + type: integer + description: >- + Set the soft limit on the number of file descriptors synapse can use. Zero + is used to indicate synapse should set the soft limit to the hard limit. + default: 0 + examples: + - 3 + presence: + type: object + description: >- + Presence tracking allows users to see the state (e.g online/offline) of + other local and remote users. This option replaces the previous top-level + `use_presence` option. + properties: + enabled: + type: ["boolean", "string"] + description: >- + Set to false to disable presence tracking on this homeserver. + + + Can also be set to a special value of "untracked" which ignores + updates received via clients and federation, while still accepting + updates from the [module API](../../modules/index.md). + + + *The "untracked" option was added in Synapse 1.96.0.* + oneOf: + - type: boolean + - type: string + const: untracked + default: true + include_offline_users_on_sync: + type: boolean + description: >- + When clients perform an initial or `full_state` sync, presence results + for offline users are not included by default. Setting + `include_offline_users_on_sync` to `true` will always include offline + users in the results. + default: false + examples: + - enabled: false + include_offline_users_on_sync: false + require_auth_for_profile_requests: + type: boolean + description: >- + Whether to require authentication to retrieve profile data (avatars, + display names) of other users through the client API. Note that profile + data is also available via the federation API, unless + `allow_profile_lookup_over_federation` is set to false. + default: false + examples: + - true + limit_profile_requests_to_users_who_share_rooms: + type: boolean + description: >- + Use this option to require a user to share a room with another user in + order to retrieve their profile information. Only checked on Client-Server + requests. Profile requests from other servers should be checked by the + requesting server. + default: false + examples: + - true + include_profile_data_on_invite: + type: boolean + description: >- + Use this option to prevent a user's profile data from being retrieved and + displayed in a room until they have joined it. By default, a user's + profile data is included in an invite event, regardless of the values of + the above two settings, and whether or not the users share a server. + default: true + examples: + - false + allow_public_rooms_without_auth: + type: boolean + description: + If set to true, removes the need for authentication to access the server's + public rooms directory through the client API, meaning that anyone can + query the room directory. + default: false + examples: + - true + allow_public_rooms_over_federation: + type: boolean + description: >- + If set to true, allows any other homeserver to fetch the server's public + rooms directory via federation. + default: false + examples: + - true + default_room_version: + type: string + description: >- + The default room version for newly created rooms on this server. + + + Known room versions are listed + [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + + + For example, for room version 1, `default_room_version` should be set to + "1". + + + _Changed in Synapse 1.76:_ the default version room version was increased + from [9](https://spec.matrix.org/v1.5/rooms/v9/) to + [10](https://spec.matrix.org/v1.5/rooms/v10/). + default: "10" + examples: + - "8" + gc_thresholds: + type: ["array", "null"] + description: >- + The garbage collection threshold parameters to pass to `gc.set_threshold`, + if defined. + default: null + examples: + - - 700 + - 10 + - 10 + gc_min_interval: + type: array + description: >- + The minimum time in seconds between each GC for a generation, regardless + of the GC thresholds. This ensures that we don't do GC too frequently. A + value of `[1s, 10s, 30s]` indicates that a second must pass between + consecutive generation 0 GCs, etc. + default: + - 1s + - 10s + - 30s + examples: + - - 0.5s + - 30s + - 1m + filter_timeline_limit: + type: integer + description: >- + Set the limit on the returned events in the timeline in the get and sync + operations. A value of -1 means no upper limit. + default: 100 + examples: + - 5000 + block_non_admin_invites: + type: boolean + description: >- + Whether room invites to users on this server should be blocked (except + those sent by local server admins). + default: false + examples: + - true + enable_search: + type: boolean + description: >- + If set to false, new messages will not be indexed for searching and users + will receive errors when searching for messages. + default: true + examples: + - false + ip_range_blacklist: + type: array + description: >- + This option prevents outgoing requests from being sent to the specified + blacklisted IP address CIDR ranges. If this option is not specified then + it defaults to private IP address ranges (see the example below). + + + The blacklist applies to the outbound requests for federation, identity + servers, push servers, and for checking key validity for third-party + invite events. + + + (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly + listed here, since they correspond to unroutable addresses.) + + + This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + + + Note: The value is ignored when an HTTP proxy is in use. + items: + type: string + default: + - 127.0.0.0/8 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 192.0.0.0/24 + - 169.254.0.0/16 + - 192.88.99.0/24 + - 198.18.0.0/15 + - 192.0.2.0/24 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 224.0.0.0/4 + - "::1/128" + - "fe80::/10" + - "fc00::/7" + - "2001:db8::/32" + - "ff00::/8" + - "fec0::/10" + ip_range_whitelist: + type: array + description: >- + List of IP address CIDR ranges that should be allowed for federation, + identity servers, push servers, and for checking key validity for + third-party invite events. This is useful for specifying exceptions to + wide-ranging blacklisted target IP ranges – e.g. for communication with a + push server only visible in your network. + + + This whitelist overrides `ip_range_blacklist`. + items: + type: string + default: [] + examples: + - - 192.168.1.1 + listeners: + type: array + description: >- + List of ports that Synapse should listen on, their purpose and their + configuration. + + + Valid resource names are: + + + * `client`: the client-server API (/_matrix/client). Also implies `media` + and `static`. If configuring the main process, the Synapse Admin API + (/_synapse/admin) is also implied. + + + * `consent`: user consent forms (/_matrix/consent). See + [here](../../consent_tracking.md) for more. + + + * `federation`: the server-server API (/_matrix/federation). Also implies + `media`, `keys`, `openid` + + + * `keys`: the key discovery API (/_matrix/key). + + + * `media`: the media API (/_matrix/media). + + + * `metrics`: the metrics interface. See [here](../../metrics-howto.md). + (Not compatible with Unix sockets) + + + * `openid`: OpenID authentication. See [here](../../openid.md). + + + * `replication`: the HTTP replication API (/_synapse/replication). See + [here](../../workers.md). + + + * `static`: static resources under synapse/static (/_matrix/static). + (Mostly useful for "fallback authentication".) + + + * `health`: the [health check + endpoint](../../reverse_proxy.md#health-check-endpoint). This endpoint is + by default active for all other resources and does not have to be + activated separately. This is only useful if you want to use the health + endpoint explicitly on a dedicated port or for [workers](../../workers.md) + and containers without listener e.g. [application + services](../../workers.md#notifying-application-services). + items: + type: object + properties: + port: + type: integer + description: The TCP port to bind to. + tag: + type: ["string", "null"] + description: >- + An alias for the port in the logger name. If set the tag is logged + instead of the port. Default to `None`, is optional and only valid + for listener with `type: http`. See the docs [request log + format](../administration/request_log.md). + bind_addresses: + type: ["array", "null"] + description: >- + A list of local addresses to listen on. The default is "all local + interfaces". + items: + type: string + type: + type: string + description: >- + The type of listener. Normally `http`, but other valid options are + [`manhole`](../../manhole.md) and + [`metrics`](../../metrics-howto.md). + enum: + - http + - manhole + - metrics + tls: + type: boolean + description: >- + Set to true to enable TLS for this listener. Will use the TLS + key/cert specified in tls_private_key_path/tls_certificate_path. + x_forwarded: + type: boolean + description: >- + Only valid for an `http` listener. Set to true to use the + X-Forwarded-For header as the client IP. Useful when Synapse is + behind a [reverse-proxy](../../reverse_proxy.md). + request_id_header: + type: ["string", "null"] + description: >- + The header extracted from each incoming request that is used as the + basis for the request ID. The request ID is used in + [logs](../administration/request_log.md#request-log-format) and + tracing to correlate and match up requests. When unset, Synapse will + automatically generate sequential request IDs. This option is useful + when Synapse is behind a [reverse-proxy](../../reverse_proxy.md). + + + _Added in Synapse 1.68.0._ + resources: + type: array + description: >- + Only valid for an `http` listener. A list of resources to host on this port. + items: + type: object + properties: + names: + type: array + description: >- + A list of names of HTTP resources. See below for a list of + valid resource names. + items: + type: string + enum: + - client + - consent + - federation + - keys + - media + - metrics + - openid + - replication + - static + - health + compress: + type: boolean + description: >- + Set to true to enable gzip compression on HTTP bodies for this + resource. This is currently only supported with the `client`, + `consent`, `metrics` and `federation` resources. + additional_resources: + type: object + description: >- + Only valid for an `http` listener. A map of additional endpoints + which should be loaded via dynamic modules. + additionalProperties: + type: object + properties: + module: + type: string + config: + type: object + path: + type: string + description: >- + A path and filename for a Unix socket. Make sure it is located in a + directory with read and write permissions, and that it already + exists (the directory will not be created). Defaults to `None`. + + * **Note**: The use of both `path` and `port` options for the same + `listener` is not compatible. + + * The `x_forwarded` option defaults to true when using Unix sockets + and can be omitted. + + * Other options that would not make sense to use with a UNIX socket, + such as `bind_addresses` and `tls` will be ignored and can be + removed. + + + _Added in Synapse 1.89.0_: Unix socket support + mode: + type: ["integer", "null"] + description: >- + The file permissions to set on the UNIX socket. Defaults to `666` if + unset or null. + + + **Note:** Must be set as `type: http` (does not support `metrics` + and `manhole`). Also make sure that `metrics` is not included in + `resources` -> `names` + + + _Added in Synapse 1.89.0_: Unix socket support + default: [] + examples: + - - port: 8448 + type: http + tls: true + resources: + - names: + - client + - federation + - - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: + - "::1" + - 127.0.0.1 + resources: + - names: + - client + - federation + compress: false + additional_resources: + /_matrix/my/custom/endpoint: + module: my_module.CustomRequestHandler + config: {} + - port: 9000 + bind_addresses: + - "::1" + - 127.0.0.1 + type: manhole + - - path: /run/synapse/main_public.sock + type: http + resources: + - names: + - client + - federation + manhole: + type: ["integer", "null"] + description: >- + Turn on the Twisted telnet manhole service on the given port. + + + This can also be set by the `--manhole` argument when starting Synapse. + default: null + examples: + - 1234 + manhole_settings: + type: object + description: >- + Connection settings for the manhole. You can find more information on the + manhole [here](../../manhole.md). + properties: + username: + type: ["string", "null"] + description: The username for the manhole. This defaults to "matrix". + password: + type: ["string", "null"] + description: The password for the manhole. This defaults to "rabbithole". + ssh_priv_key_path: + type: ["string", "null"] + description: >- + The private SSH key used to encrypt the manhole traffic. If left + unset, then hardcoded and non-secret keys are used, which could allow + traffic to be intercepted if sent over a public network. + ssh_pub_key_path: + type: ["string", "null"] + description: >- + The public SSH key corresponsing to `ssh_priv_key_path`. If left + unset, a hardcoded key is used. + examples: + - username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub + dummy_events_threshold: + type: integer + description: >- + Forward extremities can build up in a room due to networking delays + between homeservers. Once this happens in a large room, calculation of the + state of that room can become quite expensive. To mitigate this, once the + number of forward extremities reaches a given threshold, Synapse will send + an `org.matrix.dummy_event` event, which will reduce the forward + extremities in the room. + + + This setting defines the threshold (i.e. number of forward extremities in + the room) at which dummy events are sent. + default: 10 + examples: + - 5 + delete_stale_devices_after: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + An optional duration. If set, Synapse will run a daily background task to + log out and delete any device that hasn't been accessed for more than the + specified amount of time. + + + A value of null means devices are never pruned. + + + **Note:** This task will always run on the main process, regardless of the + value of `run_background_tasks_on`. This is due to workers currently not + having the ability to delete devices. + default: null + examples: + - 1y + email: + type: object + description: >- + Configuration for sending emails from Synapse. + + + Server admins can configure custom templates for email content. See + [here](../../templates.md) for more information. + properties: + smtp_host: + type: string + description: The hostname of the outgoing SMTP server to use. + default: localhost + smtp_port: + type: ["string", "null"] + description: >- + The port on the mail server for outgoing SMTP. If null or unset, 465 + is used if `force_tls` is true, else 25. + + + _Changed in Synapse 1.64.0:_ the default port is now aware of + `force_tls`. + default: null + smtp_user: + type: ["string", "null"] + description: Username for authentication to the SMTP server. + default: null + smtp_pass: + type: ["string", "null"] + description: Password for authentication to the SMTP server. + default: null + force_tls: + type: boolean + description: >- + By default, Synapse connects over plain text and then optionally + upgrades to TLS via STARTTLS. If this option is set to true, TLS is + used from the start (Implicit TLS), and the option + `require_transport_security` is ignored. It is recommended to enable + this if supported by your mail server. + + + _New in Synapse 1.64.0._ + default: false + require_transport_security: + type: boolean + description: >- + Set to true to require TLS transport security for SMTP. By default, + Synapse will connect over plain text, and will then switch to TLS via + STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. + default: false + enable_tls: + type: boolean + description: >- + By default, if the server supports TLS, it will be used, and the + server must present a certificate that is valid for `tlsname`. If this + option is set to false, TLS will not be used. + default: true + tlsname: + type: string + description: >- + The domain name the SMTP server's TLS certificate must be valid for, + defaulting to `smtp_host`. + notif_from: + type: ["string", "null"] + description: >- + Defines the "From" address to use when sending emails. It must be set + if email sending is enabled. The placeholder `%(app)s` will be + replaced by the application name, which is normally set in `app_name`, + but may be overridden by the Matrix client application. Note that the + placeholder must be written `%(app)s`, including the trailing 's'. + default: null + app_name: + type: string + description: >- + Defines the default value for `%(app)s` in `notif_from` and email subjects. + default: Matrix + enable_notifs: + type: boolean + description: >- + Set to true to allow users to receive e-mail notifications. If this is + not set, users can configure e-mail notifications but will not receive + them. + default: false + notif_for_new_users: + type: boolean + description: >- + Set to false to disable automatic subscription to email notifications + for new users. + default: true + notif_delay_before_mail: + $ref: "#/$defs/duration" + description: >- + The time to wait before emailing about a notification. This gives the + user a chance to view the message via push or an open client. + + + _New in Synapse 1.99.0._ + default: 10m + client_base_url: + type: string + description: >- + Custom URL for client links within the email notifications. (This + setting used to be called `riot_base_url`; the old name is still + supported for backwards-compatibility but is now deprecated.) + default: "https://matrix.to" + validation_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Configures the time that a validation email will expire after sending. + default: 1h + invite_client_location: + type: ["string", "null"] + description: >- + The web client location to direct users to during an invite. This is + passed to the identity server as the `org.matrix.web_client_location` + key. If null or unset no guidance is given to the identity server. + default: null + subjects: + type: object + description: >- + Subjects to use when sending emails from Synapse. The placeholder + `%(app)s` will be replaced with the value of the `app_name` setting, + or by a value dictated by the Matrix client application. In addition, + each subject can use the following placeholders: `%(person)s`, which + will be replaced by the displayname of the user(s) that sent the + message(s), e.g. "Alice and Bob", and `%(room)s`, which will be + replaced by the name of the room the message(s) have been sent to, + e.g. "My super room". In addition, emails related to account + administration will can use the `%(server_name)s` placeholder, which + will be replaced by the value of the `server_name` setting in your + Synapse configuration. + properties: + message_from_person_in_room: + type: string + description: >- + Subject to use to notify about one message from one or more + user(s) in a room which has a name. + default: >- + [%(app)s] You have a message on %(app)s from %(person)s in the + %(room)s room... + message_from_person: + type: string + description: >- + Subject to use to notify about one message from one or more + user(s) in a room which doesn't have a name. + default: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: + type: string + description: >- + Subject to use to notify about multiple messages from one or more + users in a room which doesn't have a name. + default: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: + type: string + description: >- + Subject to use to notify about multiple messages in a room which + has a name. + default: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: + type: string + description: >- + Subject to use to notify about multiple messages in multiple rooms. + default: >- + [%(app)s] You have messages on %(app)s in the %(room)s room and others... + messages_from_person_and_others: + type: string + description: >- + Subject to use to notify about multiple messages from multiple + persons in multiple rooms. This is similar to the setting above + except it's used when the room in which the notification was + triggered has no name. + default: >- + [%(app)s] You have messages on %(app)s from %(person)s and others... + invite_from_person_to_room: + type: string + description: >- + Subject to use to notify about an invite to a room which has a name. + default: >- + [%(app)s] %(person)s has invited you to join the %(room)s room on + %(app)s... + invite_from_person: + type: string + description: >- + Subject to use to notify about an invite to a room which doesn't + have a name. + default: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: + type: string + description: Subject to use when sending a password reset email. + default: "[%(server_name)s] Password reset" + email_validation: + type: string + description: >- + Subject to use when sending a verification email to assert an + address's ownership. + default: "[%(server_name)s] Validate your email" + examples: + - smtp_host: mail.server + smtp_port: 587 + smtp_user: exampleusername + smtp_pass: examplepassword + force_tls: true + require_transport_security: true + enable_tls: false + tlsname: mail.server.example.com + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: "https://app.element.io" + subjects: + message_from_person_in_room: >- + [%(app)s] You have a message on %(app)s from %(person)s in the + %(room)s room... + message_from_person: >- + [%(app)s] You have a message on %(app)s from %(person)s... + messages_from_person: >- + [%(app)s] You have messages on %(app)s from %(person)s... + messages_in_room: >- + [%(app)s] You have messages on %(app)s in the %(room)s room... + messages_in_room_and_others: >- + [%(app)s] You have messages on %(app)s in the %(room)s room and others... + messages_from_person_and_others: >- + [%(app)s] You have messages on %(app)s from %(person)s and others... + invite_from_person_to_room: >- + [%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s... + invite_from_person: >- + [%(app)s] %(person)s has invited you to chat on %(app)s... + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" + max_event_delay_duration: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + The maximum allowed duration by which sent events can be delayed, as + per + [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140). + Must be a positive value if set. + + + If null or unset, sending of delayed events is disallowed. + default: null + examples: + - 24h + admin_contact: + type: ["string", "null"] + description: How to reach the server admin, used in `ResourceLimitError`. + default: null + examples: + - "mailto:admin@server.com" + hs_disabled: + type: boolean + description: >- + Blocks users from connecting to the homeserver and provides the + human-readable reason given in `hs_disabled_message`. + default: false + examples: + - true + hs_disabled_message: + type: string + description: Human-readable reason why the connection was blocked. + default: Homeserver is currently blocked + examples: + - Reason for why the HS is blocked + limit_usage_by_mau: + type: boolean + description: >- + This option disables/enables monthly active user blocking. Used in cases + where the admin or server owner wants to limit to the number of monthly + active users. When enabled and a limit is reached the server returns a + `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. If + this is enabled, a value for `max_mau_value` must also be set. + + + See [Monthly Active Users](../administration/monthly_active_users.md) for + details on how to configure MAU. + default: false + examples: + - true + max_mau_value: + type: integer + description: >- + This option sets the hard limit of monthly active users above which the + server will start blocking user actions if `limit_usage_by_mau` is + enabled. + default: 0 + examples: + - 50 + mau_trial_days: + type: integer + description: >- + The option `mau_trial_days` is a means to add a grace period for active + users. It means that users must be active for the specified number of days + before they can be considered active and guards against the case where + lots of users sign up in a short space of time never to return after their + initial session. + default: 0 + examples: + - 5 + mau_appservice_trial_days: + type: object + description: >- + The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but + applies a different trial number if the user was registered by an + appservice. A value of 0 means no trial days are applied. Appservices not + listed in this dictionary use the value of `mau_trial_days` instead. + additionalProperties: + type: integer + default: {} + examples: + - my_appservice_id: 3 + another_appservice_id: 6 + mau_limit_alerting: + type: boolean + description: >- + Limit client-side alerting should the mau limit be reached. This is useful + for small instances where the admin has 5 mau seats (say) for 5 specific + people and no interest increasing the mau limit further. + default: true + examples: + - false + mau_stats_only: + type: boolean + description: >- + If enabled, the metrics for the number of monthly active users will be + populated, however no one will be limited based on these numbers. If + `limit_usage_by_mau` is true, this is implied to be true. + default: false + examples: + - true + mau_limit_reserved_threepids: + type: array + description: >- + Sometimes the server admin will want to ensure certain accounts are never + blocked by mau checking. These accounts are specified by this option. Add + accounts by specifying the `medium` and `address` of the reserved threepid + (3rd party identifier). + items: + type: object + properties: + medium: + type: string + description: Medium of the account threepid. + address: + type: string + description: Address of the account threepid. + default: [] + examples: + - - medium: email + address: reserved_user@example.com + server_context: + type: ["string", "null"] + description: >- + This option is used by phonehome stats to group together related servers. + default: null + examples: + - context + limit_remote_rooms: + type: object + description: >- + When this option is enabled, the room "complexity" will be checked before + a user joins a new remote room. If it is above the complexity limit, the + server will disallow joining, or will instantly leave. This is useful for + homeservers that are resource-constrained. Room complexity is an arbitrary + measure based on factors such as the number of users in the room. + properties: + enabled: + type: boolean + description: Whether this check is enabled. + default: false + complexity: + type: number + description: The limit above which rooms cannot be joined. + default: 1.0 + complexity_error: + type: string + description: >- + Override the error which is returned when the room is too complex with + a custom message. + default: >- + Your homeserver is unable to join rooms this large or complex. Please + speak to your server administrator, or upgrade your instance to join + this room. + admins_can_join: + type: boolean + description: Allow server admins to join complex rooms. + default: false + examples: + - enabled: true + complexity: 0.5 + complexity_error: I can't let you do that, Dave. + admins_can_join: true + require_membership_for_aliases: + type: boolean + description: Whether to require a user to be in the room to add an alias to it. + default: true + examples: + - false + allow_per_room_profiles: + type: boolean + description: >- + Whether to allow per-room membership profiles through the sending of + membership events with profile information that differs from the target's + global profile. + default: true + examples: + - false + max_avatar_size: + oneOf: + - $ref: "#/$defs/bytes" + - type: "null" + description: >- + The largest permissible file size in bytes for a user avatar. Defaults to + no restriction. Use M for MB and K for KB. + + + Note that user avatar changes will not work if this is set without using + Synapse's media repository. + default: null + examples: + - 10M + allowed_avatar_mimetypes: + type: ["array", "null"] + description: >- + The MIME types allowed for user avatars. Defaults to no restriction. + + + Note that user avatar changes will not work if this is set without using + Synapse's media repository. + items: + type: string + default: null + examples: + - - image/png + - image/jpeg + - image/gif + redaction_retention_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to keep redacted events in unredacted form in the database. After + this period redacted events get replaced with their redacted form in the + DB. + + + Synapse will check whether the rentention period has concluded for + redacted events every 5 minutes. Thus, even if this option is set to `0`, + Synapse may still take up to 5 minutes to purge redacted events from the + database. Set to `null` to disable. + default: 7d + examples: + - 28d + forgotten_room_retention_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to keep locally forgotten rooms before purging them from the DB. + A value of `null` means it's disabled. + default: null + examples: + - 28d + user_ips_max_age: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long to track users' last seen time and IPs in the database. Set to + `null` to disable clearing out of old rows. + default: 28d + examples: + - 14d + request_token_inhibit_3pid_errors: + type: boolean + description: >- + Inhibits the `/requestToken` endpoints from returning an error that might + leak information about whether an e-mail address is in use or not on this + homeserver. Note that for some endpoints the error situation is the e-mail + already being used, and for others the error is entering the e-mail being + unused. If this option is enabled, instead of returning an error, these + endpoints will act as if no error happened and return a fake session ID + (`sid`) to clients. + default: false + examples: + - true + next_link_domain_whitelist: + type: ["array", "null"] + description: >- + A list of domains that the domain portion of `next_link` parameters must + match. + + + This parameter is optionally provided by clients while requesting + validation of an email or phone number, and maps to a link that users will + be automatically redirected to after validation succeeds. Clients can make + use this parameter to aid the validation process. + + + The whitelist is applied whether the homeserver or an identity server is + handling validation. + + + The default value is no whitelist functionality; all domains are allowed. + Setting this value to an empty list will instead disallow all domains. + default: null + examples: + - matrix.org + templates: + type: object + description: >- + These options define templates to use when generating email or HTML page + contents. + + + See [here](../../templates.md) for more information about using custom + templates. + properties: + custom_template_directory: + type: ["string", "null"] + description: >- + Determines which directory Synapse will try to find template files in + to use to generate email or HTML page contents. If not set, or a file + is not found within the template directory, a default template from + within the Synapse package will be used. + default: null + examples: + - custom_template_directory: /path/to/custom/templates/ + retention: + type: object + description: >- + This option and the associated options determine message retention policy + at the server level. + + + Room admins and mods can define a retention period for their rooms using + the `m.room.retention` state event, and server admins can cap this period + by setting the `allowed_lifetime_min` and `allowed_lifetime_max` config + options. + + + If this feature is enabled, Synapse will regularly look for and purge + events which are older than the room's maximum retention period. Synapse + will also filter events received over federation so that events that + should have been purged are ignored and not stored again. + + + The message retention policies feature is disabled by default. You can + read more about this feature [here](../../message_retention_policies.md). + properties: + enabled: + type: boolean + description: Enforce message retention policies + default: false + default_policy: + type: object + description: >- + Default message retention policy. If set, Synapse will apply it to + rooms that lack the `m.room.retention` state event. + properties: + min_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Minimum message retention time of the default message retention + policy. Synapse doesn't take this option into account yet. + default: null + max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Maximum message retention time of the default message retention policy. + default: null + allowed_lifetime_min: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Retention policy limit. If set, and the state of a room contains a + `m.room.retention` event in its state which contains a `min_lifetime` + that's beyond this bound, Synapse will cap the room's policy to these + limits when running purge jobs. + default: null + allowed_lifetime_max: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Retention policy limit. If set, and the state of a room contains a + `m.room.retention` event in its state which contains a `max_lifetime` + that's beyond this bound, Synapse will cap the room's policy to these + limits when running purge jobs. + default: null + purge_jobs: + type: ["array", "null"] + description: >- + Server admins can define the settings of the background jobs purging + the events whose lifetime has expired under the `purge_jobs` section. + + + If no configuration is provided for this option, a single job will be + set up to delete expired events in every room daily. + + + Each job's configuration defines which range of message lifetimes the + job takes care of. For example, if `shortest_max_lifetime` is "2d" and + `longest_max_lifetime` is "3d", the job will handle purging expired + events in rooms whose state defines a `max_lifetime` that's both + higher than 2 days, and lower than or equal to 3 days. Both the + minimum and the maximum value of a range are optional, e.g. a job with + no `shortest_max_lifetime` and a `longest_max_lifetime` of "3d" will + handle every room with a retention policy whose `max_lifetime` is + lower than or equal to three days. + + + The rationale for this per-job configuration is that some rooms might + have a retention policy with a low `max_lifetime`, where history needs + to be purged of outdated messages on a more frequent basis than for + the rest of the rooms (e.g. every 12h), but not want that purge to be + performed by a job that's iterating over every room it knows, which + could be heavy on the server. + + + If any purge job is configured, it is strongly recommended to have at + least a single job with neither `shortest_max_lifetime` nor + `longest_max_lifetime` set, or one job without `shortest_max_lifetime` + and one job without `longest_max_lifetime` set. Otherwise some rooms + might be ignored, even if `allowed_lifetime_min` and + `allowed_lifetime_max` are set, because capping a room's policy to + these values is done after the policies are retrieved from Synapse's + database (which is done using the range specified in a purge job's + configuration). + items: + type: object + properties: + shortest_max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Apply job to rooms that have a `max_lifetime` higher than + `shortest_max_lifetime`. A value of `null` never excludes any + room. + longest_max_lifetime: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Apply job to rooms that have a `max_lifetime` lower than or + equal to `shortest_max_lifetime`. A value of `null` never + excludes any room. + interval: + $ref: "#/$defs/duration" + description: How often to run the job. + default: null + examples: + - enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d + tls_certificate_path: + type: ["string", "null"] + description: >- + This option specifies a PEM-encoded X509 certificate for TLS. This + certificate, as of Synapse 1.0, will need to be a valid and verifiable + certificate, signed by a recognised Certificate Authority. + + + Be sure to use a `.pem` file that includes the full certificate chain + including any intermediate certificates (for instance, if using certbot, + use `fullchain.pem` as your certificate, not `cert.pem`). + default: null + examples: + - CONFDIR/SERVERNAME.tls.crt + tls_private_key_path: + type: ["string", "null"] + description: PEM-encoded private key for TLS. + default: null + examples: + - CONFDIR/SERVERNAME.tls.key + federation_verify_certificates: + type: boolean + description: >- + Whether to verify TLS server certificates for outbound federation + requests. To disable certificate verification, set the option to false. + default: true + examples: + - false + federation_client_minimum_tls_version: + type: string + description: >- + The minimum TLS version that will be used for outbound federation + requests. + + + Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note that setting + this value higher than `"1.2"` will prevent federation to most of the + public Matrix network: only configure it to `"1.3"` if you have an + entirely private federation setup and you can ensure TLS 1.3 support. + default: "1" + examples: + - "1.2" + federation_certificate_verification_whitelist: + type: array + description: >- + Skip federation certificate verification on a given whitelist of domains. + + + This setting should only be used in very specific cases, such as + federation over Tor hidden services and similar. For private networks of + homeservers, you likely want to use a private CA instead. + + + Only effective if `federation_verify_certificates` is `true`. + items: + type: string + default: [] + examples: + - - lon.example.com + - "*.domain.com" + - "*.onion" + federation_custom_ca_list: + type: array + description: >- + List of custom certificate authorities for federation traffic. + + + This setting should only normally be used within a private network of + homeservers. + + + Note that this list will replace those that are provided by your operating + environment. Certificates must be in PEM format. + items: + type: string + default: [] + examples: + - - myCA1.pem + - myCA2.pem + - myCA3.pem + federation_domain_whitelist: + type: array + description: >- + Restrict federation to the given whitelist of domains. N.B. we recommend + also firewalling your federation listener to limit inbound federation + traffic as early as possible, rather than relying purely on this + application-layer restriction. If not specified, the default is to + whitelist everything. + + + Note: this does not stop a server from joining rooms that servers not on + the whitelist are in. As such, this option is really only useful to + establish a "private federation", where a group of servers all whitelist + each other and have the same whitelist. + items: + type: string + default: [] + examples: + - - lon.example.com + - nyc.example.com + - syd.example.com + federation_whitelist_endpoint_enabled: + type: boolean + description: >- + Enables an endpoint for fetching the federation whitelist config. + + + The request method and path is `GET + /_synapse/client/v1/config/federation_whitelist`, and the response format + is: + + + ```json + + { + "whitelist_enabled": true, // Whether the federation whitelist is being enforced + "whitelist": [ // Which server names are allowed by the whitelist + "example.com" + ] + } + + ``` + + + If `whitelist_enabled` is `false` then the server is permitted to federate + with all others. + + + The endpoint requires authentication. + default: false + examples: + - true + federation_metrics_domains: + type: array + description: >- + Report prometheus metrics on the age of PDUs being sent to and received + from the given domains. This can be used to give an idea of "delay" on + inbound and outbound federation, though be aware that any delay can be due + to problems at either end or with the intermediate network. + items: + type: string + default: [] + examples: + - - matrix.org + - example.com + allow_profile_lookup_over_federation: + type: boolean + description: >- + Set to false to disable profile lookup over federation. By default, the + Federation API allows other homeservers to obtain profile data of any user + on this homeserver. + default: true + examples: + - false + allow_device_name_lookup_over_federation: + type: boolean + description: >- + Set this option to true to allow device display name lookup over + federation. By default, the Federation API prevents other homeservers from + obtaining the display names of any user devices on this homeserver. + default: false + examples: + - true + federation: + type: object + description: >- + The federation section defines some sub-options related to federation. + + + The following options are related to configuring timeout and retry logic + for one request, independently of the others. Short retry algorithm is + used when something or someone will wait for the request to have an + answer, while long retry is used for requests that happen in the + background, like sending a federation transaction. + + + `destination_*` options control the retry logic when communicating with a + specific homeserver destination. Unlike the previous configuration + options, these values apply across all requests for a given destination + and the state of the backoff is stored in the database. + properties: + client_timeout: + $ref: "#/$defs/duration" + description: Timeout for the federation requests. + default: 60s + max_short_retry_delay: + $ref: "#/$defs/duration" + description: Maximum delay to be used for the short retry algo. + default: 2s + max_long_retry_delay: + $ref: "#/$defs/duration" + description: Maximum delay to be used for the long retry algo. + default: 60s + max_short_retries: + type: integer + description: Maximum number of retries for the short retry algo. + default: 3 + max_long_retries: + type: integer + description: Maximum number of retries for the long retry algo. + default: 10 + destination_min_retry_interval: + $ref: "#/$defs/duration" + description: "The initial backoff, after the first request fails." + default: 10m + destination_retry_multiplier: + type: integer + description: >- + How much we multiply the backoff by after each subsequent fail. + default: 2 + destination_max_retry_interval: + $ref: "#/$defs/duration" + description: A cap on the backoff. + default: 1w + examples: + - client_timeout: 180s + max_short_retry_delay: 7s + max_long_retry_delay: 100s + max_short_retries: 5 + max_long_retries: 20 + destination_min_retry_interval: 30s + destination_retry_multiplier: 5 + destination_max_retry_interval: 12h + event_cache_size: + $ref: "#/$defs/size" + description: >- + The number of events to cache in memory. Defaults to 10K. Like other + caches, this is affected by `caches.global_factor` (see below). + + + For example, the default is 10K and the global_factor default is 0.5. + + + Since 10K * 0.5 is 5K then the event cache size will be 5K. + + + The cache affected by this configuration is named as "\*getEvent\*". + + + Note that this option is not part of the `caches` section. + default: 10K + examples: + - 15K + caches: + type: object + description: >- + A cache "factor" is a multiplier that can be applied to each of Synapse's + caches in order to increase or decrease the maximum number of entries that + can be stored. + io.element.post_description: >- + ### Reloading cache factors + + + The cache factors (i.e. `caches.global_factor` and + `caches.per_cache_factors`) may be reloaded at any time by sending a + [`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse + using e.g. + + + ```commandline + + kill -HUP [PID_OF_SYNAPSE_PROCESS] + + ``` + + + If you are running multiple workers, you must individually update the + worker config file and send this signal to each worker process. + + + If you're using the [example systemd + service](https://github.com/element-hq/synapse/blob/develop/contrib/systemd/matrix-synapse.service) + file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by + using `systemctl reload matrix-synapse`. + properties: + global_factor: + type: number + description: >- + Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over setting + through the config file. + + + Defaults to 0.5, which will halve the size of all caches. + + + Note that changing this value also affects the HTTP connection pool. + default: 0.5 + per_cache_factors: + type: object + description: >- + A dictionary of cache name to cache factor for that individual cache. + Overrides the global cache factor for a given cache. + + + These can also be set through environment variables comprised of + `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital letters and + underscores. Setting by environment variable takes priority over + setting through the config file. Ex. + `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + + Some caches have '*' and other characters that are not alphanumeric or + underscores. These caches can be named with or without the special + characters stripped. For example, to specify the cache factor for + `*stateGroupCache*` via an environment variable would be + `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + additionalProperties: + type: number + default: {} + expire_caches: + type: boolean + description: >- + Controls whether cache entries are evicted after a specified time + period. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + default: true + cache_entry_ttl: + $ref: "#/$defs/duration" + description: >- + If `expire_caches` is enabled, this flag controls how long an entry + can be in a cache without having been accessed before being evicted. + default: 30m + sync_response_cache_duration: + $ref: "#/$defs/duration" + description: >- + Controls how long the results of a /sync request are cached for after + a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. A + value of zero means that sync responses are not cached. + + + *Changed in Synapse 1.62.0*: The default was changed from 0 to 2m. + default: 2m + cache_autotuning: + type: object + description: >- + `cache_autotuning` and its sub-options `max_cache_memory_usage`, + `target_cache_memory_usage`, and `min_cache_ttl` work in conjunction + with each other to maintain a balance between cache memory usage and + cache entry availability. You must be using + [jemalloc](../administration/admin_faq.md#help-synapse-is-slow-and-eats-all-my-ramcpu) + to utilize this option, and all three of the options must be specified + for this feature to work. This option defaults to off, enable it by + providing values for the sub-options listed below. Please note that + the feature will not work and may cause unstable behavior (such as + excessive emptying of caches or exceptions) if all of the values are + not provided. Please see the [Config Conventions](#config-conventions) + for information on how to specify memory size and cache expiry + durations. + properties: + max_cache_memory_usage: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Sets a ceiling on how much memory the cache can use before caches + begin to be continuously evicted. They will continue to be evicted + until the memory usage drops below the + `target_cache_memory_usage`, set in the setting below, or until + the `min_cache_ttl` is hit. + default: null + target_cache_memory_usage: + oneOf: + - $ref: "#/$defs/bytes" + - type: "null" + description: Sets a rough target for the desired memory usage of the caches. + default: null + min_cache_ttl: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + Sets a limit under which newer cache entries are not evicted and + is only applied when caches are actively being + evicted/`max_cache_memory_usage` has been exceeded. This is to + protect hot caches from being emptied while Synapse is evicting + due to memory. + default: null + examples: + - global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + sync_response_cache_duration: 2m + cache_autotuning: + max_cache_memory_usage: 1024M + target_cache_memory_usage: 758M + min_cache_ttl: 5m + database: + $ref: "#/$defs/database" + examples: + - name: sqlite3 + args: + database: /path/to/homeserver.db + - name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + dbname: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + databases: + type: object + description: >- + The `databases` option allows specifying a mapping between certain + database tables and database host details, spreading the load of a single + Synapse instance across multiple database backends. This is often referred + to as "database sharding". This option is only supported for PostgreSQL + database backends. + + + **Important note:** This is a supported option, but is not currently used + in production by the Matrix.org Foundation. Proceed with caution and + always make backups. + + + `databases` is a dictionary of arbitrarily-named database entries. Each + entry is equivalent to the value of the `database` homeserver config + option (see above), with the addition of a `data_stores` key. + `data_stores` is an array of strings that specifies the data store(s) (a + defined label for a set of tables) that should be stored on the associated + database backend entry. + + + The currently defined values for `data_stores` are: + + + * `"state"`: Database that relates to state groups will be stored in this + database. + + Specifically, that means the following tables: + * `state_groups` + * `state_group_edges` + * `state_groups_state` + + And the following sequences: + * `state_groups_seq_id` + + * `"main"`: All other database tables and sequences. + + + All databases will end up with additional tables used for tracking + database schema migrations and any pending background updates. Synapse + will create these automatically on startup when checking for and/or + performing database schema migrations. + + + To migrate an existing database configuration (e.g. all tables on a single + database) to a different configuration (e.g. the "main" data store on one + database, and "state" on another), do the following: + + + 1. Take a backup of your existing database. Things can and do go wrong and + database corruption is no joke! + + 2. Ensure all pending database migrations have been applied and background + updates have run. The simplest way to do this is to use the + `update_synapse_database` script supplied with your Synapse installation. + + ```sh + update_synapse_database --database-config homeserver.yaml --run-background-updates + ``` + + 3. Copy over the necessary tables and sequences from one database to the + other. Tables relating to database migrations, schemas, schema versions + and background updates should **not** be copied. + + As an example, say that you'd like to split out the "state" data store from an existing database which currently contains all data stores. + + Simply copy the tables and sequences defined above for the "state" datastore from the existing database to the secondary database. As noted above, additional tables will be created in the secondary database when Synapse is started. + + 4. Modify/create the `databases` option in your `homeserver.yaml` to match + the desired database configuration. + + 5. Start Synapse. Check that it starts up successfully and that things + generally seem to be working. + + 6. Drop the old tables that were copied in step 3. + + + Only one of the options `database` or `databases` may be specified in your + config, but not both. + additionalProperties: + $ref: "#/$defs/database" + properties: + data_stores: + type: array + items: + type: string + enum: + - state + - main + default: {} + examples: + - basement_box: + name: psycopg2 + txn_limit: 10000 + data_stores: + - main + args: + user: synapse_user + password: secretpassword + dbname: synapse_main + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + my_other_database: + name: psycopg2 + txn_limit: 10000 + data_stores: + - state + args: + user: synapse_user + password: secretpassword + dbname: synapse_state + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 + log_config: + type: ["string", "null"] + description: >- + This option specifies a yaml python logging config file as described + [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). + default: null + examples: + - CONFDIR/SERVERNAME.log.config + rc_message: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for client messaging. + + + This is a ratelimiting option for messages that ratelimits sending based + on the account the client is using. + default: + per_second: 0.2 + burst_count: 10.0 + examples: + - per_second: 0.5 + burst_count: 15.0 + rc_registration: + $ref: "#/$defs/rc" + description: >- + This option ratelimits registration requests based on the client's IP address. + default: + per_second: 0.17 + burst_count: 3.0 + examples: + - per_second: 0.15 + burst_count: 2.0 + rc_registration_token_validity: + $ref: "#/$defs/rc" + description: >- + This option checks the validity of registration tokens that ratelimits + requests based on the client's IP address. + default: + per_second: 0.1 + burst_count: 5.0 + examples: + - per_second: 0.3 + burst_count: 6.0 + rc_login: + type: object + description: This option specifies several limits for login. + properties: + address: + $ref: "#/$defs/rc" + description: Ratelimits login requests based on the client's IP address. + default: + per_second: 0.003 + burst_count: 5.0 + account: + $ref: "#/$defs/rc" + description: >- + Ratelimits login requests based on the account the client is + attempting to log into. + default: + per_second: 0.003 + burst_count: 5.0 + failed_attempts: + $ref: "#/$defs/rc" + description: >- + Ratelimits login requests based on the account the client is + attempting to log into, based on the amount of failed login attempts + for this account. + default: + per_second: 0.17 + burst_count: 3.0 + examples: + - address: + per_second: 0.15 + burst_count: 5.0 + account: + per_second: 0.18 + burst_count: 4.0 + failed_attempts: + per_second: 0.19 + burst_count: 7.0 + rc_admin_redaction: + $ref: "#/$defs/rc" + description: >- + This option sets ratelimiting redactions by room admins. If this is not + explicitly set then it uses the same ratelimiting as per `rc_message`. + This is useful to allow room admins to deal with abuse quickly. + examples: + - per_second: 1.0 + burst_count: 50.0 + rc_joins: + type: object + description: This option allows for ratelimiting number of rooms a user can join. + properties: + local: + $ref: "#/$defs/rc" + description: Ratelimits when users are joining rooms the server is already in. + default: + per_second: 0.1 + burst_count: 10.0 + remote: + $ref: "#/$defs/rc" + description: >- + Ratelimits when users are trying to join rooms not on the server + (which can be more computationally expensive than restricting + locally). + default: + per_second: 0.01 + burst_count: 10.0 + examples: + - local: + per_second: 0.2 + burst_count: 15.0 + remote: + per_second: 0.03 + burst_count: 12.0 + rc_joins_per_room: + $ref: "#/$defs/rc" + description: >- + This option allows admins to ratelimit joins to a room based on the number + of recent joins (local or remote) to that room. It is intended to mitigate + mass-join spam waves which target multiple homeservers. + + + _Added in Synapse 1.64.0._ + default: + per_second: 1.0 + burst_count: 10.0 + examples: + - per_second: 1.0 + burst_count: 10.0 + rc_3pid_validation: + $ref: "#/$defs/rc" + description: >- + This option ratelimits how often a user or IP can attempt to validate a 3PID. + default: + per_second: 0.003 + burst_count: 5.0 + examples: + - per_second: 0.003 + burst_count: 5.0 + rc_invites: + type: object + description: >- + This option sets ratelimiting how often invites can be sent in a room or + to a specific user. + + + Client requests that invite user(s) when [creating a + room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) + will count against the `rc_invites.per_room` limit, whereas client + requests to [invite a single user to a + room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) + will count against both the `rc_invites.per_user` and + `rc_invites.per_room` limits. + + + Federation requests to invite a user will count against the + `rc_invites.per_user` limit only, as Synapse presumes ratelimiting by room + will be done by the sending server. + + + _Changed in version 1.63:_ added the `per_issuer` limit. + properties: + per_room: + $ref: "#/$defs/rc" + description: Applies to the room of the invitation. + default: + per_second: 0.3 + burst_count: 10.0 + per_user: + $ref: "#/$defs/rc" + description: >- + Applies to the *receiver* of the invite, rather than the sender, + meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a + single user cannot *receive* more than a burst of 5 invites at a + time. + default: + per_second: 0.003 + burst_count: 5.0 + per_issuer: + $ref: "#/$defs/rc" + description: >- + Applies to the *issuer* of the invite, meaning that a + `rc_invite.per_issuer.burst_count` of 5 mandates that single user + cannot *send* more than a burst of 5 invites at a time. + default: + per_second: 0.3 + burst_count: 10.0 + examples: + - per_room: + per_second: 0.5 + burst_count: 5.0 + per_user: + per_second: 0.004 + burst_count: 3.0 + per_issuer: + per_second: 0.5 + burst_count: 5.0 + rc_third_party_invite: + $ref: "#/$defs/rc" + description: >- + This option ratelimits 3PID invites (i.e. invites sent to a third-party ID + such as an email address or a phone number) based on the account that's + sending the invite. + default: + per_second: 0.2 + burst_count: 10.0 + rc_media_create: + $ref: "#/$defs/rc" + description: >- + This option ratelimits creation of MXC URIs via the + `/_matrix/media/v1/create` endpoint based on the account that's creating + the media. + default: + per_second: 10.0 + burst_count: 50.0 + rc_federation: + type: object + description: Defines limits on federation requests. + properties: + window_size: + type: integer + description: Window size in milliseconds. + default: 1000 + sleep_limit: + type: integer + description: >- + Number of federation requests from a single server in a window before + the server will delay processing the request. + default: 10 + sleep_delay: + type: integer + description: >- + Duration in milliseconds to delay processing events from remote + servers by if they go over the sleep limit. + default: 500 + reject_limit: + type: integer + description: >- + Maximum number of concurrent federation requests allowed from a single server. + default: 50 + concurrent: + type: integer + description: >- + Number of federation requests to concurrently process from a single server. + default: 3 + examples: + - window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 + rc_presence: + type: object + description: This option sets ratelimiting for presence. + properties: + per_user: + $ref: "#/$defs/rc" + description: >- + Sets rate limits on how often a specific users' presence updates are + evaluated. Ratelimited presence updates sent via sync are ignored, and + no error is returned to the client. This option also sets the rate + limit for the [`PUT /_matrix/client/v3/presence/{userId}/status`] + endpoint. + + + [`PUT /_matrix/client/v3/presence/{userId}/status`]: + + default: + per_user: + per_second: 0.1 + burst_count: 1.0 + examples: + - per_user: + per_second: 0.05 + burst_count: 1.0 + rc_delayed_event_mgmt: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for delayed event management. + + + This is a ratelimiting option that ratelimits attempts to restart, cancel, + or view delayed events based on the sending client's account and device + ID. + + + Attempts to create or send delayed events are ratelimited not by this + setting, but by `rc_message`. + + + Setting this to a high value allows clients to make delayed event + management requests often (such as repeatedly restarting a delayed event + with a short timeout, or restarting several different delayed events all + at once) without the risk of being ratelimited. + default: + per_user: + per_second: 1.0 + burst_count: 5.0 + examples: + - per_second: 2.0 + burst_count: 20.0 + federation_rr_transactions_per_room_per_second: + type: integer + description: >- + Sets outgoing federation transaction frequency for sending read-receipts, + per-room. + + + If we end up trying to send out more read-receipts, they will get buffered + up into fewer transactions. + default: 50 + examples: + - 40 + enable_authenticated_media: + type: boolean + description: >- + When set to true, all subsequent media uploads will be marked as + authenticated, and will not be available over legacy unauthenticated media + endpoints (`/_matrix/media/(r0|v3|v1)/download` and + `/_matrix/media/(r0|v3|v1)/thumbnail`) – requests for authenticated media + over these endpoints will result in a 404. All media, including + authenticated media, will be available over the authenticated media + endpoints `_matrix/client/v1/media/download` and + `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this + option to true will still be available over the legacy endpoints. Note if + the setting is switched to false after enabling, media marked as + authenticated will be available over legacy endpoints. Defaults to true + (previously false). In a future release of Synapse, this option will be + removed and become always-on. + + + In all cases, authenticated requests to download media will succeed, but + for unauthenticated requests, this case-by-case breakdown describes + whether media downloads are permitted: + + + * `enable_authenticated_media = False`: + * unauthenticated client or homeserver requesting local media: allowed + * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache, or as long as the remote homeserver does not require authentication to retrieve the media + * `enable_authenticated_media = True`: + * unauthenticated client or homeserver requesting local media: allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + * unauthenticated client or homeserver requesting remote media: the same as for local media; allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist); otherwise denied. + + It is especially notable that media downloaded before this option existed + (in older Synapse versions), or whilst this option was set to `False`, + will perpetually be available over the legacy, unauthenticated endpoint, + even after this option is set to `True`. This is for backwards + compatibility with older clients and homeservers that do not yet support + requesting authenticated media; those older clients or homeservers will + not be cut off from media they can already see. + + + _Changed in Synapse 1.120:_ This option now defaults to `True` when not + set, whereas before this version it defaulted to `False`. + default: true + examples: + - false + enable_media_repo: + type: boolean + description: >- + Enable the media store service in the Synapse master. Set to false if you + are using a separate media store worker. + default: true + examples: + - false + media_store_path: + type: string + description: Directory where uploaded images and attachments are stored. + default: media_store + examples: + - DATADIR/media_store + max_pending_media_uploads: + type: integer + description: >- + How many *pending media uploads* can a given user have? A pending media + upload is a created MXC URI that (a) is not expired (the + `unused_expires_at` timestamp has not passed) and (b) the media has not + yet been uploaded for. + default: 5 + examples: + - 5 + unused_expiration_time: + $ref: "#/$defs/duration" + description: How long to wait in milliseconds before expiring created media IDs. + default: 24h + examples: + - 1h + media_storage_providers: + type: array + description: >- + Media storage providers allow media to be stored in different locations. + items: + type: object + properties: + module: + type: string + description: "Type of resource, e.g. `file_system`." + store_local: + type: boolean + description: Whether to store newly uploaded local files. + store_remote: + type: boolean + description: Whether to store newly downloaded local files. + store_synchronous: + type: boolean + description: Whether to wait for successful storage for local uploads. + config: + type: object + description: Sets a path to the resource through the `directory` option. + properties: + directory: + type: string + description: Path to the resource. + default: [] + examples: + - - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory + max_upload_size: + $ref: "#/$defs/bytes" + description: >- + The largest allowed upload size in bytes. + + + If you are using a reverse proxy you may also need to set this value in + your reverse proxy's config. Notably Nginx has a small max body size by + default. See [here](../../reverse_proxy.md) for more on using a reverse + proxy with Synapse. + default: 50M + examples: + - 60M + max_image_pixels: + $ref: "#/$defs/bytes" + description: Maximum number of pixels that will be thumbnailed. + default: 32M + examples: + - 35M + remote_media_download_burst_count: + $ref: "#/$defs/bytes" + description: >- + Remote media downloads are ratelimited using a [leaky bucket + algorithm](https://en.wikipedia.org/wiki/Leaky_bucket), where a given + "bucket" is keyed to the IP address of the requester when requesting + remote media downloads. This configuration option sets the size of the + bucket against which the size in bytes of downloads are penalized – if the + bucket is full, i.e. a given number of bytes have already been downloaded, + further downloads will be denied until the bucket drains. See also + `remote_media_download_per_second` which determines the rate at which the + "bucket" is emptied and thus has available space to authorize new + requests. + default: 500MiB + examples: + - 200M + remote_media_download_per_second: + $ref: "#/$defs/bytes" + description: >- + Works in conjunction with `remote_media_download_burst_count` to ratelimit + remote media downloads – this configuration option determines the rate at + which the "bucket" (see above) leaks in bytes per second. As requests are + made to download remote media, the size of those requests in bytes is + added to the bucket, and once the bucket has reached it's capacity, no + more requests will be allowed until a number of bytes has "drained" from + the bucket. This setting determines the rate at which bytes drain from the + bucket, with the practical effect that the larger the number, the faster + the bucket leaks, allowing for more bytes downloaded over a shorter period + of time. Defaults to 87KiB per second. See also + `remote_media_download_burst_count`. + default: 87KiB + examples: + - 40K + prevent_media_downloads_from: + type: array + description: >- + A list of domains to never download media from. Media from these domains + that is already downloaded will not be deleted, but will be inaccessible + to users. This option does not affect admin APIs trying to + download/operate on media. + + + This will not prevent the listed domains from accessing media themselves. + It simply prevents users on this server from downloading media originating + from the listed servers. + + + This will have no effect on media originating from the local server. This + only affects media downloaded from other Matrix servers, to control URL + previews see + [`url_preview_ip_range_blacklist`](#url_preview_ip_range_blacklist) or + [`url_preview_url_blacklist`](#url_preview_url_blacklist). + items: + type: string + default: [] + examples: + - - evil.example.org + - evil2.example.org + dynamic_thumbnails: + type: boolean + description: >- + Whether to generate new thumbnails on the fly to precisely match the + resolution requested by the client. If true then whenever a new resolution + is requested by the client the server will generate a new thumbnail. If + false the server will pick a thumbnail from a precalculated list. + default: false + examples: + - true + thumbnail_sizes: + type: array + description: List of thumbnails to precalculate when an image is uploaded. + items: + type: object + properties: + width: + type: integer + description: Width of the generated thumbnail. + height: + type: integer + description: Height of the generated thumbnail. + method: + type: string + enum: + - crop + - scale + description: >- + Method to fit the thumbnail dimensions. Current options are `crop` + and `scale`. + default: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale + media_retention: + type: object + description: >- + Controls whether local media and entries in the remote media cache (media + that is downloaded from other homeservers) should be removed under certain + conditions, typically for the purpose of saving space. + + + Purging media files will be the carried out by the media worker (that is, + the worker that has the `enable_media_repo` homeserver config option set + to `true`). This may be the main process. + + + The `media_retention.local_media_lifetime` and + `media_retention.remote_media_lifetime` config options control whether + media will be purged if it has not been accessed in a given amount of + time. Note that media is "accessed" when loaded in a room in a client, or + otherwise downloaded by a local or remote user. If the media has never + been accessed, the media's creation time is used instead. Both thumbnails + and the original media will be removed. If either of these options are + unset, then media of that type will not be purged. + + + Local or cached remote media that has been + [quarantined](../../admin_api/media_admin_api.md#quarantining-media-in-a-room) + will not be deleted. Similarly, local media that has been marked as + [protected from + quarantine](../../admin_api/media_admin_api.md#protecting-media-from-being-quarantined) + will not be deleted. + properties: + local_media_lifetime: + description: >- + Duration without access to a local media resource after which it will + be purged. If the media has never been accessed, the media's creation + time is used instead. Both thumbnails and the original media will be + removed. If unset or null, local media will not be purged. + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + default: null + remote_media_lifetime: + description: >- + Duration without access to a remote media resource after which it will + be purged. If the media has never been accessed, the media's creation + time is used instead. Both thumbnails and the original media will be + removed. If unset or null, remote media will not be purged. + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + default: null + examples: + - local_media_lifetime: 90d + remote_media_lifetime: 14d + url_preview_enabled: + type: boolean + description: >- + This setting determines whether the preview URL API is enabled. Set to + true to enable. If enabled you must specify a + `url_preview_ip_range_blacklist` blacklist. + default: false + examples: + - true + url_preview_ip_range_blacklist: + type: ["array", "null"] + description: >- + List of IP address CIDR ranges that the URL preview spider is denied from + accessing. There are no defaults: you must explicitly specify a list for + URL previewing to work. You should specify any internal services in your + network that you do not want synapse to try to connect to, otherwise + anyone in any Matrix room could cause your synapse to issue arbitrary GET + requests to your internal services, causing serious security issues. + + + (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly + listed here, since they correspond to unroutable addresses.) + + + This must be specified if `url_preview_enabled` is set. It is recommended + that you use the following example list as a starting point. + + + Note: The value is ignored when an HTTP proxy is in use. + items: + type: string + default: null + examples: + - - 127.0.0.0/8 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 192.0.0.0/24 + - 169.254.0.0/16 + - 192.88.99.0/24 + - 198.18.0.0/15 + - 192.0.2.0/24 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 224.0.0.0/4 + - "::1/128" + - "fe80::/10" + - "fc00::/7" + - "2001:db8::/32" + - "ff00::/8" + - "fec0::/10" + url_preview_ip_range_whitelist: + type: array + description: >- + This option sets a list of IP address CIDR ranges that the URL preview + spider is allowed to access even if they are specified in + `url_preview_ip_range_blacklist`. This is useful for specifying exceptions + to wide-ranging blacklisted target IP ranges – e.g. for enabling URL + previews for a specific private website only visible in your network. + items: + type: string + default: [] + examples: + - - 192.168.1.1 + url_preview_url_blacklist: + type: array + description: >- + Optional list of URL matches that the URL preview spider is denied from + accessing. This is a usability feature, not a security one. You should use + `url_preview_ip_range_blacklist` in preference to this, otherwise someone + could define a public DNS entry that points to a private IP address and + circumvent the blacklist. Applications that perform redirects or serve + different content when detecting that Synapse is accessing them can also + bypass the blacklist. This is more useful if you know there is an entire + shape of URL that you know that you do not want Synapse to preview. + + + Each list entry is a dictionary of url component attributes as returned by + urlparse.urlsplit as applied to the absolute form of the URL. See + [here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) + for more information. Some examples are: + + + * `username` + + * `netloc` + + * `scheme` + + * `path` + + + The values of the dictionary are treated as a filename match pattern + applied to that component of URLs, unless they start with a ^ in which + case they are treated as a regular expression match. If all the specified + component matches for a given list item succeed, the URL is blacklisted. + items: + type: object + default: [] + examples: + - - username: "*" + - netloc: google.com + - netloc: "*.google.com" + - scheme: http + - netloc: www.acme.com + path: /foo + - netloc: "^[0-9]+.[0-9]+.[0-9]+.[0-9]+$" + max_spider_size: + $ref: "#/$defs/bytes" + description: The largest allowed URL preview spidering size in bytes. + default: 10M + examples: + - 8M + url_preview_accept_language: + type: array + description: >- + A list of values for the Accept-Language HTTP header used when downloading + webpages during URL preview generation. This allows Synapse to specify the + preferred languages that URL previews should be in when communicating with + remote servers. + + + Each value is a IETF language tag; a 2-3 letter identifier for a language, + optionally followed by subtags separated by `-`, specifying a country or + region variant. + + + Multiple values can be provided, and a weight can be added to each by + using quality value syntax (;q=). `*` translates to any language. + items: + type: string + default: + - en + examples: + - - en-UK + - en-US;q=0.9 + - fr;q=0.8 + - "*;q=0.7" + oembed: + type: object + description: >- + oEmbed allows for easier embedding content from a website. It can be used + for generating URLs previews of services which support it. A default list + of oEmbed providers is included with Synapse. + properties: + disable_default_providers: + type: boolean + description: Do not use Synapse's default list of oEmbed providers. + default: false + additional_providers: + type: array + description: >- + Additional files with oEmbed configuration (each should be in the form + of providers.json). + items: + type: string + default: [] + examples: + - disable_default_providers: true + additional_providers: + - oembed/my_providers.json + recaptcha_public_key: + type: ["string", "null"] + description: >- + This homeserver's ReCAPTCHA public key. Must be specified if + [`enable_registration_captcha`](#enable_registration_captcha) is enabled. + default: null + examples: + - YOUR_PUBLIC_KEY + recaptcha_private_key: + type: ["string", "null"] + description: >- + This homeserver's ReCAPTCHA private key. Must be specified if + [`enable_registration_captcha`](#enable_registration_captcha) is enabled. + default: null + examples: + - YOUR_PRIVATE_KEY + enable_registration_captcha: + type: boolean + description: >- + Set to `true` to require users to complete a CAPTCHA test when registering + an account. Requires a valid ReCaptcha public/private key. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + default: false + examples: + - true + recaptcha_siteverify_api: + type: string + description: The API endpoint to use for verifying `m.login.recaptcha` responses. + default: "https://www.recaptcha.net/recaptcha/api/siteverify" + examples: + - "https://my.recaptcha.site" + turn_uris: + type: array + description: The public URIs of the TURN server to give to clients. + items: + type: string + default: [] + examples: + - - "turn:example.org" + turn_shared_secret: + type: ["string", "null"] + description: The shared secret used to compute passwords for the TURN server. + default: null + examples: + - YOUR_SHARED_SECRET + turn_shared_secret_path: + type: ["string", "null"] + description: >- + An alternative to [`turn_shared_secret`](#turn_shared_secret): allows the + shared secret to be specified in an external file. + + + The file should be a plain text file, containing only the shared secret. + Synapse reads the shared secret from the given file once at startup. + + + _Added in Synapse 1.116.0._ + default: null + examples: + - /path/to/secrets/file + turn_username: + type: ["string", "null"] + description: TURN server username if not using a token. + default: null + examples: + - TURNSERVER_USERNAME + turn_password: + type: ["string", "null"] + description: TURN server password if not using a token. + default: null + examples: + - TURNSERVER_PASSWORD + turn_user_lifetime: + $ref: "#/$defs/duration" + description: How long generated TURN credentials last. + default: 1h + examples: + - 2h + turn_allow_guests: + type: boolean + description: >- + Whether guests should be allowed to use the TURN server. If false, VoIP + will be unreliable for guests. However, it does introduce a slight + security risk as it allows users to connect to arbitrary endpoints without + having first signed up for a valid account (e.g. by passing a CAPTCHA). + default: true + examples: + - false + enable_registration: + type: boolean + description: >- + Enable registration for new users. + + + It is highly recommended that if you enable registration, you set one or + more or the following options, to avoid abuse of your server by "bots": + + + * [`enable_registration_captcha`](#enable_registration_captcha) + + * [`registrations_require_3pid`](#registrations_require_3pid) + + * [`registration_requires_token`](#registration_requires_token) + + + (In order to enable registration without any verification, you must also + set + [`enable_registration_without_verification`](#enable_registration_without_verification).) + + + Note that even if this setting is disabled, new accounts can still be + created via the admin API if + [`registration_shared_secret`](#registration_shared_secret) is set. + default: false + examples: + - true + enable_registration_without_verification: + type: boolean + description: >- + Enable registration without email or captcha verification. Note: this + option is *not* recommended, as registration without verification is a + known vector for spam and abuse. Has no effect unless + [`enable_registration`](#enable_registration) is also enabled. + default: false + examples: + - true + registrations_require_3pid: + type: array + description: >- + If this is set, users must provide all of the specified types of + [3PID](https://spec.matrix.org/latest/appendices/#3pid-types) when + registering an account. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + items: + type: string + default: [] + examples: + - - email + - msisdn + disable_msisdn_registration: + type: boolean + description: >- + Explicitly disable asking for MSISDNs from the registration flow + (overrides `registrations_require_3pid` if MSISDNs are set as required). + default: false + examples: + - true + allowed_local_3pids: + type: ["array", "null"] + description: >- + Mandate that users are only allowed to associate certain formats of 3PIDs + with accounts on this server, as specified by the `medium` and `pattern` + sub-options. `pattern` is a [Perl-like regular + expression](https://docs.python.org/3/library/re.html#module-re). + + + More information about 3PIDs, allowed `medium` types and their `address` + syntax can be found [in the Matrix + spec](https://spec.matrix.org/latest/appendices/#3pid-types). + items: + type: object + description: Item allowing a given pattern for the specified 3PID medium. + properties: + medium: + $ref: "#/$defs/3pidmedium" + description: The medium for which to allow 3PID association. + pattern: + type: string + description: >- + A [Perl-like regular + expression](https://docs.python.org/3/library/re.html#module-re) + allowing association of a 3PID to a local account if it matches the + given format. + default: null + examples: + - - medium: email + pattern: "^[^@]+@matrix\\.org$" + - medium: email + pattern: "^[^@]+@vector\\.im$" + - medium: msisdn + pattern: "^44\\d{10}$" + enable_3pid_lookup: + type: boolean + description: Enable 3PIDs lookup requests to identity servers from this server. + default: true + examples: + - false + registration_requires_token: + type: boolean + description: >- + Require users to submit a token during registration. Tokens can be managed + using the admin [API](../administration/admin_api/registration_tokens.md). + Disabling this option will not delete any tokens previously generated. + + + Note that [`enable_registration`](#enable_registration) must also be set + to allow account registration. + default: false + examples: + - true + registration_shared_secret: + type: ["string", "null"] + description: >- + If set, allows registration of standard or admin accounts by anyone who + has the shared secret, even if + [`enable_registration`](#enable_registration) is not set. + + + This is primarily intended for use with the `register_new_matrix_user` + script (see [Registering a + user](../../setup/installation.md#registering-a-user)); however, the + interface is [documented](../../admin_api/register_api.html). + + + Replacing an existing `registration_shared_secret` with a new one requires + users of the [Shared-Secret Registration + API](../../admin_api/register_api.html) to start using the new secret for + requesting any further one-time nonces. + + + > ⚠️ **Warning** – The additional consequences of replacing + [`macaroon_secret_key`](#macaroon_secret_key) will apply in case it + delegates to `registration_shared_secret`. + + + See also + [`registration_shared_secret_path`](#registration_shared_secret_path). + default: null + examples: + - "" + registration_shared_secret_path: + type: ["string", "null"] + description: >- + An alternative to + [`registration_shared_secret`](#registration_shared_secret): allows the + shared secret to be specified in an external file. + + + The file should be a plain text file, containing only the shared secret. + + + If this file does not exist, Synapse will create a new shared secret on + startup and store it in this file. + + + _Added in Synapse 1.67.0._ + default: null + examples: + - /path/to/secrets/file + bcrypt_rounds: + type: integer + description: >- + Set the number of bcrypt rounds used to generate password hash. Larger + numbers increase the work factor needed to generate the hash. The default + number is 12 (which equates to 2^12 rounds). N.B. that increasing this + will exponentially increase the time required to register or login - e.g. + 24 => 2^24 rounds which will take >20 mins. + default: 12 + examples: + - 14 + allow_guest_access: + type: boolean + description: >- + Allows users to register as guests without a password/email/etc, and + participate in rooms hosted on this server which have been made accessible + to anonymous users. + default: false + examples: + - true + default_identity_server: + type: ["string", "null"] + description: >- + The identity server which we suggest that clients should use when users + log in on this server. + + + (By default, no suggestion is made, so it is left up to the client. This + setting is ignored unless `public_baseurl` is also explicitly set.) + default: null + examples: + - "https://matrix.org" + account_threepid_delegates: + type: object + properties: + msisdn: + type: ["string", "null"] + description: Identity server base URI for MSISDN (phone numbers). See above. + description: >- + Delegate verification of phone numbers to an identity server. + + + When a user wishes to add a phone number to their account, we need to + verify that they actually own that phone number, which requires sending + them a text message (SMS). Currently Synapse does not support sending + those texts itself and instead delegates the task to an identity server. + The base URI for the identity server to be used is specified by the + `account_threepid_delegates.msisdn` option. + + + If this is left unspecified, Synapse will not allow users to add phone + numbers to their account. + + + (Servers handling the these requests must answer the `/requestToken` + endpoints defined by the Matrix Identity Service API + [specification](https://matrix.org/docs/spec/identity_service/latest).) + + + *Deprecated in Synapse 1.64.0*: The `email` option is deprecated. + + + *Removed in Synapse 1.66.0*: The `email` option has been removed. If + present, Synapse will report a configuration error on startup. + default: {} + examples: + - msisdn: "http://localhost:8090" + enable_set_displayname: + type: boolean + description: >- + Whether users are allowed to change their displayname after it has been + initially set. Useful when provisioning users based on the contents of a + third-party directory. + + + Does not apply to server administrators. + default: true + examples: + - false + enable_set_avatar_url: + type: boolean + description: >- + Whether users are allowed to change their avatar after it has been + initially set. Useful when provisioning users based on the contents of a + third-party directory. + + + Does not apply to server administrators. + default: true + examples: + - false + enable_3pid_changes: + type: boolean + description: >- + Whether users can change the third-party IDs associated with their + accounts (email address and msisdn). + default: true + examples: + - false + auto_join_rooms: + type: array + description: >- + Users who register on this homeserver will automatically be joined to the + rooms listed under this option. + + + By default, any room aliases included in this list will be created as a + publicly joinable room when the first user registers for the homeserver. + If the room already exists, make certain it is a publicly joinable room, + i.e. the join rule of the room must be set to `public`. You can find more + options relating to auto-joining rooms below. + + + As Spaces are just rooms under the hood, Space aliases may also be used. + items: + type: string + default: [] + examples: + - - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" + autocreate_auto_join_rooms: + type: boolean + description: >- + Where `auto_join_rooms` are specified, setting this flag ensures that the + rooms exist by creating them when the first user on the homeserver + registers. This option will not create Spaces. + + + By default the auto-created rooms are publicly joinable from any federated + server. Use the `autocreate_auto_join_rooms_federated` and + `autocreate_auto_join_room_preset` settings to customise this behaviour. + + + Setting to false means that if the rooms are not manually created, users + cannot be auto-joined since they do not exist. + default: true + examples: + - false + autocreate_auto_join_rooms_federated: + type: boolean + description: >- + Whether the rooms listed in `auto_join_rooms` that are auto-created are + available via federation. Only has an effect if + `autocreate_auto_join_rooms` is true. + + + Note that whether a room is federated cannot be modified after creation. + + + If true, the room will be joinable from other servers. If false, users + from other homeservers are prevented from joining these rooms. + default: true + examples: + - false + autocreate_auto_join_room_preset: + type: string + description: >- + The room preset to use when auto-creating one of `auto_join_rooms`. Only + has an effect if `autocreate_auto_join_rooms` is true. + + + Possible values for this option are: + + * "public_chat": the room is joinable by anyone, including federated + servers if `autocreate_auto_join_rooms_federated` is true (the default). + + * "private_chat": an invitation is required to join these rooms. + + * "trusted_private_chat": an invitation is required to join this room and + the invitee is assigned a power level of 100 upon joining the room. + + + Each preset will set up a room in the same manner as if it were provided + as the `preset` parameter when calling the [`POST + /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) + Client-Server API endpoint. + + + If a value of "private_chat" or "trusted_private_chat" is used then + `auto_join_mxid_localpart` must also be configured. + enum: + - public_chat + - private_chat + - trusted_private_chat + default: public_chat + examples: + - private_chat + auto_join_mxid_localpart: + type: ["string", "null"] + description: >- + The local part of the user id which is used to create `auto_join_rooms` if + `autocreate_auto_join_rooms` is true. If this is not provided then the + initial user account that registers will be used to create the rooms. + + + The user id is also used to invite new users to any auto-join rooms which + are set to invite-only. + + + It *must* be configured if `autocreate_auto_join_room_preset` is set to + "private_chat" or "trusted_private_chat". + + + Note that this must be specified in order for new users to be correctly + invited to any auto-join rooms which have been set to invite-only (either + at the time of creation or subsequently). + + + Note that, if the room already exists, this user must be joined and have + the appropriate permissions to invite new members. + default: null + examples: + - system + auto_join_rooms_for_guests: + type: boolean + description: >- + When `auto_join_rooms` is specified, setting this flag to false prevents + guest accounts from being automatically joined to the rooms. + default: true + examples: + - false + inhibit_user_in_use_error: + type: boolean + description: >- + Whether to inhibit errors raised when registering a new account if the + user ID already exists. If turned on, requests to `/register/available` + will always show a user ID as available, and Synapse won't raise an error + when starting a registration with a user ID that already exists. However, + Synapse will still raise an error if the registration completes and the + username conflicts. + default: false + examples: + - true + allow_underscore_prefixed_registration: + type: boolean + description: >- + Whether users are allowed to register with a underscore-prefixed + localpart. By default, AppServices use prefixes like `_example` to + namespace their associated ghost users. If turned on, this may result in + clashes or confusion. Useful when provisioning users from an external + identity provider. + default: false + examples: + - true + session_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that a user's session remains valid for, after they log in. + + + Note that this is not currently compatible with guest logins. + + + Note also that this is calculated at login time: changes are not applied + retrospectively to users who have already logged in. + default: infinity + examples: + - 24h + refreshable_access_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that an access token remains valid for, if the session is using + refresh tokens. + + + For more information about refresh tokens, please see the + [manual](user_authentication/refresh_tokens.md). + + + Note that this only applies to clients which advertise support for refresh + tokens. + + + Note also that this is calculated at login time and refresh time: changes + are not applied to existing sessions until they are refreshed. + default: 5m + examples: + - 10m + refresh_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that a refresh token remains valid for (provided that it is not + exchanged for another one first). This option can be used to automatically + log-out inactive sessions. Please see the manual for more information. + + + Note also that this is calculated at login time and refresh time: changes + are not applied to existing sessions until they are refreshed. + default: infinity + examples: + - 24h + nonrefreshable_access_token_lifetime: + $ref: "#/$defs/duration" + description: >- + Time that an access token remains valid for, if the session is NOT using + refresh tokens. + + + Please note that not all clients support refresh tokens, so setting this + to a short value may be inconvenient for some users who will then be + logged out frequently. + + + Note also that this is calculated at login time: changes are not applied + retrospectively to existing sessions for users that have already logged + in. + default: infinity + examples: + - 24h + ui_auth: + oneOf: + - $ref: "#/$defs/duration" + - type: object + properties: + session_timeout: + $ref: "#/$defs/duration" + description: >- + The amount of time to allow a user-interactive authentication session to + be active. + + + This defaults to 0, meaning the user is queried for their credentials + before every action, but this can be overridden to allow a single + validation to be re-used. This weakens the protections afforded by the + user-interactive authentication process, by allowing for multiple (and + potentially different) operations to use the same validation session. + + + This is ignored for potentially "dangerous" operations (including + deactivating an account, modifying an account password, adding a 3PID, and + minting additional login tokens). + + + Use the `session_timeout` sub-option here to change the time allowed for + credential validation. + default: 0 + examples: + - session_timeout: 15s + login_via_existing_session: + type: object + description: >- + Matrix supports the ability of an existing session to mint a login token + for another client. + + + Synapse disables this by default as it has security ramifications – a + malicious client could use the mechanism to spawn more than one session. + properties: + enabled: + type: boolean + description: "Enable login via existing session." + default: false + require_ui_auth: + type: boolean + description: Require user-interactive authentication. + default: true + token_timeout: + $ref: "#/$defs/duration" + description: Duration of time the generated token is valid. + default: 5m + examples: + - enabled: true + require_ui_auth: false + token_timeout: 5m + enable_metrics: + type: boolean + description: Set to true to enable collection and rendering of performance metrics. + default: false + examples: + - true + sentry: + type: object + description: >- + Use this option to enable sentry integration. Provide the DSN assigned to + you by sentry with the `dsn` setting. + + + An optional `environment` field can be used to specify an environment. + This allows for log maintenance based on different environments, ensuring + better organization and analysis. + + + NOTE: While attempts are made to ensure that the logs don't contain any + sensitive information, this cannot be guaranteed. By enabling this option + the sentry server may therefore receive sensitive information, and it in + turn may then disseminate sensitive information through insecure + notification channels if so configured. + properties: + dsn: + type: ["string", "null"] + description: >- + The DSN assigned by sentry. If unset or null, sentry integration is + disabled. + default: null + environment: + type: ["string", "null"] + description: Sentry environment. + default: null + examples: + - environment: production + dsn: ... + metrics_flags: + type: object + description: >- + Flags to enable Prometheus metrics which are not suitable to be enabled by + default, either for performance reasons or limited use. Currently the only + option is `known_servers`. + properties: + known_servers: + type: boolean + description: >- + Publishes `synapse_federation_known_servers`, a gauge of the number of + servers this homeserver knows about, including itself. May cause + performance problems on large homeservers. + default: false + examples: + - known_servers: true + report_stats: + type: boolean + description: >- + Whether or not to report homeserver usage statistics. This is originally + set when generating the config. Set this option to true or false to change + the current behavior. See [Reporting Homeserver Usage + Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md) + for information on what data is reported. + + + Statistics will be reported 5 minutes after Synapse starts, and then every + 3 hours after that. + default: false + examples: + - true + report_stats_endpoint: + type: string + description: The endpoint to report homeserver usage statistics to. + default: https://matrix.org/report-usage-stats/push + examples: + - https://example.com/report-usage-stats/push + room_prejoin_state: + type: object + description: >- + This setting controls the state that is shared with users upon receiving + an invite to a room, or in reply to a knock on a room. By default, the + following state events are shared with users: + + + - `m.room.join_rules` + + - `m.room.canonical_alias` + + - `m.room.avatar` + + - `m.room.encryption` + + - `m.room.name` + + - `m.room.create` + + - `m.room.topic` + + + *Changed in Synapse 1.74:* admins can filter the events in prejoin state + based on their state key. + properties: + disable_default_event_types: + type: boolean + description: >- + Set to `true` to disable the above defaults. If this is enabled, only + the event types listed in `additional_event_types` are shared. + default: false + additional_event_types: + type: array + description: >- + A list of additional state events to include in the events to be + shared. By default, this list is empty (so only the default event + types are shared). + + + Each entry in this list should be either a single string or a list of + two strings. + + * A standalone string `t` represents all events with type `t` (i.e. + with no restrictions on state keys). + + * A pair of strings `[t, s]` represents a single event with type `t` + and state key `s`. The same type can appear in two entries with + different state keys: in this situation, both state keys are included + in prejoin state. + items: + type: ["string", "array"] + items: + type: string + default: [] + examples: + - disable_default_event_types: false + additional_event_types: + - org.example.custom.event.typeA + - - org.example.custom.event.typeB + - foo + - - org.example.custom.event.typeC + - bar + - - org.example.custom.event.typeC + - baz + track_puppeted_user_ips: + type: boolean + description: >- + We record the IP address of clients used to access the API for various + reasons, including displaying it to the user in the "Where you're signed + in" dialog. + + + By default, when puppeting another user via the admin API, the client IP + address is recorded against the user who created the access token (ie, the + admin user), and *not* the puppeted user. + + + Set this option to true to also record the IP address against the puppeted + user. (This also means that the puppeted user will count as an "active" + user for the purpose of monthly active user tracking – see + `limit_usage_by_mau` etc above.) + default: false + examples: + - true + app_service_config_files: + type: array + description: A list of application service config files to use. + items: + type: string + default: [] + examples: + - - app_service_1.yaml + - app_service_2.yaml + track_appservice_user_ips: + type: boolean + description: >- + Set to true to enable tracking of application service IP addresses. + Implicitly enables MAU tracking for application service users. + default: false + examples: + - true + use_appservice_legacy_authorization: + type: boolean + description: >- + Whether to send the application service access tokens via the + `access_token` query parameter per older versions of the Matrix + specification. Defaults to false. Set to true to enable sending access + tokens via a query parameter. + + + **Enabling this option is considered insecure and is not recommended.** + default: false + examples: + - true + macaroon_secret_key: + type: ["string", "null"] + description: >- + A secret which is used to sign + + - access token for guest users, + + - short-term login token used during SSO logins (OIDC or SAML2) and + + - token used for unsubscribing from email notifications. + + + If none is specified, the `registration_shared_secret` is used, if one is + given; otherwise, a secret key is derived from the signing key. + + + > ⚠️ **Warning** – Replacing an existing `macaroon_secret_key` with a new + one will lead to invalidation of access tokens for all guest users. It + will also break unsubscribe links in emails sent before the change. An + unlucky user might encounter a broken SSO login flow and would have to + start again. + default: null + examples: + - "" + macaroon_secret_key_path: + type: ["string", "null"] + description: >- + An alternative to [`macaroon_secret_key`](#macaroon_secret_key): allows + the secret key to be specified in an external file. + + + The file should be a plain text file, containing only the secret key. + Synapse reads the secret key from the given file once at startup. + + + _Added in Synapse 1.121.0._ + default: null + examples: + - /path/to/secrets/file + form_secret: + type: ["string", "null"] + description: >- + A secret which is used to calculate HMACs for form values, to stop + falsification of values. Must be specified for the User Consent forms to + work. + + + Replacing an existing `form_secret` with a new one might break the user + consent page for an unlucky user and require them to reopen the page from + a new link. + default: null + examples: + - "" + form_secret_path: + type: ["string", "null"] + description: >- + An alternative to [`form_secret`](#form_secret): allows the secret to be + specified in an external file. + + + The file should be a plain text file, containing only the secret. Synapse + reads the secret from the given file once at startup. + + + _Added in Synapse 1.126.0._ + default: null + examples: + - /path/to/secrets/file + signing_key_path: + type: ["string", "null"] + description: >- + Path to the signing key to sign events and federation requests with. + + + *New in Synapse 1.67*: If this file does not exist, Synapse will create a + new signing key on startup and store it in this file. + default: null + examples: + - CONFDIR/SERVERNAME.signing.key + old_signing_keys: + type: object + description: >- + The keys that the server used to sign messages with but won't use to sign + new messages. + + + It is possible to build an entry from an old `signing.key` file using the + `export_signing_key` script which is provided with synapse. + + + If you have lost the private key file, you can ask another server you + trust to tell you the public keys it has seen from your server. To fetch + the keys from `matrix.org`, try something like: + + + ``` + + curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com | + jq '.server_keys | map(.verify_keys) | add' + ``` + additionalProperties: + type: object + properties: + key: + type: string + description: The base64-encoded public key. + expired_ts: + type: integer + description: >- + Time, in milliseconds since the unix epoch, that the key was last used. + default: {} + examples: + - "ed25519:id": + key: base64string + expired_ts: 123456789123 + key_refresh_interval: + $ref: "#/$defs/duration" + description: >- + How long key response published by this server is valid for. Used to set the + `valid_until_ts` in `/key/v2` APIs. Determines how quickly servers will query to + check which keys are still valid. + default: 1d + examples: + - 2d + trusted_key_servers: + type: array + description: >- + The trusted servers to download signing keys from. + + + When we need to fetch a signing key, each server is tried in parallel. + + + Normally, the connection to the key server is validated via TLS + certificates. Additional security can be provided by configuring a `verify + key`, which will make synapse check that the response is signed by that + key. + + + This setting supersedes an older setting named `perspectives`. The old + format is still supported for backwards-compatibility, but it is + deprecated. + + + `trusted_key_servers` defaults to matrix.org, but using it will generate a + warning on start-up. To suppress this warning, set + `suppress_key_server_warning` to true. + + + If the use of a trusted key server has to be deactivated, e.g. in a + private federation or for privacy reasons, this can be realised by setting + an empty array (`trusted_key_servers: []`). Then Synapse will request the + keys directly from the server that owns the keys. If Synapse does not get + keys directly from the server, the events of this server will be rejected. + items: + server_name: + type: string + description: The name of the server. Required. + verify_keys: + type: ["object", "null"] + description: >- + An optional map from key id to base64-encoded public key. If + specified, we will check that the response is signed by at least one + of the given keys. + additionalProperties: + type: string + accept_keys_insecurely: + type: boolean + description: >- + Normally, if `verify_keys` is unset, and + `federation_verify_certificates` is not `true`, synapse will refuse to + start, because this would allow anyone who can spoof DNS responses to + masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure + connection to the key server, you can set this to `true` to override + this behaviour. + default: + - server_name: matrix.org + examples: + - - server_name: my_trusted_server.example.com + verify_keys: + "ed25519:auto": abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr + - server_name: my_other_trusted_server.example.com + - - server_name: matrix.org + suppress_key_server_warning: + type: boolean + description: >- + Set the following to true to disable the warning that is emitted when the + `trusted_key_servers` include "matrix.org". See above. + default: false + examples: + - true + key_server_signing_keys_path: + type: ["string", "null"] + description: >- + The signing keys to use when acting as a trusted key server. If not + specified defaults to the server signing key. + + + Can contain multiple keys, one per line. + default: null + examples: + - key_server_signing_keys.key + saml2_config: + type: object + description: >- + Enable SAML2 for registration and login. Uses pysaml2. To learn more about + pysaml and to find a full list options for configuring pysaml, read the + docs [here](https://pysaml2.readthedocs.io/en/latest/). + + + At least one of `sp_config` or `config_path` must be set in this section + to enable SAML login. You can either put your entire pysaml config inline + using the `sp_config` option, or you can specify a path to a psyaml config + file with the sub-option `config_path`. + + + Once SAML support is enabled, a metadata file will be exposed at + `https://:/_synapse/client/saml2/metadata.xml`, which you + may be able to use to configure your SAML IdP with. Alternatively, you can + manually configure the IdP to use an ACS location of + `https://:/_synapse/client/saml2/authn_response`. + properties: + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to offer + the user a choice of login mechanisms. + idp_icon: + type: ["string", "null"] + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the URL from the source of the event.) + idp_brand: + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + sp_config: + type: ["object", "null"] + description: >- + Configuration for the pysaml2 Service Provider. See pysaml2 docs for + format of config. Default values will be used for the `entityid` and + `service` settings, so it is not normally necessary to specify them + unless you need to override them. Here are a few useful sub-options + for configuring pysaml: + + * `metadata`: Point this to the IdP's metadata. You must provide + either a local file via the `local` attribute or (preferably) a URL + via the `remote` attribute. + + * `accepted_time_diff: 3`: Allowed clock difference in seconds between + the homeserver and IdP. Defaults to 0. + + * `service`: By default, the user has to go to our login page first. + If you'd like to allow IdP-initiated login, set `allow_unsolicited` to + true under `sp` in the `service` section. + default: null + config_path: + type: ["string", "null"] + description: Specify a separate pysaml2 configuration file. + default: null + saml_session_lifetime: + $ref: "#/$defs/duration" + description: >- + The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. + default: 15m + user_mapping_provider: + type: object + description: >- + Using this option, an external module can be provided as a custom + solution to mapping attributes returned from a saml provider onto a + matrix user. + properties: + module: + type: string + description: The custom module's class. + config: + type: object + description: >- + Custom configuration values for the module. Use the values + provided in the example if you are using the built-in + user_mapping_provider, or provide your own config values for a + custom class if you are using one. This section will be passed as + a Python dictionary to the module's `parse_config` method. The + built-in provider takes the following two options: + + * `mxid_source_attribute`: The SAML attribute (after mapping via + the attribute maps) to use to derive the Matrix ID from. It is + "uid" by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still + defined, its value will be used instead. + + * `mxid_mapping`: The mapping system to use for mapping the saml + attribute onto a matrix ID. Options include: `hexencode` (which + maps unpermitted characters to `=xx`) and `dotreplace` (which + replaces unpermitted characters with `.`). The default is + `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its + value will be used instead. + grandfathered_mxid_source_attribute: + type: string + description: >- + In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. + For backwards-compatibility, we will look for `user_ids` matching such + a pattern before creating a new account. This setting controls the + SAML attribute which will be used for this backwards-compatibility + lookup. Typically it should be "uid", but if the attribute maps are + changed, it may be necessary to change it. + default: uid + attribute_requirements: + type: array + description: >- + It is possible to configure Synapse to only allow logins if SAML + attributes match particular values. The requirements can be listed + under `attribute_requirements` as shown in the example. All of the + listed attributes must match for the login to be permitted. Values can + be specified in a `one_of` list to allow multiple values for an + attribute. + items: + type: object + description: Item allowing a specific SAML attribute. + properties: + attribute: + type: string + description: SAML attribute for which to allow logins. + value: + type: string + description: Value the SAML attribute must match. + one_of: + type: array + description: List of values the SAML attribute must all match. + items: + type: string + required: + - attribute + idp_entityid: + type: ["string", "null"] + description: >- + If the metadata XML contains multiple IdP entities then the + `idp_entityid` option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this + option. + default: null + examples: + - sp_config: + metadata: + local: + - saml2/idp.xml + remote: + - url: "https://our_idp/metadata.xml" + accepted_time_diff: 3 + service: + sp: + allow_unsolicited: true + description: + - My awesome SP + - en + name: + - Test SP + - en + ui_info: + display_name: + - lang: en + text: Display Name is the descriptive name of your service. + description: + - lang: en + text: >- + Description should be a short paragraph explaining the purpose + of the service. + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: + - Matrix + - Element + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + organization: + name: Example com + display_name: + - - Example co + - en + url: "http://example.com" + contact_person: + - given_name: Bob + sur_name: the Sysadmin + email_address: + - admin@example.com + contact_type: technical + saml_session_lifetime: 5m + user_mapping_provider: + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + grandfathered_mxid_source_attribute: upn + attribute_requirements: + - attribute: userGroup + value: staff + - attribute: department + one_of: + - sales + - admins + idp_entityid: "https://our_idp/entityid" + oidc_providers: + type: array + description: >- + List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for + registration and login. See [here](../../openid.md) for information on how + to configure these options. + + + For backwards compatibility, it is also possible to configure a single + OIDC provider via an `oidc_config` setting. This is now deprecated and + admins are advised to migrate to the `oidc_providers` format. (When doing + that migration, use `oidc` for the `idp_id` to ensure that existing users + continue to be recognised.) + + + It is possible to configure Synapse to only allow logins if certain + attributes match particular values in the OIDC userinfo. The requirements + can be listed under `attribute_requirements` as shown here: + + ```yaml + + attribute_requirements: + - attribute: family_name + one_of: ["Stephensson", "Smith"] + - attribute: groups + value: "admin" + # If `value` or `one_of` are not specified, the attribute only needs + # to exist, regardless of value. + - attribute: picture + ``` + + + `attribute` is a required field, while `value` and `one_of` are optional. + + + All of the listed attributes must match for the login to be permitted. + Additional attributes can be added to userinfo by expanding the `scopes` + section of the OIDC config to retrieve additional information from the + OIDC provider. + + + If the OIDC claim is a list, then the attribute must match any value in + the list. Otherwise, it must exactly match the value of the claim. Using + the example above, the `family_name` claim MUST be either "Stephensson" or + "Smith", but the `groups` claim MUST contain "admin". + items: + type: object + properties: + idp_id: + type: string + description: >- + A unique identifier for this identity provider. Used internally by + Synapse; should be a single word such as "github". Note that, if + this is changed, users authenticating via that provider will no + longer be recognised as the same user! (Use "oidc" here if you are + migrating from an old `oidc_config` configuration.) + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + idp_icon: + type: string + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) + room and then copy the URL from the source of the event.) + idp_brand: + type: string + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + discover: + type: boolean + description: >- + Set to false to disable the use of the OIDC discovery mechanism to + discover endpoints. Defaults to true. + issuer: + type: string + description: >- + Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + client_id: + type: string + description: Required. OAuth2 client id to use. + client_secret: + type: ["string", "null"] + description: >- + OAuth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is + `none`. Must be omitted if `client_secret_path` is specified. + client_secret_path: + type: ["string", "null"] + description: >- + Path to the OAuth2 client secret to use. With that it's not + necessary to leak secrets into the config file itself. Mutually + exclusive with `client_secret`. Can be omitted if + `client_secret_jwt_key` is specified. + + + *Added in Synapse 1.91.0.* + client_secret_jwt_key: + type: ["object", "null"] + description: >- + Alternative to client_secret: details of a key used to create a JSON + Web Token to be used as an OAuth2 client secret. + properties: + key: + type: ["string", "null"] + description: >- + A pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + key_file: + type: ["string", "null"] + description: >- + Path to the file containing a pem-encoded signing key. Required + unless `key` is given. + jwt_header: + type: object + description: >- + Dictionary giving properties to include in the JWT header. Must + include the key `alg`. + properties: + alg: + type: string + description: >- + Algorithm used to sign the JWT, such as ES256, using the JWA + identifiers in RFC7518. + jwt_payload: + type: object + description: >- + Optional dictionary giving properties to include in the JWT + payload. Normally this should include an `iss` key. + client_auth_method: + type: ["string", "null"] + enum: + - client_secret_basic + - client_secret_post + - none + - null + description: >- + Auth method to use when exchanging the token. Valid values are + `client_secret_basic` (default), `client_secret_post` and `none`. + pkce_method: + type: ["string", "null"] + enum: + - auto + - always + - never + - null + description: >- + Whether to use proof key for code exchange when requesting and + exchanging the token. Valid values are: `auto`, `always`, or + `never`. Defaults to `auto`, which uses PKCE if supported during + metadata discovery. Set to `always` to force enable PKCE or `never` + to force disable PKCE. + id_token_signing_alg_values_supported: + type: array + description: >- + List of the JWS signing algorithms (`alg` values) that are supported + for signing the `id_token`. + + + This is *not* required if `discovery` is disabled. We default to + supporting `RS256` in the downstream usage if no algorithms are + configured here or in the discovery document. + + + According to the spec, the algorithm `"RS256"` MUST be included. The + absolute rigid approach would be to reject this provider as + non-compliant if it's not included but we simply allow whatever and + see what happens (you're the one that configured the value and + cooperating with the identity provider). + + + The `alg` value `"none"` MAY be supported but can only be used if + the Authorization Endpoint does not include `id_token` in the + `response_type` (ex. `/authorize?response_type=code` where `none` + can apply, `/authorize?response_type=code%20id_token` where `none` + can't apply) (such as when using the Authorization Code Flow). + items: + type: string + scopes: + type: ["array", "null"] + description: >- + List of scopes to request. This should normally include the "openid" + scope. Defaults to `["openid"]`. + items: + type: string + authorization_endpoint: + type: string + description: >- + The OAuth2 authorization endpoint. Required if provider discovery is + disabled. + token_endpoint: + type: string + description: >- + The OAuth2 token endpoint. Required if provider discovery is disabled. + userinfo_endpoint: + type: string + description: >- + The OIDC userinfo endpoint. Required if discovery is disabled and + the "openid" scope is not requested. + jwks_uri: + type: string + description: >- + URI where to fetch the JWKS. Required if discovery is disabled and + the "openid" scope is used. + skip_verification: + type: boolean + description: >- + Set to `true` to skip metadata verification. Use this if you are + connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + user_profile_method: + type: ["string", "null"] + enum: + - auto + - userinfo_endpoint + - null + description: >- + Whether to fetch the user profile from the userinfo endpoint, or to + rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. Defaults to `auto`, + which uses the userinfo endpoint if `openid` is not included in + `scopes`. Set to `userinfo_endpoint` to always use the userinfo + endpoint. + redirect_uri: + type: ["string", "null"] + description: >- + An optional string, that if set will override the `redirect_uri` + parameter sent in the requests to the authorization and token + endpoints. Useful if you want to redirect the client to another + endpoint as part of the OIDC login. Be aware that the client must + then call Synapse's OIDC callback URL + (`/_synapse/client/oidc/callback`) manually + afterwards. Must be a valid URL including scheme and path. + additional_authorization_parameters: + type: object + description: >- + String to string dictionary that will be passed as additional + parameters to the authorization grant URL. + additionalProperties: + type: string + passthrough_authorization_parameters: + type: array + description: >- + List of parameters that will be passed through from the redirect + endpoint to the authorization grant URL. + items: + type: string + allow_existing_users: + type: boolean + description: >- + Set to true to allow a user logging in via OIDC to match a + pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + enable_registration: + type: boolean + description: >- + Set to `false` to disable automatic registration of new users. This + allows the OIDC SSO flow to be limited to sign in only, rather than + automatically registering users that have a valid SSO login but do + not have a pre-registered account. Defaults to true. + user_mapping_provider: + type: object + description: >- + Configuration for how attributes returned from a OIDC provider are + mapped onto a matrix user. + + + When rendering, the Jinja2 templates are given a `user` variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + properties: + module: + type: string + description: >- + The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. See [OpenID + Mapping + Providers](../../sso_mapping_providers.md#openid-mapping-providers) + for information on implementing a custom mapping provider. + config: + type: object + description: >- + Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + + For the default provider, the following settings are available: + + + * `subject_template`: Jinja2 template for a unique identifier + for the user. Defaults to `{{ user.sub }}`, which OpenID Connect + compliant providers should provide. + + This replaces and overrides `subject_claim`. + + * `subject_claim`: name of the claim containing a unique + identifier for the user. Defaults to `sub`, which OpenID Connect + compliant providers should provide. + + *Deprecated in Synapse v1.75.0.* + + * `picture_template`: Jinja2 template for an url for the user's + profile picture. Defaults to `{{ user.picture }}`, which OpenID + Connect compliant providers should provide and has to refer to a + direct image file such as PNG, JPEG, or GIF image file. + + This replaces and overrides `picture_claim`. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + * `picture_claim`: name of the claim containing an url for the + user's profile picture. Defaults to "picture", which OpenID + Connect compliant providers should provide and has to refer to a + direct image file such as PNG, JPEG, or GIF image file. + + Currently only supported in monolithic (single-process) server configurations where the media repository runs within the Synapse process. + + *Deprecated in Synapse v1.75.0.* + + * `localpart_template`: Jinja2 template for the localpart of the + MXID. If this is not set, the user will be prompted to choose + their own username (see the documentation for the + `sso_auth_account_details.html` template). This template can use + the `localpart_from_email` filter. + + + * `confirm_localpart`: Whether to prompt the user to validate + (or change) the generated localpart (see the documentation for + the "sso_auth_account_details.html" template), instead of + registering the account right away. + + + * `display_name_template`: Jinja2 template for the display name + to set on first login. If unset, no displayname will be set. + + + * `email_template`: Jinja2 template for the email address of the + user. If unset, no email address will be added to the account. + + + * `extra_attributes`: a map of Jinja2 templates for extra + attributes to send back to the client during login. Note that + these are non-standard and clients will ignore them without + modifications. + backchannel_logout_enabled: + type: boolean + description: >- + Set to `true` to process OIDC Back-Channel Logout notifications. + Those notifications are expected to be received on + `/_synapse/client/oidc/backchannel_logout`. Defaults to `false`. + backchannel_logout_ignore_sub: + type: boolean + description: >- + By default, the OIDC Back-Channel Logout feature checks that the + `sub` claim matches the subject claim received during login. This + check can be disabled by setting this to `true`. Defaults to + `false`. + + + You might want to disable this if the `subject_claim` returned by + the mapping provider is not `sub`. + default: [] + examples: + - - idp_id: my_idp + idp_name: My OpenID provider + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: provided-by-your-issuer + client_secret: provided-by-your-issuer + client_auth_method: client_secret_post + scopes: + - openid + - profile + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + additional_authorization_parameters: + acr_values: 2fa + passthrough_authorization_parameters: + - login_hint + skip_verification: true + enable_registration: true + user_mapping_provider: + config: + subject_claim: id + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: synapseUsers + cas_config: + type: object + description: Enable Central Authentication Service (CAS) for registration and login. + properties: + enabled: + type: boolean + description: Set this to true to enable authorization against a CAS server. + default: false + idp_name: + type: string + description: >- + A user-facing name for this identity provider, which is used to offer + the user a choice of login mechanisms. + idp_icon: + type: ["string", "null"] + description: >- + An optional icon for this identity provider, which is presented by + clients and Synapse's own IdP picker page. If given, must be an MXC + URI of the format `mxc:///`. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the URL from the source of the event.) + default: null + idp_brand: + type: ["string", "null"] + description: >- + An optional brand for this identity provider, allowing clients to + style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options + here. + default: null + server_url: + type: string + description: The URL of the CAS authorization endpoint. + protocol_version: + type: ["integer", "null"] + description: >- + The CAS protocol version. (Version 3 is required if you want to use + `required_attributes`). + default: null + displayname_attribute: + type: ["string", "null"] + description: >- + The attribute of the CAS response to use as the display name. If no + name is given here, no displayname will be set. + default: null + required_attributes: + type: object + description: >- + It is possible to configure Synapse to only allow logins if CAS + attributes match particular values. All of the keys given below must + exist and the values must match the given value. Alternately if the + given value is `None` then any value is allowed (the attribute just + must exist). All of the listed attributes must match for the login to + be permitted. + additionalProperties: + type: ["string", "null"] + default: {} + enable_registration: + type: boolean + description: >- + Set to `false` to disable automatic registration of new users. This + allows the CAS SSO flow to be limited to sign in only, rather than + automatically registering users that have a valid SSO login but do not + have a pre-registered account. + default: true + allow_numeric_ids: + type: boolean + description: >- + Set to `true` allow numeric user IDs. This allows CAS SSO flow to + provide user IDs composed of numbers only. These identifiers will be + prefixed by the letter "u" by default. The prefix can be configured + using the `numeric_ids_prefix` option. Be careful to choose the prefix + correctly to avoid any possible conflicts (e.g. user 1234 becomes + u1234 when a user u1234 already exists). + default: false + numeric_ids_prefix: + type: string + description: >- + The prefix you wish to add in front of a numeric user ID when the + `allow_numeric_ids` option is set to `true`. Only alphanumeric + characters are allowed. + + + *Added in Synapse 1.93.0.* + default: u + examples: + - enabled: true + server_url: "https://cas-server.com" + protocol_version: 3 + displayname_attribute: name + required_attributes: + userGroup: staff + department: None + enable_registration: true + allow_numeric_ids: true + numeric_ids_prefix: numericuser + sso: + type: object + description: >- + Additional settings to use with single-sign on systems such as OpenID + Connect, SAML2 and CAS. + + + Server admins can configure custom templates for pages related to SSO. See + [here](../../templates.md) for more information. + properties: + client_whitelist: + type: ["array", "null"] + description: >- + A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be + subject to an additional confirmation step after the SSO login is + completed. + + + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after + the hostname: "https://my.client/". + + + The login fallback page (used by clients that don't natively support + the required login flows) is whitelisted in addition to any URLs in + this list. By default, this list contains only the login fallback + page. + items: + type: string + default: null + update_profile_information: + type: boolean + description: >- + Use this setting to keep a user's profile fields in sync with + information from the identity provider. Currently only syncing the + displayname is supported. Fields are checked on every SSO login, and + are updated if necessary. Note that enabling this option will override + user profile information, regardless of whether users have opted-out + of syncing that information when first signing in. + default: false + examples: + - client_whitelist: + - "https://riot.im/develop" + - "https://my.custom.client/" + update_profile_information: true + jwt_config: + type: object + description: >- + JSON web token integration. The following settings can be used to make + Synapse JSON web tokens for authentication, instead of its internal + password database. + + + Each JSON Web Token needs to contain a "sub" (subject) claim, which is + used as the localpart of the mxid. + + + Additionally, the expiration time ("exp"), not before time ("nbf"), and + issued at ("iat") claims are validated if present. + + + Note that this is a non-standard login type and client support is expected + to be non-existent. + + + See [here](../../jwt.md) for more. + properties: + enabled: + type: boolean + description: Set to true to enable authorization using JSON web tokens. + default: false + secret: + type: string + description: >- + This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is + set to true. + algorithm: + type: string + description: >- + The algorithm used to sign (or HMAC) the JSON web token. Supported + algorithms are listed [here (section + JWS)](https://docs.authlib.org/en/latest/specs/rfc7518.html). Required + if `enabled` is set to true. + subject_claim: + type: ["string", "null"] + description: Name of the claim containing a unique identifier for the user. + default: sub + display_name_claim: + type: ["string", "null"] + description: >- + Name of the claim containing the display name for the user. If + provided, the display name will be set to the value of this claim upon + first login. + default: null + issuer: + type: ["string", "null"] + description: >- + The issuer to validate the "iss" claim against. If provided the "iss" + claim will be required and validated for all JSON web tokens. + default: null + audiences: + type: ["array", "null"] + description: >- + A list of audiences to validate the "aud" claim against. If provided + the "aud" claim will be required and validated for all JSON web + tokens. Note that if the "aud" claim is included in a JSON web token + then validation will fail without configuring audiences. + items: + type: string + default: null + examples: + - enabled: true + secret: provided-by-your-issuer + algorithm: provided-by-your-issuer + subject_claim: name_of_claim + display_name_claim: name_of_claim + issuer: provided-by-your-issuer + audiences: + - provided-by-your-issuer + password_config: + type: object + description: Use this setting to enable password-based logins. + properties: + enabled: + type: ["boolean", "string"] + enum: + - true + - false + - only_for_reauth + description: >- + Set to false to disable password authentication. Set to + `only_for_reauth` to allow users with existing passwords to use them + to reauthenticate (not log in), whilst preventing new users from + setting passwords. + default: true + localdb_enabled: + type: boolean + description: >- + Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful if + you have other `password_providers`. + default: true + pepper: + type: ["string", "null"] + description: >- + Set the value here to a secret random string for extra security. DO + NOT CHANGE THIS AFTER INITIAL SETUP! + default: null + policy: + type: object + description: >- + Define and enforce a password policy, such as minimum lengths for + passwords, etc. This is an implementation of MSC2000. + properties: + enabled: + type: boolean + description: Set to true to enable. + default: false + minimum_length: + type: integer + description: Minimum accepted length for a password. + default: 0 + require_digit: + type: boolean + description: Whether a password must contain at least one digit. + default: false + require_symbol: + type: boolean + description: >- + Whether a password must contain at least one symbol. A symbol is + any character that's not a number or a letter. + default: false + require_lowercase: + type: boolean + description: Whether a password must contain at least one lowercase letter. + default: false + require_uppercase: + type: boolean + description: Whether a password must contain at least one uppercase letter. + default: false + examples: + - enabled: false + localdb_enabled: false + pepper: EVEN_MORE_SECRET + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true + push: + type: object + description: This setting defines options for push notifications. + properties: + enabled: + type: boolean + description: >- + Enables or disables push notification calculation. Note, disabling + this will also stop unread counts being calculated for rooms. This + mode of operation is intended for homeservers which may only have bots + or appservice users connected, or are otherwise not interested in + push/unread counters. + default: true + include_content: + type: boolean + description: >- + Clients requesting push notifications can either have the body of the + message sent in the notification poke along with other details like + the sender, or just the event ID and room ID (`event_id_only`). If + clients choose to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. For modern Android devices the notification content + will still appear because it is loaded by the app. iPhone, however + will send a notification saying only that a message arrived and who it + came from. Set to false to only include the event ID and room ID in + push notification payloads. + default: true + group_unread_count_by_room: + type: boolean + description: >- + When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages + for the user, or the number of *rooms* the user has unread messages + in. If true, push clients will see the number of rooms with unread + messages in them. Set to false to instead send the number of unread + messages. + default: true + jitter_delay: + $ref: "#/$defs/duration" + description: >- + Delays push notifications by a random amount up to the given duration. + Useful for mitigating timing attacks. Optional. + + + _Added in Synapse 1.84.0._ + default: 0s + examples: + - enabled: true + include_content: false + group_unread_count_by_room: false + jitter_delay: 10s + encryption_enabled_by_default_for_room_type: + type: string + description: >- + Controls whether locally-created rooms should be end-to-end encrypted by + default. + + + Possible options are "all", "invite", and "off". They are defined as: + + + * "all": any locally-created room + + * "invite": any room created with the `private_chat` or + `trusted_private_chat` room creation presets + + * "off": this option will take no effect + + + Note that this option will only affect rooms created after it is set. It + will also not affect rooms created by other servers. + enum: + - all + - invite + - "off" + default: "off" + examples: + - invite + user_directory: + type: object + description: This setting defines options related to the user directory. + properties: + enabled: + type: boolean + description: >- + Defines whether users can search the user directory. If false then + empty responses are returned to all queries. + default: true + search_all_users: + type: boolean + description: >- + Defines whether to search all users visible to your homeserver at the + time the search is performed. If set to true, will return all users + known to the homeserver matching the search query. If false, search + results will only contain users visible in public rooms and users + sharing a room with the requester. + + + NB. If you set this to true, and the last time the user_directory + search indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + + + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via the API following the instructions [for + running background + updates](../administration/admin_api/background_updates.md#run), set + to true to return search results containing all known users, even if + that user does not share a room with the requester. + default: false + prefer_local_users: + type: boolean + description: >- + Defines whether to prefer local users in search query results. If set + to true, local users are more likely to appear above remote users when + searching the user directory. + default: false + exclude_remote_users: + type: boolean + description: If set to true, the search will only return local users. + default: false + show_locked_users: + type: boolean + description: Defines whether to show locked users in search query results. + default: false + examples: + - enabled: false + search_all_users: true + prefer_local_users: true + exclude_remote_users: false + show_locked_users: true + user_consent: + type: object + description: >- + For detailed instructions on user consent configuration, see + [here](../../consent_tracking.md). + + + Parts of this section are required if enabling the `consent` resource + under [`listeners`](#listeners), in particular `template_dir` and + `version`. + properties: + template_dir: + type: string + description: >- + Gives the location of the templates for the HTML forms. This directory + should contain one subdirectory per language (eg, `en`, `fr`), and + each language directory should contain the policy document (named as + .html) and a success page (success.html). + version: + type: number + description: >- + Specifies the "current" version of the policy document. It defines the + version to be served by the consent resource if there is no `v` + parameter. + server_notice_content: + type: object + description: >- + If enabled, will send a user a "Server Notice" asking them to consent + to the privacy policy. The [`server_notices` section](#server_notices) + must also be configured for this to work. Notices will *not* be sent + to guest users unless `send_server_notice_to_guests` is set to true. + properties: + msgtype: + type: string + description: Message type of the notice event. + body: + type: string + description: Message template for the server notice event body. + send_server_notice_to_guests: + type: boolean + description: Send server notices to guest users, too. + default: false + block_events_error: + type: ["string", "null"] + description: >- + If set, will block any attempts to send events until the user consents + to the privacy policy. The value of the setting is used as the text of + the error. + default: null + require_at_registration: + type: boolean + description: >- + If enabled, will add a step to the registration process, similar to + how captcha works. Users will be required to accept the policy before + their account is created. + policy_name: + type: string + description: Human-readable name of the privacy policy. + default: Privacy Policy + examples: + - template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy + stats: + type: object + description: >- + Settings for local room and user statistics collection. See + [here](../../room_and_user_statistics.md) for more. + properties: + enabled: + type: boolean + description: >- + Set to false to disable room and user statistics. Note that doing so + may cause certain features (such as the room directory) not to work + correctly. + default: true + examples: + - enabled: false + server_notices: + type: object + description: >- + Use this setting to enable a room which can be used to send notices from + the server to users. It is a special room which users cannot leave; + notices in the room come from a special "notices" user id. + + + If you use this setting, you *must* define the `system_mxid_localpart` + sub-setting, which defines the id of the user which will be used to send + the notices. + + + Note that the name, topic and avatar of existing server notice rooms will + only be updated when a new notice event is sent. + properties: + system_mxid_display_name: + type: string + description: Display name of the "notices" user. + default: Notices + system_mxid_avatar_url: + type: ["string", "null"] + description: Avatar for the "notices" user. + default: null + room_name: + type: string + description: Room name of the server notices room. + default: Server Notices + room_avatar_url: + type: ["string", "null"] + description: >- + Room avatar to use for server notice rooms. If set to the empty string + `""`, notice rooms will not be given an avatar. + + + _Added in Synapse 1.99.0._ + default: null + room_topic: + type: ["string", "null"] + description: >- + Topic to use for server notice rooms. If set to the empty string `""`, + notice rooms will not be given a topic. Defaults to the empty string. + + + _Added in Synapse 1.99.0._ + default: null + auto_join: + type: boolean + description: >- + If true, the user will be automatically joined to the room instead of + being invited. + + + _Added in Synapse 1.98.0._ + default: false + examples: + - system_mxid_localpart: notices + system_mxid_display_name: Server Notices + system_mxid_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: Server Notices + room_avatar_url: "mxc://example.com/oumMVlgDnLYFaPVkExemNVVZ" + room_topic: >- + Room used by your server admin to notice you of important information + auto_join: true + enable_room_list_search: + type: boolean + description: >- + Set to false to disable searching the public room list. When disabled + blocks searching local and remote room lists for local and remote users by + always returning an empty list for all queries. + default: true + examples: + - false + alias_creation_rules: + type: ["array", "null"] + description: >- + The `alias_creation_rules` option allows server admins to prevent unwanted + alias creation on this server. + + + This setting is an optional list of 0 or more rules. By default, no list + is provided, meaning that all alias creations are permitted. + + + Otherwise, requests to create aliases are matched against each rule in + order. The first rule that matches decides if the request is allowed or + denied. If no rule matches, the request is denied. In particular, this + means that configuring an empty list of rules will deny every alias + creation request. + + + Each of the glob patterns is optional, defaulting to `*` ("match + anything"). Note that the patterns match against fully qualified IDs, e.g. + against `@alice:example.com`, `#room:example.com` and + `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. + + + Each rule is a YAML object containing four fields, each of which is an + optional string + items: + type: object + properties: + user_id: + type: ["string", "null"] + description: Glob pattern that matches against the creator of the alias. + alias: + type: ["string", "null"] + description: Glob pattern that matches against the alias being created. + room_id: + type: ["string", "null"] + description: >- + Glob pattern that matches against the room ID the alias is being pointed at. + action: + type: string + enum: + - allow + - deny + description: >- + Either `allow` or `deny`. What to do with the request if the rule + matches. Defaults to `allow`. + default: null + examples: + - null + - - action: allow + - [] + - - action: deny + - - user_id: "@bad_user:example.com" + action: deny + - action: allow + - - room_id: "!forbiddenRoom:example.com" + action: deny + - action: allow + room_list_publication_rules: + type: ["array", "null"] + description: >- + The `room_list_publication_rules` option allows server admins to prevent + unwanted entries from being published in the public room list. + + + The format of this option is the same as that for + [`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or + more rules. By default, no list is provided, meaning that no one may + publish to the room list (except server admins). + + + Otherwise, requests to publish a room are matched against each rule in + order. The first rule that matches decides if the request is allowed or + denied. If no rule matches, the request is denied. In particular, this + means that configuring an empty list of rules will deny every alias + creation request. + + + Requests to create a public (public as in published to the room directory) + room which violates the configured rules will result in the room being + created but not published to the room directory. + + + Each of the glob patterns is optional, defaulting to `*` ("match + anything"). Note that the patterns match against fully qualified IDs, e.g. + against `@alice:example.com`, `#room:example.com` and + `!abcdefghijk:example.com` instead of `alice`, `room` and `abcedgghijk`. + + + Each rule is a YAML object containing four fields, each of which is an + optional string. + + + _Changed in Synapse 1.126.0: The default was changed to deny publishing to + the room list by default_ + items: + type: object + properties: + user_id: + type: ["string", "null"] + description: Glob pattern that matches against the user publishing the room. + alias: + type: ["string", "null"] + description: >- + Glob pattern that matches against one of published room's aliases. + + - If the room has no aliases, the alias match fails unless `alias` + is unspecified or `*`. + + - If the room has exactly one alias, the alias match succeeds if the + `alias` pattern matches that alias. + + - If the room has two or more aliases, the alias match succeeds if + the pattern matches at least one of the aliases. + room_id: + type: ["string", "null"] + description: >- + Glob pattern that matches against the room ID of the room being published. + action: + type: string + enum: + - allow + - deny + description: >- + Either `allow` or `deny`. What to do with the request if the rule + matches. Defaults to `allow`. + default: null + examples: + - null + - - action: deny + - [] + - - action: allow + - - user_id: "@bad_user:example.com" + action: deny + - action: allow + - - room_id: "!forbiddenRoom:example.com" + action: deny + - action: allow + - - alias: "#*potato*:example.com" + action: deny + - action: allow + default_power_level_content_override: + type: object + description: >- + The `default_power_level_content_override` option controls the default + power levels for rooms. + + + Useful if you know that your users need special permissions in rooms that + they create (e.g. to send particular types of state events without needing + an elevated power level). This takes the same shape as the + `power_level_content_override` parameter in the /createRoom API, but is + applied before that parameter. + + + Note that each key provided inside a preset (for example `events` in the + example below) will overwrite all existing defaults inside that key. So in + Example #1, newly-created private_chat rooms will have no rules for any + event types except `com.example.foo`. + + + The default power levels for each preset are: + + + ```yaml + + "m.room.name": 50 + + "m.room.power_levels": 100 + + "m.room.history_visibility": 100 + + "m.room.canonical_alias": 50 + + "m.room.avatar": 50 + + "m.room.tombstone": 100 + + "m.room.server_acl": 100 + + "m.room.encryption": 100 + + ``` + + + In Example #2 the default power-levels for a preset are maintained, but + the power level for a new key is set. + default: {} + examples: + - private_chat: + events: + com.example.foo: 0 + trusted_private_chat: null + public_chat: null + - private_chat: + events: + com.example.foo: 0 + m.room.name: 50 + m.room.power_levels: 100 + m.room.history_visibility: 100 + m.room.canonical_alias: 50 + m.room.avatar: 50 + m.room.tombstone: 100 + m.room.server_acl: 100 + m.room.encryption: 100 + trusted_private_chat: null + public_chat: null + forget_rooms_on_leave: + type: boolean + description: >- + Set to true to automatically forget rooms for users when they leave them, + either normally or via a kick or ban. + default: false + examples: + - true + exclude_rooms_from_sync: + type: array + description: >- + A list of rooms to exclude from sync responses. This is useful for server + administrators wishing to group users into a room without these users + being able to see it from their client. + items: + type: string + default: [] + examples: + - - "!foo:example.com" + opentracing: + type: object + description: >- + These settings enable and configure opentracing, which implements + distributed tracing. This allows you to observe the causal chains of + events across servers including requests, key lookups etc., across any + server running synapse or any other services which support opentracing + (specifically those implemented with Jaeger). + properties: + enabled: + type: boolean + description: Whether tracing is enabled. Set to true to enable. + default: false + homeserver_whitelist: + type: array + description: >- + The list of homeservers we wish to send and receive span contexts and + span baggage. See [here](../../opentracing.md) for more. This is a + list of regexes which are matched against the `server_name` of the + homeserver. If the list is empty, no servers are matched. + items: + type: string + default: [] + force_tracing_for_users: + type: array + description: >- + A list of the matrix IDs of users whose requests will always be + traced, even if the tracing system would otherwise drop the traces due + to probabilistic sampling. + items: + type: string + default: [] + jaeger_config: + type: object + description: >- + Jaeger can be configured to sample traces at different rates. All + configuration options provided by Jaeger can be set here. Jaeger's + configuration is mostly related to trace sampling which is documented + [here](https://www.jaegertracing.io/docs/latest/sampling/). + default: {} + examples: + - enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + jaeger_config: + sampler: + type: const + param: 1 + logging: false + worker_replication_secret: + type: ["string", "null"] + description: >- + A shared secret used by the replication APIs on the main process to + authenticate HTTP requests from workers. + + + If unset or null, traffic between the workers and the main process is not + authenticated. + + + Replacing an existing `worker_replication_secret` with a new one will + break communication with all workers that have not yet updated their + secret. + default: null + examples: + - secret_secret + worker_replication_secret_path: + type: ["string", "null"] + description: >- + An alternative to + [`worker_replication_secret`](#worker_replication_secret): allows the + secret to be specified in an external file. + + + The file should be a plain text file, containing only the secret. Synapse + reads the secret from the given file once at startup. + + + _Added in Synapse 1.126.0._ + default: null + examples: + - /path/to/secrets/file + start_pushers: + type: boolean + description: >- + Unnecessary to set if using [`pusher_instances`](#pusher_instances) with + [`generic_workers`](../../workers.md#synapseappgeneric_worker). + + + Controls sending of push notifications on the main process. Set to `false` + if using a [pusher worker](../../workers.md#synapseapppusher). + default: true + examples: + - false + pusher_instances: + type: array + description: >- + It is possible to scale the processes that handle sending push + notifications to [sygnal](https://github.com/matrix-org/sygnal) and email + by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) + and adding it's [`worker_name`](#worker_name) to a `pusher_instances` map. + Doing so will remove handling of this function from the main process. + Multiple workers can be added to this map, in which case the work is + balanced across them. Ensure the main process and all pusher workers are + restarted after changing this option. + items: + type: string + default: [] + examples: + - - pusher_worker1 + - - pusher_worker1 + - pusher_worker2 + send_federation: + type: boolean + description: >- + Unnecessary to set if using + [`federation_sender_instances`](#federation_sender_instances) with + [`generic_workers`](../../workers.md#synapseappgeneric_worker). + + + Controls sending of outbound federation transactions on the main process. + Set to `false` if using a [federation sender + worker](../../workers.md#synapseappfederation_sender). + default: true + examples: + - false + federation_sender_instances: + type: array + description: >- + It is possible to scale the processes that handle sending outbound + federation requests by running a + [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding + it's [`worker_name`](#worker_name) to a `federation_sender_instances` map. + Doing so will remove handling of this function from the main process. + Multiple workers can be added to this map, in which case the work is + balanced across them. + + + The way that the load balancing works is any outbound federation request + will be assigned to a federation sender worker based on the hash of the + destination server name. This means that all requests being sent to the + same destination will be processed by the same worker instance. Multiple + `federation_sender_instances` are useful if there is a federation with + multiple servers. + + + This configuration setting must be shared between all workers handling + federation sending, and if changed all federation sender workers must be + stopped at the same time and then started, to ensure that all instances + are running with the same config (otherwise events may be dropped). + items: + type: string + default: [] + examples: + - - federation_sender1 + - - federation_sender1 + - federation_sender2 + instance_map: + type: object + description: >- + When using workers this should be a map from [`worker_name`](#worker_name) + to the HTTP replication listener of the worker, if configured, and to the + main process. Each worker declared under + [`stream_writers`](../../workers.md#stream-writers) and + [`outbound_federation_restricted_to`](#outbound_federation_restricted_to) + needs a HTTP replication listener, and that listener should be included in + the `instance_map`. The main process also needs an entry on the + `instance_map`, and it should be listed under `main` **if even one other + worker exists**. Ensure the port matches with what is declared inside the + `listener` block for a `replication` listener. + additionalProperties: + type: object + default: {} + examples: + - main: + host: localhost + port: 8030 + worker1: + host: localhost + port: 8034 + other: + host: localhost + port: 8035 + tls: true + - main: + path: /run/synapse/main_replication.sock + worker1: + path: /run/synapse/worker1_replication.sock + stream_writers: + type: object + description: >- + Experimental: When using workers you can define which workers should + handle writing to streams such as event persistence and typing + notifications. Any worker specified here must also be in the + [`instance_map`](#instance_map). + + + See the list of available streams in the [worker + documentation](../../workers.md#stream-writers). + properties: + events: + type: string + description: Name of a worker assigned to the `events` stream. + typing: + type: string + description: Name of a worker assigned to the `typing` stream. + to_device: + type: string + description: Name of a worker assigned to the `to_device` stream. + account_data: + type: string + description: Name of a worker assigned to the `account_data` stream. + receipts: + type: string + description: Name of a worker assigned to the `receipts` stream. + presence: + type: string + description: Name of a worker assigned to the `presence` stream. + push_rules: + type: string + description: Name of a worker assigned to the `push_rules` stream. + default: {} + examples: + - events: worker1 + typing: worker1 + outbound_federation_restricted_to: + type: array + description: >- + When using workers, you can restrict outbound federation traffic to only + go through a specific subset of workers. Any worker specified here must + also be in the [`instance_map`](#instance_map). + [`worker_replication_secret`](#worker_replication_secret) must also be + configured to authorize inter-worker communication. + + + Also see the [worker + documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers) + for more info. + + + _Added in Synapse 1.89.0._ + items: + type: string + default: [] + examples: + - - federation_sender1 + - federation_sender2 + run_background_tasks_on: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#background-tasks) that is used to run + background tasks (e.g. cleaning up expired data). If not provided this + defaults to the main process. + default: null + examples: + - worker1 + update_user_directory_from_worker: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#updating-the-user-directory) that is used to + update the user directory. If not provided this defaults to the main + process. + + + _Added in Synapse 1.59.0._ + default: null + examples: + - worker1 + notify_appservices_from_worker: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#notifying-application-services) that is used + to send output traffic to Application Services. If not provided this + defaults to the main process. + + + _Added in Synapse 1.59.0._ + default: null + examples: + - worker1 + media_instance_running_background_jobs: + type: ["string", "null"] + description: >- + The [worker](../../workers.md#synapseappmedia_repository) that is used to + run background tasks for media repository. If running multiple media + repositories you must configure a single instance to run the background + tasks. If not provided this defaults to the main process or your single + `media_repository` worker. + + + _Added in Synapse 1.16.0._ + default: null + examples: + - worker1 + redis: + type: object + description: >- + Configuration for Redis when using workers. This *must* be enabled when + using workers. + + + _Added in Synapse 1.78.0._ + + + _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, + private\_key\_file, ca\_file and ca\_path attributes_ + + + _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ + + + _Changed in Synapse 1.116.0: Added password\_path_ + properties: + enabled: + type: boolean + description: Whether to use Redis support. + default: false + host: + type: string + description: Optional host to use to connect to Redis. + default: localhost + port: + type: integer + description: Optional port to use to connect to Redis. + default: 6379 + path: + type: string + description: >- + The full path to a local Unix socket file. **If this is used, `host` + and `port` are ignored.** + default: /tmp/redis.sock + password: + type: ["string", "null"] + description: Optional password if configured on the Redis instance. + default: null + password_path: + type: ["string", "null"] + description: >- + Alternative to `password`, reading the password from an external file. + The file should be a plain text file, containing only the password. + Synapse reads the password from the given file once at startup. + default: null + dbid: + type: ["string", "null"] + description: >- + Optional redis dbid if needs to connect to specific redis logical db. + default: null + use_tls: + type: boolean + description: Whether to use a TLS connection. + default: false + certificate_file: + type: ["string", "null"] + description: Optional path to the certificate file. + default: null + private_key_file: + type: ["string", "null"] + description: Optional path to the private key file. + default: null + ca_file: + type: ["string", "null"] + description: >- + Optional path to the CA certificate file. Use this one or `ca_path` + default: null + ca_path: + type: ["string", "null"] + description: >- + Optional path to the folder containing the CA certificate file. Use + this one or `ca_file` + default: null + examples: + - enabled: true + host: localhost + port: 6379 + password_path: "" + dbid: "" + worker_app: + type: string + description: >- + The type of worker. The currently available worker applications are listed + in [worker documentation](../../workers.md#available-worker-applications). + + + The most common worker is the + [`synapse.app.generic_worker`](../../workers.md#synapseappgeneric_worker). + examples: + - synapse.app.generic_worker + worker_name: + type: string + description: >- + A unique name for the worker. The worker needs a name to be addressed in + further parameters and identification in log files. We strongly recommend + giving each worker a unique `worker_name`. + examples: + - generic_worker1 + worker_listeners: + type: array + description: >- + A worker can handle HTTP requests. To do so, a `worker_listeners` option + must be declared, in the same way as the [`listeners` option](#listeners) + in the shared config. + + + Workers declared in [`stream_writers`](#stream_writers) and + [`instance_map`](#instance_map) will need to include a `replication` + listener here, in order to accept internal HTTP requests from other + workers. + + + Example #2 is using UNIX sockets with a `replication` listener. + default: [] + examples: + - - type: http + port: 8083 + resources: + - names: + - client + - federation + - - type: http + path: /run/synapse/worker_replication.sock + resources: + - names: + - replication + - type: http + path: /run/synapse/worker_public.sock + resources: + - names: + - client + - federation + worker_manhole: + type: ["integer", "null"] + description: >- + A worker may have a listener for [`manhole`](../../manhole.md). It allows + server administrators to access a Python shell on the worker. + + + The example below is a short form for + + ```yaml + + worker_listeners: + - port: 9000 + bind_addresses: ['127.0.0.1'] + type: manhole + ``` + + + It needs also an additional [`manhole_settings`](#manhole_settings) + configuration. + default: null + examples: + - 9000 + worker_daemonize: + type: boolean + description: >- + Specifies whether the worker should be started as a daemon process. If + Synapse is being managed by [systemd](../../systemd-with-workers/), this + option must be omitted or set to `false`. + default: false + examples: + - true + worker_pid_file: + type: ["string", "null"] + description: >- + When running a worker as a daemon, we need a place to store the + [PID](https://en.wikipedia.org/wiki/Process_identifier) of the worker. + This option defines the location of that "pid file". + + + This option is required if `worker_daemonize` is `true` and ignored + otherwise. + + + See also the [`pid_file` option](#pid_file) option for the main Synapse + process. + default: null + examples: + - DATADIR/generic_worker1.pid + worker_log_config: + type: ["string", "null"] + description: >- + This option specifies a yaml python logging config file as described + [here](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema). + See also the [`log_config` option](#log_config) option for the main + Synapse process. + default: null + examples: + - /etc/matrix-synapse/generic-worker-log.yaml + background_updates: + type: object + description: >- + Background updates are database updates that are run in the background in + batches. The duration, minimum batch size, default batch size, whether to + sleep between batches and if so, how long to sleep can all be configured. + This is helpful to speed up or slow down the updates. + properties: + background_update_duration_ms: + type: integer + description: How long in milliseconds to run a batch of background updates for. + default: 100 + sleep_enabled: + type: boolean + description: Whether to sleep between updates. + default: true + sleep_duration_ms: + type: integer + description: If sleeping between updates, how long in milliseconds to sleep for. + default: 1000 + min_batch_size: + type: integer + description: >- + Minimum size a batch of background updates can be. Must be greater than 0. + default: 1 + default_batch_size: + type: integer + description: >- + The batch size to use for the first iteration of a new background update. + default: 100 + examples: + - background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 + auto_accept_invites: + type: object + description: >- + Automatically accepting invites controls whether users are presented with + an invite request or if they are instead automatically joined to a room + when receiving an invite. Set the `enabled` sub-option to true to enable + auto-accepting invites. + + + NOTE: Care should be taken not to enable this setting if the + `synapse_auto_accept_invite` module is enabled and installed. The two + modules will compete to perform the same task and may result in undesired + behaviour. For example, multiple join events could be generated from a + single invite. + properties: + enabled: + type: boolean + description: Whether to run the auto-accept invites logic. + default: false + only_for_direct_messages: + type: boolean + description: >- + Whether invites should be automatically accepted for all room types, + or only for direct messages. + default: false + only_from_local_users: + type: boolean + description: >- + Whether to only automatically accept invites from users on this homeserver. + default: false + worker_to_run_on: + type: ["string", "null"] + description: >- + Which worker to run this module on. This must match the "worker_name". + If not set or `null`, invites will be accepted on the main process. + default: null + examples: + - enabled: true + only_for_direct_messages: true + only_from_local_users: true + worker_to_run_on: worker_1 +$defs: + bytes: + type: ["string", "integer"] + io.element.type_name: byte size + duration: + type: ["string", "integer"] + io.element.type_name: duration + size: + type: ["string", "integer"] + io.element.type_name: size + 3pidmedium: + type: string + enum: + - email + - msisdn + rc: + type: object + properties: + per_second: + type: number + description: Maximum number of requests a client can send per second. + burst_count: + type: number + description: >- + Maximum number of requests a client can send before being throttled. + database: + type: object + description: >- + The `database` setting defines the database that synapse uses to store all + of its data. + + + For more information on using Synapse with Postgres, see + [here](../../postgres.md). + properties: + name: + type: string + enum: + - sqlite3 + - psycopg2 + description: >- + This option specifies the database engine to use: either `sqlite3` + (for SQLite) or `psycopg2` (for PostgreSQL). If no name is specified + Synapse will default to SQLite. + default: sqlite3 + txn_limit: + type: integer + description: >- + Gives the maximum number of transactions to run per connection before + reconnecting. 0 means no limit. + default: 0 + allow_unsafe_locale: + type: boolean + description: >- + This option is specific to Postgres. Under the default behavior, + Synapse will refuse to start if the postgres db is set to a non-C + locale. You can override this behavior (which is *not* recommended) by + setting `allow_unsafe_locale` to true. Note that doing so may corrupt + your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and + [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + default: false + args: + type: object + description: >- + Gives options which are passed through to the database engine, except + for options starting with `cp_`, which are used to configure the + Twisted connection pool. For a reference to valid arguments, see: + + * for + [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + + * for + [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + + * for [the connection + pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) diff --git a/schema/v1/Do not edit files in this folder b/schema/v1/Do not edit files in this folder new file mode 100644 index 0000000000..c95c3bfed0 --- /dev/null +++ b/schema/v1/Do not edit files in this folder @@ -0,0 +1,2 @@ +If you want to update the meta schema, copy this folder and increase its version +number instead. diff --git a/schema/v1/meta.schema.json b/schema/v1/meta.schema.json new file mode 100644 index 0000000000..0c2c46f1a9 --- /dev/null +++ b/schema/v1/meta.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": true, + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/meta-data": true, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": true, + "https://json-schema.org/draft/2020-12/vocab/content": true, + "https://element-hq.github.io/synapse/latest/schema/v1/vocab/documentation": false + }, + "$ref": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "io.element.type_name": { + "type": "string", + "description": "Human-readable type of a schema that is displayed instead of the standard JSON Schema types like `object` or `integer`. In case the JSON Schema type contains `null`, this information should be presented alongside the human-readable type name.", + "examples": ["duration", "byte size"] + }, + "io.element.post_description": { + "type": "string", + "description": "Additional description of a schema, better suited to be placed less prominently in the generated documentation, e.g., at the end of a section after listings of items and properties.", + "examples": [ + "### Advanced uses\n\nThe spent coffee grounds can be added to compost for improving soil and growing plants." + ] + } + } +} diff --git a/schema/v1/vocab/documentation.html b/schema/v1/vocab/documentation.html new file mode 100644 index 0000000000..6a45f4beb3 --- /dev/null +++ b/schema/v1/vocab/documentation.html @@ -0,0 +1,11 @@ + + + + + + Redirecting to ../meta.schema.json… + + +

Redirecting to ../meta.schema.json…

+ + diff --git a/scripts-dev/gen_config_documentation.py b/scripts-dev/gen_config_documentation.py new file mode 100755 index 0000000000..8e9d402c6a --- /dev/null +++ b/scripts-dev/gen_config_documentation.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +"""Generate Synapse documentation from JSON Schema file.""" + +import json +import re +import sys +from typing import Any, Optional + +import yaml + +HEADER = """ + +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +In addition, configuration options referring to size use the following suffixes: + +* `K` = KiB, or 1024 bytes +* `M` = MiB, or 1,048,576 bytes +* `G` = GiB, or 1,073,741,824 bytes +* `T` = TiB, or 1,099,511,627,776 bytes + +For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes +for a user avatar. + +## Config Validation + +The configuration file can be validated with the following command: +```bash +python -m synapse.config read -c +``` + +To validate the entire file, omit `read `: +```bash +python -m synapse.config -c +``` + +To see how to set other options, check the help reference: +```bash +python -m synapse.config --help +``` + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "### `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. +""" +SECTION_HEADERS = { + "modules": { + "title": "Modules", + "description": ( + "Server admins can expand Synapse's functionality with external " + "modules.\n\n" + "See [here](../../modules/index.md) for more documentation on how " + "to configure or create custom modules for Synapse." + ), + }, + "server_name": { + "title": "Server", + "description": "Define your homeserver name and other base options.", + }, + "admin_contact": { + "title": "Homeserver blocking", + "description": "Useful options for Synapse admins.", + }, + "tls_certificate_path": { + "title": "TLS", + "description": "Options related to TLS.", + }, + "federation_domain_whitelist": { + "title": "Federation", + "description": "Options related to federation.", + }, + "event_cache_size": { + "title": "Caching", + "description": "Options related to caching.", + }, + "database": { + "title": "Database", + "description": "Config options related to database settings.", + }, + "log_config": { + "title": "Logging", + "description": ("Config options related to logging."), + }, + "rc_message": { + "title": "Ratelimiting", + "description": ( + "Options related to ratelimiting in Synapse.\n\n" + "Each ratelimiting configuration is made of two parameters:\n" + "- `per_second`: number of requests a client can send per second.\n" + "- `burst_count`: number of requests a client can send before " + "being throttled." + ), + }, + "enable_authenticated_media": { + "title": "Media Store", + "description": "Config options related to Synapse's media store.", + }, + "recaptcha_public_key": { + "title": "Captcha", + "description": ( + "See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha." + ), + }, + "turn_uris": { + "title": "TURN", + "description": ("Options related to adding a TURN server to Synapse."), + }, + "enable_registration": { + "title": "Registration", + "description": ( + "Registration can be rate-limited using the parameters in the " + "[Ratelimiting](#ratelimiting) section of this manual." + ), + }, + "session_lifetime": { + "title": "User session management", + "description": ("Config options related to user session management."), + }, + "enable_metrics": { + "title": "Metrics", + "description": ("Config options related to metrics."), + }, + "room_prejoin_state": { + "title": "API Configuration", + "description": ("Config settings related to the client/server API."), + }, + "signing_key_path": { + "title": "Signing Keys", + "description": ("Config options relating to signing keys."), + }, + "saml2_config": { + "title": "Single sign-on integration", + "description": ( + "The following settings can be used to make Synapse use a single sign-on provider for authentication, instead of its internal password database.\n\n" + "You will probably also want to set the following options to `false` to disable the regular login/registration flows:\n" + "* [`enable_registration`](#enable_registration)\n" + "* [`password_config.enabled`](#password_config)" + ), + }, + "push": { + "title": "Push", + "description": ("Configuration settings related to push notifications."), + }, + "encryption_enabled_by_default_for_room_type": { + "title": "Rooms", + "description": ("Config options relating to rooms."), + }, + "opentracing": { + "title": "Opentracing", + "description": ("Configuration options related to Opentracing support."), + }, + "worker_replication_secret": { + "title": "Coordinating workers", + "description": ( + "Configuration options related to workers which belong in the main config file (usually called `homeserver.yaml`). A Synapse deployment can scale horizontally by running multiple Synapse processes called _workers_. Incoming requests are distributed between workers to handle higher loads. Some workers are privileged and can accept requests from other workers.\n\n" + "As a result, the worker configuration is divided into two parts.\n\n" + "1. The first part (in this section of the manual) defines which shardable tasks are delegated to privileged workers. This allows unprivileged workers to make requests to a privileged worker to act on their behalf.\n" + "2. [The second part](#individual-worker-configuration) controls the behaviour of individual workers in isolation.\n\n" + "For guidance on setting up workers, see the [worker documentation](../../workers.md)." + ), + }, + "worker_app": { + "title": "Individual worker configuration", + "description": ( + "These options configure an individual worker, in its worker configuration file. They should be not be provided when configuring the main process.\n\n" + "Note also the configuration above for [coordinating a cluster of workers](#coordinating-workers).\n\n" + "For guidance on setting up workers, see the [worker documentation](../../workers.md)." + ), + }, + "background_updates": { + "title": "Background Updates", + "description": ("Configuration settings related to background updates."), + }, + "auto_accept_invites": { + "title": "Auto Accept Invites", + "description": ( + "Configuration settings related to automatically accepting invites." + ), + }, +} +INDENT = " " + + +has_error = False + + +def error(text: str) -> None: + global has_error + print(f"ERROR: {text}", file=sys.stderr) + has_error = True + + +def indent(text: str, first_line: bool = True) -> str: + """Indents each non-empty line of the given text.""" + text = re.sub(r"(\n)([^\n])", r"\1" + INDENT + r"\2", text) + if first_line: + text = re.sub(r"^([^\n])", INDENT + r"\1", text) + + return text + + +def em(s: Optional[str]) -> str: + """Add emphasis to text.""" + return f"*{s}*" if s else "" + + +def a(s: Optional[str], suffix: str = " ") -> str: + """Appends a space if the given string is not empty.""" + return s + suffix if s else "" + + +def p(s: Optional[str], prefix: str = " ") -> str: + """Prepend a space if the given string is not empty.""" + return prefix + s if s else "" + + +def resolve_local_refs(schema: dict) -> dict: + """Returns the given schema with local $ref properties replaced by their keywords. + + Crude approximation that will override keywords. + """ + defs = schema["$defs"] + + def replace_ref(d: Any) -> Any: + if isinstance(d, dict): + the_def = {} + if "$ref" in d: + # Found a "$ref" key. + def_name = d["$ref"].removeprefix("#/$defs/") + del d["$ref"] + the_def = defs[def_name] + + new_dict = {k: replace_ref(v) for k, v in d.items()} + if common_keys := (new_dict.keys() & the_def.keys()) - {"properties"}: + print( + f"WARN: '{def_name}' overrides keys '{common_keys}'", + file=sys.stderr, + ) + + new_dict_props = new_dict.get("properties", {}) + the_def_props = the_def.get("properties", {}) + if common_props := new_dict_props.keys() & the_def_props.keys(): + print( + f"WARN: '{def_name}' overrides properties '{common_props}'", + file=sys.stderr, + ) + if merged_props := {**new_dict_props, **the_def_props}: + return {**new_dict, **the_def, "properties": merged_props} + else: + return {**new_dict, **the_def} + + elif isinstance(d, list): + return [replace_ref(v) for v in d] + else: + return d + + return replace_ref(schema) + + +def sep(values: dict) -> str: + """Separator between parts of the description.""" + # If description is multiple paragraphs already, add new ones. Otherwise + # append to same paragraph. + return "\n\n" if "\n\n" in values.get("description", "") else " " + + +def type_str(values: dict) -> str: + """Type of the current value.""" + if t := values.get("io.element.type_name"): + # Allow custom overrides for the type name, for documentation clarity + return f"({t})" + if not (t := values.get("type")): + return "" + if not isinstance(t, list): + t = [t] + joined = "|".join(t) + return f"({joined})" + + +def items(values: dict) -> str: + """A block listing properties of array items.""" + if not (items := values.get("items")): + return "" + if not (item_props := items.get("properties")): + return "" + return "\nOptions for each entry include:\n\n" + "\n".join( + sub_section(k, v) for k, v in item_props.items() + ) + + +def properties(values: dict) -> str: + """A block listing object properties.""" + if not (properties := values.get("properties")): + return "" + return "\nThis setting has the following sub-options:\n\n" + "\n".join( + sub_section(k, v) for k, v in properties.items() + ) + + +def sub_section(prop: str, values: dict) -> str: + """Formats a bullet point about the given sub-property.""" + sep = lambda: globals()["sep"](values) + type_str = lambda: globals()["type_str"](values) + items = lambda: globals()["items"](values) + properties = lambda: globals()["properties"](values) + + def default() -> str: + try: + default = values["default"] + return f"Defaults to `{json.dumps(default)}`." + except KeyError: + return "" + + def description() -> str: + if not (description := values.get("description")): + error(f"missing description for {prop}") + return "MISSING DESCRIPTION\n" + + return f"{description}{p(default(), sep())}\n" + + return ( + f"* `{prop}`{p(type_str())}: " + + f"{indent(description(), first_line=False)}" + + indent(items()) + + indent(properties()) + ) + + +def section(prop: str, values: dict) -> str: + """Formats a section about the given property.""" + sep = lambda: globals()["sep"](values) + type_str = lambda: globals()["type_str"](values) + items = lambda: globals()["items"](values) + properties = lambda: globals()["properties"](values) + + def is_simple_default() -> bool: + """Whether the given default is simple enough for a one-liner.""" + if not (d := values.get("default")): + return True + return not isinstance(d, dict) and not isinstance(d, list) + + def default_str() -> str: + try: + default = values["default"] + except KeyError: + t = values.get("type", []) + if "object" == t or "object" in t: + # Skip objects as they probably have child defaults. + return "" + return "There is no default for this option." + + if not is_simple_default(): + # Show complex defaults as a code block instead. + return "" + return f"Defaults to `{json.dumps(default)}`." + + def header() -> str: + try: + title = SECTION_HEADERS[prop]["title"] + description = SECTION_HEADERS[prop]["description"] + return f"## {title}\n\n{description}\n\n---\n" + except KeyError: + return "" + + def title() -> str: + return f"### `{prop}`\n" + + def description() -> str: + if not (description := values.get("description")): + error(f"missing description for {prop}") + return "MISSING DESCRIPTION\n" + return f"\n{a(em(type_str()))}{description}{p(default_str(), sep())}\n" + + def example_str(example: Any) -> str: + return "```yaml\n" + f"{yaml.dump({prop: example}, sort_keys=False)}" + "```\n" + + def default_example() -> str: + if is_simple_default(): + return "" + default_cfg = example_str(values["default"]) + return f"\nDefault configuration:\n{default_cfg}" + + def examples() -> str: + if not (examples := values.get("examples")): + return "" + + examples_str = "\n".join(example_str(e) for e in examples) + + if len(examples) >= 2: + return f"\nExample configurations:\n{examples_str}" + else: + return f"\nExample configuration:\n{examples_str}" + + def post_description() -> str: + # Sometimes it's helpful to have a description after the list of fields, + # e.g. with a subsection that consists only of text. + # This helps with that. + if not (description := values.get("io.element.post_description")): + return "" + return f"\n{description}\n\n" + + return ( + "---\n" + + header() + + title() + + description() + + items() + + properties() + + default_example() + + examples() + + post_description() + ) + + +def main() -> None: + def usage(err_msg: str) -> int: + script_name = (sys.argv[:1] or ["__main__.py"])[0] + print(err_msg, file=sys.stderr) + print(f"Usage: {script_name} ", file=sys.stderr) + print(f"\n{__doc__}", file=sys.stderr) + exit(1) + + def read_json_file_arg() -> Any: + if len(sys.argv) > 2: + exit(usage("Too many arguments.")) + if not (filepath := (sys.argv[1:] or [""])[0]): + exit(usage("No schema file provided.")) + with open(filepath) as f: + return yaml.safe_load(f) + + schema = read_json_file_arg() + schema = resolve_local_refs(schema) + + sections = (section(k, v) for k, v in schema["properties"].items()) + print(HEADER + "".join(sections), end="") + + if has_error: + print("There were errors.", file=sys.stderr) + exit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts-dev/release.py b/scripts-dev/release.py index c556ccaf86..5de5814b17 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -254,6 +254,12 @@ def _prepare() -> None: # Update the version specified in pyproject.toml. subprocess.check_output(["poetry", "version", new_version]) + # Update config schema $id. + schema_file = "schema/synapse-config.schema.yaml" + major_minor_version = ".".join(new_version.split(".")[:2]) + url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" + subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) From a4d8da7a1b6540652b3e0e1b4faa6c4d60b7b46c Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Tue, 3 Jun 2025 12:34:40 +0100 Subject: [PATCH 347/691] Make user_type extensible and allow default user_type to be set (#18456) --- changelog.d/18456.feature | 1 + docs/admin_api/user_admin_api.md | 3 +- .../configuration/config_documentation.md | 18 ++ synapse/api/constants.py | 10 +- synapse/config/_base.pyi | 2 + synapse/config/homeserver.py | 2 + synapse/config/user_types.py | 44 ++++ synapse/handlers/register.py | 4 + synapse/rest/admin/users.py | 8 +- .../storage/databases/main/registration.py | 15 +- tests/handlers/test_register.py | 35 +++ tests/rest/admin/test_user.py | 211 +++++++++++++++--- 12 files changed, 306 insertions(+), 47 deletions(-) create mode 100644 changelog.d/18456.feature create mode 100644 synapse/config/user_types.py diff --git a/changelog.d/18456.feature b/changelog.d/18456.feature new file mode 100644 index 0000000000..706417e341 --- /dev/null +++ b/changelog.d/18456.feature @@ -0,0 +1 @@ +Support configuration of default and extra user types. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index ee22b0db50..31baf96e58 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -163,7 +163,8 @@ Body parameters: - `locked` - **bool**, optional. If unspecified, locked state will be left unchanged. - `user_type` - **string** or null, optional. If not provided, the user type will be not be changed. If `null` is given, the user type will be cleared. - Other allowed options are: `bot` and `support`. + Other allowed options are: `bot` and `support` and any extra values defined in the homserver + [configuration](../usage/configuration/config_documentation.md#user_types). ## List Accounts ### List Accounts (V2) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index b9845dd78f..c014de794d 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -762,6 +762,24 @@ Example configuration: max_event_delay_duration: 24h ``` --- +### `user_types` + +Configuration settings related to the user types feature. + +This setting has the following sub-options: +* `default_user_type`: The default user type to use for registering new users when no value has been specified. + Defaults to none. +* `extra_user_types`: Array of additional user types to allow. These are treated as real users. Defaults to []. + +Example configuration: +```yaml +user_types: + default_user_type: "custom" + extra_user_types: + - "custom" + - "custom2" +``` + ## Homeserver blocking Useful options for Synapse admins. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index c564a8635a..1d0de60b2d 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -185,12 +185,18 @@ ServerNoticeLimitReached: Final = "m.server_notice.usage_limit_reached" class UserTypes: """Allows for user type specific behaviour. With the benefit of hindsight - 'admin' and 'guest' users should also be UserTypes. Normal users are type None + 'admin' and 'guest' users should also be UserTypes. Extra user types can be + added in the configuration. Normal users are type None or one of the extra + user types (if configured). """ SUPPORT: Final = "support" BOT: Final = "bot" - ALL_USER_TYPES: Final = (SUPPORT, BOT) + ALL_BUILTIN_USER_TYPES: Final = (SUPPORT, BOT) + """ + The user types that are built-in to Synapse. Extra user types can be + added in the configuration. + """ class RelationTypes: diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 9c4ec8f713..8b065f175d 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -59,6 +59,7 @@ from synapse.config import ( # noqa: F401 tls, tracer, user_directory, + user_types, voip, workers, ) @@ -122,6 +123,7 @@ class RootConfig: retention: retention.RetentionConfig background_updates: background_updates.BackgroundUpdateConfig auto_accept_invites: auto_accept_invites.AutoAcceptInvitesConfig + user_types: user_types.UserTypesConfig config_classes: List[Type["Config"]] = ... config_files: List[str] diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index e36c0bd6ae..0b2413a83b 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -59,6 +59,7 @@ from .third_party_event_rules import ThirdPartyRulesConfig from .tls import TlsConfig from .tracer import TracerConfig from .user_directory import UserDirectoryConfig +from .user_types import UserTypesConfig from .voip import VoipConfig from .workers import WorkerConfig @@ -107,4 +108,5 @@ class HomeServerConfig(RootConfig): ExperimentalConfig, BackgroundUpdateConfig, AutoAcceptInvitesConfig, + UserTypesConfig, ] diff --git a/synapse/config/user_types.py b/synapse/config/user_types.py new file mode 100644 index 0000000000..2d9c9f7afb --- /dev/null +++ b/synapse/config/user_types.py @@ -0,0 +1,44 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import Any, List, Optional + +from synapse.api.constants import UserTypes +from synapse.types import JsonDict + +from ._base import Config, ConfigError + + +class UserTypesConfig(Config): + section = "user_types" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + user_types: JsonDict = config.get("user_types", {}) + + self.default_user_type: Optional[str] = user_types.get( + "default_user_type", None + ) + self.extra_user_types: List[str] = user_types.get("extra_user_types", []) + + all_user_types: List[str] = [] + all_user_types.extend(UserTypes.ALL_BUILTIN_USER_TYPES) + all_user_types.extend(self.extra_user_types) + + self.all_user_types = all_user_types + + if self.default_user_type is not None: + if self.default_user_type not in all_user_types: + raise ConfigError( + f"Default user type {self.default_user_type} is not in the list of all user types: {all_user_types}" + ) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 3e86349981..d577039a4c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -115,6 +115,7 @@ class RegistrationHandler: self._user_consent_version = self.hs.config.consent.user_consent_version self._server_notices_mxid = hs.config.servernotices.server_notices_mxid self._server_name = hs.hostname + self._user_types_config = hs.config.user_types self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker @@ -306,6 +307,9 @@ class RegistrationHandler: elif default_display_name is None: default_display_name = localpart + if user_type is None: + user_type = self._user_types_config.default_user_type + await self.register_with_store( user_id=user_id, password_hash=password_hash, diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 7b8f1d1b2a..d6725eed8e 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union import attr from synapse._pydantic_compat import StrictBool, StrictInt, StrictStr -from synapse.api.constants import Direction, UserTypes +from synapse.api.constants import Direction from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, @@ -230,6 +230,7 @@ class UserRestServletV2(RestServlet): self.registration_handler = hs.get_registration_handler() self.pusher_pool = hs.get_pusherpool() self._msc3866_enabled = hs.config.experimental.msc3866.enabled + self._all_user_types = hs.config.user_types.all_user_types async def on_GET( self, request: SynapseRequest, user_id: str @@ -277,7 +278,7 @@ class UserRestServletV2(RestServlet): assert_params_in_dict(external_id, ["auth_provider", "external_id"]) user_type = body.get("user_type", None) - if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: + if user_type is not None and user_type not in self._all_user_types: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid user type") set_admin_to = body.get("admin", False) @@ -524,6 +525,7 @@ class UserRegisterServlet(RestServlet): self.reactor = hs.get_reactor() self.nonces: Dict[str, int] = {} self.hs = hs + self._all_user_types = hs.config.user_types.all_user_types def _clear_old_nonces(self) -> None: """ @@ -605,7 +607,7 @@ class UserRegisterServlet(RestServlet): user_type = body.get("user_type", None) displayname = body.get("displayname", None) - if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: + if user_type is not None and user_type not in self._all_user_types: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid user type") if "mac" not in body: diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 1aeae951c5..40c551bcb4 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -583,7 +583,9 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): await self.db_pool.runInteraction("set_shadow_banned", set_shadow_banned_txn) - async def set_user_type(self, user: UserID, user_type: Optional[UserTypes]) -> None: + async def set_user_type( + self, user: UserID, user_type: Optional[Union[UserTypes, str]] + ) -> None: """Sets the user type. Args: @@ -683,7 +685,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): retcol="user_type", allow_none=True, ) - return res is None + return res is None or res not in [UserTypes.BOT, UserTypes.SUPPORT] def is_support_user_txn(self, txn: LoggingTransaction, user_id: str) -> bool: res = self.db_pool.simple_select_one_onecol_txn( @@ -959,10 +961,12 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): return await self.db_pool.runInteraction("count_users", _count_users) async def count_real_users(self) -> int: - """Counts all users without a special user_type registered on the homeserver.""" + """Counts all users without the bot or support user_types registered on the homeserver.""" def _count_users(txn: LoggingTransaction) -> int: - txn.execute("SELECT COUNT(*) FROM users where user_type is null") + txn.execute( + f"SELECT COUNT(*) FROM users WHERE user_type IS NULL OR user_type NOT IN ('{UserTypes.BOT}', '{UserTypes.SUPPORT}')" + ) row = txn.fetchone() assert row is not None return row[0] @@ -2545,7 +2549,8 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): the user, setting their displayname to the given value admin: is an admin user? user_type: type of user. One of the values from api.constants.UserTypes, - or None for a normal user. + a custom value set in the configuration file, or None for a normal + user. shadow_banned: Whether the user is shadow-banned, i.e. they may be told their requests succeeded but we ignore them. approved: Whether to consider the user has already been approved by an diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index dda389c08b..99bd0de834 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -738,6 +738,41 @@ class RegistrationTestCase(unittest.HomeserverTestCase): self.handler.register_user(localpart="bobflimflob", auth_provider_id="saml") ) + def test_register_default_user_type(self) -> None: + """Test that the default user type is none when registering a user.""" + user_id = self.get_success(self.handler.register_user(localpart="user")) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, None) + + def test_register_extra_user_types_valid(self) -> None: + """ + Test that the specified user type is set correctly when registering a user. + n.b. No validation is done on the user type, so this test + is only to ensure that the user type can be set to any value. + """ + user_id = self.get_success( + self.handler.register_user(localpart="user", user_type="anyvalue") + ) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, "anyvalue") + + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + "default_user_type": "extra1", + } + } + ) + def test_register_extra_user_types_with_default(self) -> None: + """Test that the default_user_type in config is set correctly when registering a user.""" + user_id = self.get_success(self.handler.register_user(localpart="user")) + user_info = self.get_success(self.store.get_user_by_id(user_id)) + assert user_info is not None + self.assertEqual(user_info.user_type, "extra1") + async def get_or_create_user( self, requester: Requester, diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index f09f66da00..5f73dbdc4a 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -328,6 +328,61 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Invalid user type", channel.json_body["error"]) + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + } + } + ) + def test_extra_user_type(self) -> None: + """ + Check that the extra user type can be used when registering a user. + """ + + def nonce_mac(user_type: str) -> tuple[str, str]: + """ + Get a nonce and the expected HMAC for that nonce. + """ + channel = self.make_request("GET", self.url) + nonce = channel.json_body["nonce"] + + want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) + want_mac.update( + nonce.encode("ascii") + + b"\x00alice\x00abc123\x00notadmin\x00" + + user_type.encode("ascii") + ) + want_mac_str = want_mac.hexdigest() + + return nonce, want_mac_str + + nonce, mac = nonce_mac("extra1") + # Valid user_type + body = { + "nonce": nonce, + "username": "alice", + "password": "abc123", + "user_type": "extra1", + "mac": mac, + } + channel = self.make_request("POST", self.url, body) + self.assertEqual(200, channel.code, msg=channel.json_body) + + nonce, mac = nonce_mac("extra3") + # Invalid user_type + body = { + "nonce": nonce, + "username": "alice", + "password": "abc123", + "user_type": "extra3", + "mac": mac, + } + channel = self.make_request("POST", self.url, body) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Invalid user type", channel.json_body["error"]) + def test_displayname(self) -> None: """ Test that displayname of new user is set @@ -1186,6 +1241,80 @@ class UsersListTestCase(unittest.HomeserverTestCase): not_user_types=["custom"], ) + @override_config( + { + "user_types": { + "extra_user_types": ["extra1", "extra2"], + } + } + ) + def test_filter_not_user_types_with_extra(self) -> None: + """Tests that the endpoint handles the not_user_types param when extra_user_types are configured""" + + regular_user_id = self.register_user("normalo", "secret") + + extra1_user_id = self.register_user("extra1", "secret") + self.make_request( + "PUT", + "/_synapse/admin/v2/users/" + urllib.parse.quote(extra1_user_id), + {"user_type": "extra1"}, + access_token=self.admin_user_tok, + ) + + def test_user_type( + expected_user_ids: List[str], not_user_types: Optional[List[str]] = None + ) -> None: + """Runs a test for the not_user_types param + Args: + expected_user_ids: Ids of the users that are expected to be returned + not_user_types: List of values for the not_user_types param + """ + + user_type_query = "" + + if not_user_types is not None: + user_type_query = "&".join( + [f"not_user_type={u}" for u in not_user_types] + ) + + test_url = f"{self.url}?{user_type_query}" + channel = self.make_request( + "GET", + test_url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code) + self.assertEqual(channel.json_body["total"], len(expected_user_ids)) + self.assertEqual( + expected_user_ids, + [u["name"] for u in channel.json_body["users"]], + ) + + # Request without user_types → all users expected + test_user_type([self.admin_user, extra1_user_id, regular_user_id]) + + # Request and exclude extra1 user type + test_user_type( + [self.admin_user, regular_user_id], + not_user_types=["extra1"], + ) + + # Request and exclude extra1 and extra2 user types + test_user_type( + [self.admin_user, regular_user_id], + not_user_types=["extra1", "extra2"], + ) + + # Request and exclude empty user types → only expected the extra1 user + test_user_type([extra1_user_id], not_user_types=[""]) + + # Request and exclude an unregistered type → expect all users + test_user_type( + [self.admin_user, extra1_user_id, regular_user_id], + not_user_types=["extra3"], + ) + def test_erasure_status(self) -> None: # Create a new user. user_id = self.register_user("eraseme", "eraseme") @@ -2977,56 +3106,66 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertEqual("@user:test", channel.json_body["name"]) self.assertTrue(channel.json_body["admin"]) + def set_user_type(self, user_type: Optional[str]) -> None: + # Set to user_type + channel = self.make_request( + "PUT", + self.url_other_user, + access_token=self.admin_user_tok, + content={"user_type": user_type}, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual("@user:test", channel.json_body["name"]) + self.assertEqual(user_type, channel.json_body["user_type"]) + + # Get user + channel = self.make_request( + "GET", + self.url_other_user, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual("@user:test", channel.json_body["name"]) + self.assertEqual(user_type, channel.json_body["user_type"]) + def test_set_user_type(self) -> None: """ Test changing user type. """ # Set to support type - channel = self.make_request( - "PUT", - self.url_other_user, - access_token=self.admin_user_tok, - content={"user_type": UserTypes.SUPPORT}, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"]) - - # Get user - channel = self.make_request( - "GET", - self.url_other_user, - access_token=self.admin_user_tok, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"]) + self.set_user_type(UserTypes.SUPPORT) # Change back to a regular user + self.set_user_type(None) + + @override_config({"user_types": {"extra_user_types": ["extra1", "extra2"]}}) + def test_set_user_type_with_extras(self) -> None: + """ + Test changing user type with extra_user_types configured. + """ + + # Check that we can still set to support type + self.set_user_type(UserTypes.SUPPORT) + + # Check that we can set to an extra user type + self.set_user_type("extra2") + + # Change back to a regular user + self.set_user_type(None) + + # Try setting to invalid type channel = self.make_request( "PUT", self.url_other_user, access_token=self.admin_user_tok, - content={"user_type": None}, + content={"user_type": "extra3"}, ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertIsNone(channel.json_body["user_type"]) - - # Get user - channel = self.make_request( - "GET", - self.url_other_user, - access_token=self.admin_user_tok, - ) - - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual("@user:test", channel.json_body["name"]) - self.assertIsNone(channel.json_body["user_type"]) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Invalid user type", channel.json_body["error"]) def test_accidental_deactivation_prevention(self) -> None: """ From c47d8e0ee120727db0687f7f16abe7f560828cf8 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 3 Jun 2025 14:37:27 +0100 Subject: [PATCH 348/691] 1.131.0 --- CHANGES.md | 4 ++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2ddf511110..72091d605c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Synapse 1.131.0 (2025-06-03) + +No significant changes since 1.131.0rc1. + # Synapse 1.131.0rc1 (2025-05-28) ### Features diff --git a/debian/changelog b/debian/changelog index 294e5e449b..8ad7f3184a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.131.0) stable; urgency=medium + + * New Synapse release 1.131.0. + + -- Synapse Packaging team Tue, 03 Jun 2025 14:36:55 +0100 + matrix-synapse-py3 (1.131.0~rc1) stable; urgency=medium * New synapse release 1.131.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 762a095926..d95881b53a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.131.0rc1" +version = "1.131.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From fbe7a898f0380aa194c26b71eccf029d0ac47b5c Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 4 Jun 2025 11:30:45 +0100 Subject: [PATCH 349/691] Pass room_config argument to user_may_create_room spam checker module callback (#18486) This PR adds an additional `room_config` argument to the `user_may_create_room` spam checker module API callback. It will continue to work with implementations of `user_may_create_room` that do not expect the additional parameter. A side affect is that on a room upgrade the spam checker callback is called *after* doing some work to calculate the state rather than before. However, I hope that this is acceptable given the relative infrequency of room upgrades. --- changelog.d/18486.feature | 1 + docs/modules/spam_checker_callbacks.md | 9 +- docs/spam_checker.md | 2 +- synapse/handlers/room.py | 31 ++-- .../callbacks/spamchecker_callbacks.py | 62 +++++-- tests/module_api/test_spamchecker.py | 155 ++++++++++++++++++ 6 files changed, 230 insertions(+), 30 deletions(-) create mode 100644 changelog.d/18486.feature create mode 100644 tests/module_api/test_spamchecker.py diff --git a/changelog.d/18486.feature b/changelog.d/18486.feature new file mode 100644 index 0000000000..7e1c713081 --- /dev/null +++ b/changelog.d/18486.feature @@ -0,0 +1 @@ +Pass room_config argument to user_may_create_room spam checker module callback. diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 063099a127..ee9e8f3abb 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -159,12 +159,19 @@ _First introduced in Synapse v1.37.0_ _Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.132.0: Added the `room_config` argument. Callbacks that only expect a single `user_id` argument are still supported._ + ```python -async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] +async def user_may_create_room(user_id: str, room_config: synapse.module_api.JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] ``` Called when processing a room creation request. +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`). +* `room_config`: The contents of the body of a [/createRoom request](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) as a dictionary. + The callback must return one of: - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still decide to reject it. diff --git a/docs/spam_checker.md b/docs/spam_checker.md index 4ace3512b3..ead0f03595 100644 --- a/docs/spam_checker.md +++ b/docs/spam_checker.md @@ -63,7 +63,7 @@ class ExampleSpamChecker: async def user_may_invite(self, inviter_userid, invitee_userid, room_id): return True # allow all invites - async def user_may_create_room(self, userid): + async def user_may_create_room(self, userid, room_config): return True # allow all room creations async def user_may_create_room_alias(self, userid, room_alias): diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 763f99e028..1ccb6f7171 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -468,17 +468,6 @@ class RoomCreationHandler: """ user_id = requester.user.to_string() - spam_check = await self._spam_checker_module_callbacks.user_may_create_room( - user_id - ) - if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: - raise SynapseError( - 403, - "You are not permitted to create rooms", - errcode=spam_check[0], - additional_fields=spam_check[1], - ) - creation_content: JsonDict = { "room_version": new_room_version.identifier, "predecessor": {"room_id": old_room_id, "event_id": tombstone_event_id}, @@ -585,6 +574,24 @@ class RoomCreationHandler: if current_power_level_int < needed_power_level: user_power_levels[user_id] = needed_power_level + # We construct what the body of a call to /createRoom would look like for passing + # to the spam checker. We don't include a preset here, as we expect the + # initial state to contain everything we need. + spam_check = await self._spam_checker_module_callbacks.user_may_create_room( + user_id, + { + "creation_content": creation_content, + "initial_state": list(initial_state.items()), + }, + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + raise SynapseError( + 403, + "You are not permitted to create rooms", + errcode=spam_check[0], + additional_fields=spam_check[1], + ) + await self._send_events_for_new_room( requester, new_room_id, @@ -786,7 +793,7 @@ class RoomCreationHandler: if not is_requester_admin: spam_check = await self._spam_checker_module_callbacks.user_may_create_room( - user_id + user_id, config ) if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: raise SynapseError( diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index a86b46ba54..9b373ff67c 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -22,6 +22,7 @@ import functools import inspect import logging +from copy import deepcopy from typing import ( TYPE_CHECKING, Any, @@ -120,20 +121,24 @@ USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[ ] ], ] -USER_MAY_CREATE_ROOM_CALLBACK = Callable[ - [str], - Awaitable[ - Union[ - Literal["NOT_SPAM"], - Codes, - # Highly experimental, not officially part of the spamchecker API, may - # disappear without warning depending on the results of ongoing - # experiments. - # Use this to return additional information as part of an error. - Tuple[Codes, JsonDict], - # Deprecated - bool, - ] +USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE = Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + # Deprecated + bool, +] +USER_MAY_CREATE_ROOM_CALLBACK = Union[ + Callable[ + [str, JsonDict], + Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE], + ], + Callable[ # Single argument variant for backwards compatibility + [str], Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE] ], ] USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[ @@ -622,16 +627,41 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM async def user_may_create_room( - self, userid: str + self, userid: str, room_config: JsonDict ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: """Checks if a given user may create a room Args: userid: The ID of the user attempting to create a room + room_config: The room creation configuration which is the body of the /createRoom request """ for callback in self._user_may_create_room_callbacks: with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): - res = await delay_cancellation(callback(userid)) + checker_args = inspect.signature(callback) + # Also ensure backwards compatibility with spam checker callbacks + # that don't expect the room_config argument. + if len(checker_args.parameters) == 2: + callback_with_requester_id = cast( + Callable[ + [str, JsonDict], + Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE], + ], + callback, + ) + # We make a copy of the config to ensure the spam checker cannot modify it. + res = await delay_cancellation( + callback_with_requester_id(userid, deepcopy(room_config)) + ) + else: + callback_without_requester_id = cast( + Callable[ + [str], Awaitable[USER_MAY_CREATE_ROOM_CALLBACK_RETURN_VALUE] + ], + callback, + ) + res = await delay_cancellation( + callback_without_requester_id(userid) + ) if res is True or res is self.NOT_SPAM: continue elif res is False: diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py new file mode 100644 index 0000000000..82790222c8 --- /dev/null +++ b/tests/module_api/test_spamchecker.py @@ -0,0 +1,155 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +from typing import Literal, Union + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.rest import admin, login, room, room_upgrade_rest_servlet +from synapse.server import HomeServer +from synapse.types import Codes, JsonDict +from synapse.util import Clock + +from tests.server import FakeChannel +from tests.unittest import HomeserverTestCase + + +class SpamCheckerTestCase(HomeserverTestCase): + servlets = [ + room.register_servlets, + admin.register_servlets, + login.register_servlets, + room_upgrade_rest_servlet.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self._module_api = homeserver.get_module_api() + self.user_id = self.register_user("user", "password") + self.token = self.login("user", "password") + + def create_room(self, content: JsonDict) -> FakeChannel: + channel = self.make_request( + "POST", + "/_matrix/client/r0/createRoom", + content, + access_token=self.token, + ) + + return channel + + def test_may_user_create_room(self) -> None: + """Test that the may_user_create_room callback is called when a user + creates a room, and that it receives the correct parameters. + """ + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_config["foo"], "baa") + + def test_may_user_create_room_on_upgrade(self) -> None: + """Test that the may_user_create_room callback is called when a room is upgraded.""" + + # First, create a room to upgrade. + channel = self.create_room({"topic": "foo"}) + self.assertEqual(channel.code, 200) + room_id = channel.json_body["room_id"] + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return "NOT_SPAM" + + # Register the callback for spam checking. + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + # Now upgrade the room. + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{room_id}/upgrade", + # This will upgrade a room to the same version, but that's fine. + content={"new_version": DEFAULT_ROOM_VERSION}, + access_token=self.token, + ) + + # Check that the callback was called and the room was upgraded. + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + # Check that the initial state received by callback contains the topic event. + self.assertTrue( + any( + event[0][0] == "m.room.topic" and event[1].get("topic") == "foo" + for event in self.last_room_config["initial_state"] + ) + ) + + def test_may_user_create_room_disallowed(self) -> None: + """Test that the codes response from may_user_create_room callback is respected + and returned via the API. + """ + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return Codes.UNAUTHORIZED + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], Codes.UNAUTHORIZED) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_config["foo"], "baa") + + def test_may_user_create_room_compatibility(self) -> None: + """Test that the may_user_create_room callback is called when a user + creates a room for a module that uses the old callback signature + (without the `room_config` parameter) + """ + + async def user_may_create_room( + user_id: str, + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_user_id = user_id + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) From 379356c0ea2f5d0a0d65d900efdf157417fa6947 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 4 Jun 2025 11:33:10 +0100 Subject: [PATCH 350/691] Add media repository callbacks to module API to control media upload size (#18457) Adds new callbacks for media related functionality: - `get_media_config_for_user` - `is_user_allowed_to_upload_media_of_size` --- changelog.d/18457.feature | 1 + docs/SUMMARY.md | 1 + docs/modules/media_repository_callbacks.md | 56 ++++++++++++++ synapse/module_api/__init__.py | 20 +++++ synapse/module_api/callbacks/__init__.py | 4 + .../callbacks/media_repository_callbacks.py | 76 +++++++++++++++++++ synapse/rest/client/media.py | 11 ++- synapse/rest/media/config_resource.py | 11 ++- synapse/rest/media/upload_resource.py | 24 ++++-- tests/media/test_media_storage.py | 39 ++++++++++ tests/rest/client/test_media.py | 57 ++++++++++++++ 11 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 changelog.d/18457.feature create mode 100644 docs/modules/media_repository_callbacks.md create mode 100644 synapse/module_api/callbacks/media_repository_callbacks.py diff --git a/changelog.d/18457.feature b/changelog.d/18457.feature new file mode 100644 index 0000000000..76374dc3cd --- /dev/null +++ b/changelog.d/18457.feature @@ -0,0 +1 @@ +Add new module API callbacks that allows overriding of media repository maximum upload size. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index fd91d9fa11..abb1d5603c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -49,6 +49,7 @@ - [Background update controller callbacks](modules/background_update_controller_callbacks.md) - [Account data callbacks](modules/account_data_callbacks.md) - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md) + - [Media repository](modules/media_repository_callbacks.md) - [Porting a legacy module to the new interface](modules/porting_legacy_module.md) - [Workers](workers.md) - [Using `synctl` with Workers](synctl_workers.md) diff --git a/docs/modules/media_repository_callbacks.md b/docs/modules/media_repository_callbacks.md new file mode 100644 index 0000000000..0cee3384bb --- /dev/null +++ b/docs/modules/media_repository_callbacks.md @@ -0,0 +1,56 @@ +# Media repository callbacks + +Media repository callbacks allow module developers to customise the behaviour of the +media repository on a per user basis. Media repository callbacks can be registered +using the module API's `register_media_repository_callbacks` method. + +The available media repository callbacks are: + +### `get_media_config_for_user` + +_First introduced in Synapse v1.132.0_ + +```python +async def get_media_config_for_user(user_id: str) -> Optional[JsonDict] +``` + +Called when processing a request from a client for the +[media config endpoint](https://spec.matrix.org/latest/client-server-api/#get_matrixclientv1mediaconfig). + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. + +If the callback returns a dictionary then it will be used as the body of the response to the +client. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `None`, Synapse falls through to the next one. The value of the first +callback that does not return `None` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. + +If no module returns a non-`None` value then the default media config will be returned. + +### `is_user_allowed_to_upload_media_of_size` + +_First introduced in Synapse v1.132.0_ + +```python +async def is_user_allowed_to_upload_media_of_size(user_id: str, size: int) -> bool +``` + +Called before media is accepted for upload from a user, in case the module needs to +enforce a different limit for the particular user. + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) making the request. +* `size`: The size in bytes of media that is being requested to upload. + +If the module returns `False`, the current request will be denied with the error code +`M_TOO_LARGE` and the HTTP status code 413. + +If multiple modules implement this callback, they will be considered in order. If a callback +returns `True`, Synapse falls through to the next one. The value of the first callback that +returns `False` will be used. If this happens, Synapse will not call any of the subsequent +implementations of this callback. diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 7834da759c..4ecdf0f3bb 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -90,6 +90,10 @@ from synapse.module_api.callbacks.account_validity_callbacks import ( ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) +from synapse.module_api.callbacks.media_repository_callbacks import ( + GET_MEDIA_CONFIG_FOR_USER_CALLBACK, + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( CHECK_EVENT_FOR_SPAM_CALLBACK, CHECK_LOGIN_FOR_SPAM_CALLBACK, @@ -360,6 +364,22 @@ class ModuleApi: on_legacy_admin_request=on_legacy_admin_request, ) + def register_media_repository_callbacks( + self, + *, + get_media_config_for_user: Optional[GET_MEDIA_CONFIG_FOR_USER_CALLBACK] = None, + is_user_allowed_to_upload_media_of_size: Optional[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = None, + ) -> None: + """Registers callbacks for media repository capabilities. + Added in Synapse v1.x.x. + """ + return self._callbacks.media_repository.register_callbacks( + get_media_config_for_user=get_media_config_for_user, + is_user_allowed_to_upload_media_of_size=is_user_allowed_to_upload_media_of_size, + ) + def register_third_party_rules_callbacks( self, *, diff --git a/synapse/module_api/callbacks/__init__.py b/synapse/module_api/callbacks/__init__.py index c20d9543fb..a36c0fc7c6 100644 --- a/synapse/module_api/callbacks/__init__.py +++ b/synapse/module_api/callbacks/__init__.py @@ -27,6 +27,9 @@ if TYPE_CHECKING: from synapse.module_api.callbacks.account_validity_callbacks import ( AccountValidityModuleApiCallbacks, ) +from synapse.module_api.callbacks.media_repository_callbacks import ( + MediaRepositoryModuleApiCallbacks, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( SpamCheckerModuleApiCallbacks, ) @@ -38,5 +41,6 @@ from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( class ModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: self.account_validity = AccountValidityModuleApiCallbacks() + self.media_repository = MediaRepositoryModuleApiCallbacks(hs) self.spam_checker = SpamCheckerModuleApiCallbacks(hs) self.third_party_event_rules = ThirdPartyEventRulesModuleApiCallbacks(hs) diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py new file mode 100644 index 0000000000..6fa80a8eab --- /dev/null +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -0,0 +1,76 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional + +from synapse.types import JsonDict +from synapse.util.async_helpers import delay_cancellation +from synapse.util.metrics import Measure + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + +GET_MEDIA_CONFIG_FOR_USER_CALLBACK = Callable[[str], Awaitable[Optional[JsonDict]]] + +IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK = Callable[[str, int], Awaitable[bool]] + + +class MediaRepositoryModuleApiCallbacks: + def __init__(self, hs: "HomeServer") -> None: + self.clock = hs.get_clock() + self._get_media_config_for_user_callbacks: List[ + GET_MEDIA_CONFIG_FOR_USER_CALLBACK + ] = [] + self._is_user_allowed_to_upload_media_of_size_callbacks: List[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = [] + + def register_callbacks( + self, + get_media_config_for_user: Optional[GET_MEDIA_CONFIG_FOR_USER_CALLBACK] = None, + is_user_allowed_to_upload_media_of_size: Optional[ + IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK + ] = None, + ) -> None: + """Register callbacks from module for each hook.""" + if get_media_config_for_user is not None: + self._get_media_config_for_user_callbacks.append(get_media_config_for_user) + + if is_user_allowed_to_upload_media_of_size is not None: + self._is_user_allowed_to_upload_media_of_size_callbacks.append( + is_user_allowed_to_upload_media_of_size + ) + + async def get_media_config_for_user(self, user_id: str) -> Optional[JsonDict]: + for callback in self._get_media_config_for_user_callbacks: + with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + res: Optional[JsonDict] = await delay_cancellation(callback(user_id)) + if res: + return res + + return None + + async def is_user_allowed_to_upload_media_of_size( + self, user_id: str, size: int + ) -> bool: + for callback in self._is_user_allowed_to_upload_media_of_size_callbacks: + with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + res: bool = await delay_cancellation(callback(user_id, size)) + if not res: + return res + + return True diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 25b302370f..4c044ae900 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -102,10 +102,17 @@ class MediaConfigResource(RestServlet): self.clock = hs.get_clock() self.auth = hs.get_auth() self.limits_dict = {"m.upload.size": config.media.max_upload_size} + self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository async def on_GET(self, request: SynapseRequest) -> None: - await self.auth.get_user_by_req(request) - respond_with_json(request, 200, self.limits_dict, send_cors=True) + requester = await self.auth.get_user_by_req(request) + user_specific_config = ( + await self.media_repository_callbacks.get_media_config_for_user( + requester.user.to_string(), + ) + ) + response = user_specific_config if user_specific_config else self.limits_dict + respond_with_json(request, 200, response, send_cors=True) class ThumbnailResource(RestServlet): diff --git a/synapse/rest/media/config_resource.py b/synapse/rest/media/config_resource.py index 80462d65d3..b014e91bdb 100644 --- a/synapse/rest/media/config_resource.py +++ b/synapse/rest/media/config_resource.py @@ -40,7 +40,14 @@ class MediaConfigResource(RestServlet): self.clock = hs.get_clock() self.auth = hs.get_auth() self.limits_dict = {"m.upload.size": config.media.max_upload_size} + self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository async def on_GET(self, request: SynapseRequest) -> None: - await self.auth.get_user_by_req(request) - respond_with_json(request, 200, self.limits_dict, send_cors=True) + requester = await self.auth.get_user_by_req(request) + user_specific_config = ( + await self.media_repository_callbacks.get_media_config_for_user( + requester.user.to_string() + ) + ) + response = user_specific_config if user_specific_config else self.limits_dict + respond_with_json(request, 200, response, send_cors=True) diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index 359d006f04..572f7897fd 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -50,9 +50,12 @@ class BaseUploadServlet(RestServlet): self.server_name = hs.hostname self.auth = hs.get_auth() self.max_upload_size = hs.config.media.max_upload_size + self._media_repository_callbacks = ( + hs.get_module_api_callbacks().media_repository + ) - def _get_file_metadata( - self, request: SynapseRequest + async def _get_file_metadata( + self, request: SynapseRequest, user_id: str ) -> Tuple[int, Optional[str], str]: raw_content_length = request.getHeader("Content-Length") if raw_content_length is None: @@ -67,7 +70,14 @@ class BaseUploadServlet(RestServlet): code=413, errcode=Codes.TOO_LARGE, ) - + if not await self._media_repository_callbacks.is_user_allowed_to_upload_media_of_size( + user_id, content_length + ): + raise SynapseError( + msg="Upload request body is too large", + code=413, + errcode=Codes.TOO_LARGE, + ) args: Dict[bytes, List[bytes]] = request.args # type: ignore upload_name_bytes = parse_bytes_from_args(args, "filename") if upload_name_bytes: @@ -104,7 +114,9 @@ class UploadServlet(BaseUploadServlet): async def on_POST(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) - content_length, upload_name, media_type = self._get_file_metadata(request) + content_length, upload_name, media_type = await self._get_file_metadata( + request, requester.user.to_string() + ) try: content: IO = request.content # type: ignore @@ -152,7 +164,9 @@ class AsyncUploadServlet(BaseUploadServlet): async with lock: await self.media_repo.verify_can_upload(media_id, requester.user) - content_length, upload_name, media_type = self._get_file_metadata(request) + content_length, upload_name, media_type = await self._get_file_metadata( + request, requester.user.to_string() + ) try: content: IO = request.content # type: ignore diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 31dc32d67e..2f7cf4569b 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -1360,3 +1360,42 @@ class MediaHashesTestCase(unittest.HomeserverTestCase): store_media.sha256, SMALL_PNG_SHA256, ) + + +class MediaRepoSizeModuleCallbackTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + admin.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + self.mock_result = True # Allow all uploads by default + + hs.get_module_api().register_media_repository_callbacks( + is_user_allowed_to_upload_media_of_size=self.is_user_allowed_to_upload_media_of_size, + ) + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + async def is_user_allowed_to_upload_media_of_size( + self, user_id: str, size: int + ) -> bool: + self.last_user_id = user_id + self.last_size = size + return self.mock_result + + def test_upload_allowed(self) -> None: + self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=200) + assert self.last_user_id == self.user + assert self.last_size == len(SMALL_PNG) + + def test_upload_not_allowed(self) -> None: + self.mock_result = False + self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=413) + assert self.last_user_id == self.user + assert self.last_size == len(SMALL_PNG) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 9ad8ecf1cd..6ee761e44b 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1618,6 +1618,63 @@ class MediaConfigTest(unittest.HomeserverTestCase): ) +class MediaConfigModuleCallbackTestCase(unittest.HomeserverTestCase): + servlets = [ + media.register_servlets, + admin.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + config = self.default_config() + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + config["media_store_path"] = self.media_store_path + + provider_config = { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + config["media_storage_providers"] = [provider_config] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + + hs.get_module_api().register_media_repository_callbacks( + get_media_config_for_user=self.get_media_config_for_user, + ) + + async def get_media_config_for_user( + self, + user_id: str, + ) -> Optional[JsonDict]: + # We echo back the user_id and set a custom upload size. + return {"m.upload.size": 1024, "user_id": user_id} + + def test_media_config(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/config", + shorthand=False, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["m.upload.size"], 1024) + self.assertEqual(channel.json_body["user_id"], self.user) + + class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): servlets = [ media.register_servlets, From 28f21b403683619047668d5974219fdff8a33dfd Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 4 Jun 2025 12:26:04 +0100 Subject: [PATCH 351/691] Add user_may_send_state_event callback to spam checker module API (#18455) --- changelog.d/18455.feature | 1 + docs/modules/spam_checker_callbacks.md | 30 +++++++ synapse/module_api/__init__.py | 3 + .../callbacks/spamchecker_callbacks.py | 57 ++++++++++++ synapse/rest/client/room.py | 20 +++++ tests/module_api/test_spamchecker.py | 89 +++++++++++++++++++ 6 files changed, 200 insertions(+) create mode 100644 changelog.d/18455.feature diff --git a/changelog.d/18455.feature b/changelog.d/18455.feature new file mode 100644 index 0000000000..f1b90e4a60 --- /dev/null +++ b/changelog.d/18455.feature @@ -0,0 +1 @@ +Add user_may_send_state_event callback to spam checker module API. \ No newline at end of file diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index ee9e8f3abb..6dfa2a7b25 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -246,6 +246,36 @@ be used. If this happens, Synapse will not call any of the subsequent implementa this callback. +### `user_may_send_state_event` + +_First introduced in Synapse v1.132.0_ + +```python +async def user_may_send_state_event(user_id: str, room_id: str, event_type: str, state_key: str, content: JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"] +``` + +Called when processing a request to [send state events](https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3roomsroomidstateeventtypestatekey) to a room. + +The arguments passed to this callback are: + +* `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`) sending the state event. +* `room_id`: The ID of the room that the requested state event is being sent to. +* `event_type`: The requested type of event. +* `state_key`: The requested state key. +* `content`: The requested event contents. + +The callback must return one of: + - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. + - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. +The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback. + ### `check_username_for_spam` diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 4ecdf0f3bb..3148957229 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -107,6 +107,7 @@ from synapse.module_api.callbacks.spamchecker_callbacks import ( USER_MAY_JOIN_ROOM_CALLBACK, USER_MAY_PUBLISH_ROOM_CALLBACK, USER_MAY_SEND_3PID_INVITE_CALLBACK, + USER_MAY_SEND_STATE_EVENT_CALLBACK, SpamCheckerModuleApiCallbacks, ) from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( @@ -315,6 +316,7 @@ class ModuleApi: USER_MAY_CREATE_ROOM_ALIAS_CALLBACK ] = None, user_may_publish_room: Optional[USER_MAY_PUBLISH_ROOM_CALLBACK] = None, + user_may_send_state_event: Optional[USER_MAY_SEND_STATE_EVENT_CALLBACK] = None, check_username_for_spam: Optional[CHECK_USERNAME_FOR_SPAM_CALLBACK] = None, check_registration_for_spam: Optional[ CHECK_REGISTRATION_FOR_SPAM_CALLBACK @@ -339,6 +341,7 @@ class ModuleApi: check_registration_for_spam=check_registration_for_spam, check_media_file_for_spam=check_media_file_for_spam, check_login_for_spam=check_login_for_spam, + user_may_send_state_event=user_may_send_state_event, ) def register_account_validity_callbacks( diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index 9b373ff67c..bea5c5badf 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -173,6 +173,20 @@ USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[ ] ], ] +USER_MAY_SEND_STATE_EVENT_CALLBACK = Callable[ + [str, str, str, str, JsonDict], + Awaitable[ + Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + ] + ], +] CHECK_USERNAME_FOR_SPAM_CALLBACK = Union[ Callable[[UserProfile], Awaitable[bool]], Callable[[UserProfile, str], Awaitable[bool]], @@ -337,6 +351,9 @@ class SpamCheckerModuleApiCallbacks: USER_MAY_SEND_3PID_INVITE_CALLBACK ] = [] self._user_may_create_room_callbacks: List[USER_MAY_CREATE_ROOM_CALLBACK] = [] + self._user_may_send_state_event_callbacks: List[ + USER_MAY_SEND_STATE_EVENT_CALLBACK + ] = [] self._user_may_create_room_alias_callbacks: List[ USER_MAY_CREATE_ROOM_ALIAS_CALLBACK ] = [] @@ -372,6 +389,7 @@ class SpamCheckerModuleApiCallbacks: ] = None, check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None, check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None, + user_may_send_state_event: Optional[USER_MAY_SEND_STATE_EVENT_CALLBACK] = None, ) -> None: """Register callbacks from module for each hook.""" if check_event_for_spam is not None: @@ -396,6 +414,11 @@ class SpamCheckerModuleApiCallbacks: if user_may_create_room is not None: self._user_may_create_room_callbacks.append(user_may_create_room) + if user_may_send_state_event is not None: + self._user_may_send_state_event_callbacks.append( + user_may_send_state_event, + ) + if user_may_create_room_alias is not None: self._user_may_create_room_alias_callbacks.append( user_may_create_room_alias, @@ -683,6 +706,40 @@ class SpamCheckerModuleApiCallbacks: return self.NOT_SPAM + async def user_may_send_state_event( + self, + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: + """Checks if a given user may create a room with a given visibility + Args: + user_id: The ID of the user attempting to create a room + room_id: The ID of the room that the event will be sent to + event_type: The type of the state event + state_key: The state key of the state event + content: The content of the state event + """ + for callback in self._user_may_send_state_event_callbacks: + with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + # We make a copy of the content to ensure that the spam checker cannot modify it. + res = await delay_cancellation( + callback(user_id, room_id, event_type, state_key, deepcopy(content)) + ) + if res is self.NOT_SPAM: + continue + elif isinstance(res, synapse.api.errors.Codes): + return res, {} + else: + logger.warning( + "Module returned invalid value, rejecting room creation as spam" + ) + return synapse.api.errors.Codes.FORBIDDEN, {} + + return self.NOT_SPAM + async def user_may_create_room_alias( self, userid: str, room_alias: RoomAlias ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 03e7bc0a24..bb41575d46 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -198,6 +198,7 @@ class RoomStateEventRestServlet(RestServlet): self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() self._max_event_delay_ms = hs.config.server.max_event_delay_ms + self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/state/$eventtype @@ -289,6 +290,25 @@ class RoomStateEventRestServlet(RestServlet): content = parse_json_object_from_request(request) + is_requester_admin = await self.auth.is_server_admin(requester) + if not is_requester_admin: + spam_check = ( + await self._spam_checker_module_callbacks.user_may_send_state_event( + user_id=requester.user.to_string(), + room_id=room_id, + event_type=event_type, + state_key=state_key, + content=content, + ) + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + raise SynapseError( + 403, + "You are not permitted to send the state event", + errcode=spam_check[0], + additional_fields=spam_check[1], + ) + origin_server_ts = None if requester.app_service: origin_server_ts = parse_integer(request, "ts") diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py index 82790222c8..926fe30b43 100644 --- a/tests/module_api/test_spamchecker.py +++ b/tests/module_api/test_spamchecker.py @@ -153,3 +153,92 @@ class SpamCheckerTestCase(HomeserverTestCase): channel = self.create_room({"foo": "baa"}) self.assertEqual(channel.code, 200) self.assertEqual(self.last_user_id, self.user_id) + + def test_user_may_send_state_event(self) -> None: + """Test that the user_may_send_state_event callback is called when a state event + is sent, and that it receives the correct parameters. + """ + + async def user_may_send_state_event( + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_user_id = user_id + self.last_room_id = room_id + self.last_event_type = event_type + self.last_state_key = state_key + self.last_content = content + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_send_state_event=user_may_send_state_event + ) + + channel = self.create_room({}) + self.assertEqual(channel.code, 200) + + room_id = channel.json_body["room_id"] + + event_type = "test.event.type" + state_key = "test.state.key" + channel = self.make_request( + "PUT", + "/_matrix/client/r0/rooms/%s/state/%s/%s" + % ( + room_id, + event_type, + state_key, + ), + content={"foo": "bar"}, + access_token=self.token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_id, room_id) + self.assertEqual(self.last_event_type, event_type) + self.assertEqual(self.last_state_key, state_key) + self.assertEqual(self.last_content, {"foo": "bar"}) + + def test_user_may_send_state_event_disallows(self) -> None: + """Test that the user_may_send_state_event callback is called when a state event + is sent, and that the response is honoured. + """ + + async def user_may_send_state_event( + user_id: str, + room_id: str, + event_type: str, + state_key: str, + content: JsonDict, + ) -> Union[Literal["NOT_SPAM"], Codes]: + return Codes.FORBIDDEN + + self._module_api.register_spam_checker_callbacks( + user_may_send_state_event=user_may_send_state_event + ) + + channel = self.create_room({}) + self.assertEqual(channel.code, 200) + + room_id = channel.json_body["room_id"] + + event_type = "test.event.type" + state_key = "test.state.key" + channel = self.make_request( + "PUT", + "/_matrix/client/r0/rooms/%s/state/%s/%s" + % ( + room_id, + event_type, + state_key, + ), + content={"foo": "bar"}, + access_token=self.token, + ) + + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) From 9b2bc75ed49ea52008dc787a09337ca4a843d8e7 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 4 Jun 2025 13:09:11 +0100 Subject: [PATCH 352/691] Add ratelimit callbacks to module API to allow dynamic ratelimiting (#18458) --- changelog.d/18458.feature | 1 + docs/SUMMARY.md | 3 +- docs/modules/ratelimit_callbacks.md | 33 ++++++++++ synapse/api/ratelimiting.py | 24 ++++++- synapse/handlers/room_member.py | 3 + synapse/module_api/__init__.py | 19 +++++- synapse/module_api/callbacks/__init__.py | 4 ++ .../callbacks/ratelimit_callbacks.py | 62 +++++++++++++++++++ synapse/storage/databases/main/room.py | 2 +- tests/api/test_ratelimiting.py | 50 +++++++++++++++ 10 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18458.feature create mode 100644 docs/modules/ratelimit_callbacks.md create mode 100644 synapse/module_api/callbacks/ratelimit_callbacks.py diff --git a/changelog.d/18458.feature b/changelog.d/18458.feature new file mode 100644 index 0000000000..92f11a8a2c --- /dev/null +++ b/changelog.d/18458.feature @@ -0,0 +1 @@ +Add a new module API callback that allows overriding of per user ratelimits. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index abb1d5603c..40d70a4485 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -49,7 +49,8 @@ - [Background update controller callbacks](modules/background_update_controller_callbacks.md) - [Account data callbacks](modules/account_data_callbacks.md) - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md) - - [Media repository](modules/media_repository_callbacks.md) + - [Media repository callbacks](modules/media_repository_callbacks.md) + - [Ratelimit callbacks](modules/ratelimit_callbacks.md) - [Porting a legacy module to the new interface](modules/porting_legacy_module.md) - [Workers](workers.md) - [Using `synctl` with Workers](synctl_workers.md) diff --git a/docs/modules/ratelimit_callbacks.md b/docs/modules/ratelimit_callbacks.md new file mode 100644 index 0000000000..bf923c045e --- /dev/null +++ b/docs/modules/ratelimit_callbacks.md @@ -0,0 +1,33 @@ +# Ratelimit callbacks + +Ratelimit callbacks allow module developers to override ratelimit settings dynamically whilst +Synapse is running. Ratelimit callbacks can be registered using the module API's +`register_ratelimit_callbacks` method. + +The available ratelimit callbacks are: + +### `get_ratelimit_override_for_user` + +_First introduced in Synapse v1.132.0_ + +```python +async def get_ratelimit_override_for_user(user: str, limiter_name: str) -> Optional[RatelimitOverride] +``` + +Called when constructing a ratelimiter of a particular type for a user. The module can +return a `messages_per_second` and `burst_count` to be used, or `None` if +the default settings are adequate. The user is represented by their Matrix user ID +(e.g. `@alice:example.com`). The limiter name is usually taken from the `RatelimitSettings` key +value. + +The limiters that are currently supported are: + +- `rc_invites.per_room` +- `rc_invites.per_user` +- `rc_invites.per_issuer` + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `None`, Synapse falls through to the next one. The value of the first +callback that does not return `None` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. If no module returns a non-`None` value +then the default settings will be used. diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 8665b3b765..38e5bdaa75 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -20,7 +20,7 @@ # # -from typing import Dict, Hashable, Optional, Tuple +from typing import TYPE_CHECKING, Dict, Hashable, Optional, Tuple from synapse.api.errors import LimitExceededError from synapse.config.ratelimiting import RatelimitSettings @@ -28,6 +28,12 @@ from synapse.storage.databases.main import DataStore from synapse.types import Requester from synapse.util import Clock +if TYPE_CHECKING: + # To avoid circular imports: + from synapse.module_api.callbacks.ratelimit_callbacks import ( + RatelimitModuleApiCallbacks, + ) + class Ratelimiter: """ @@ -72,12 +78,14 @@ class Ratelimiter: store: DataStore, clock: Clock, cfg: RatelimitSettings, + ratelimit_callbacks: Optional["RatelimitModuleApiCallbacks"] = None, ): self.clock = clock self.rate_hz = cfg.per_second self.burst_count = cfg.burst_count self.store = store self._limiter_name = cfg.key + self._ratelimit_callbacks = ratelimit_callbacks # A dictionary representing the token buckets tracked by this rate # limiter. Each entry maps a key of arbitrary type to a tuple representing: @@ -165,6 +173,20 @@ class Ratelimiter: if override and not override.messages_per_second: return True, -1.0 + if requester and self._ratelimit_callbacks: + # Check if the user has a custom rate limit for this specific limiter + # as returned by the module API. + module_override = ( + await self._ratelimit_callbacks.get_ratelimit_override_for_user( + requester.user.to_string(), + self._limiter_name, + ) + ) + + if module_override: + rate_hz = module_override.messages_per_second + burst_count = module_override.burst_count + # Override default values if set time_now_s = _time_now_s if _time_now_s is not None else self.clock.time() rate_hz = rate_hz if rate_hz is not None else self.rate_hz diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 70cbbc352b..b0e750c9c7 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -158,6 +158,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_room, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) # Ratelimiter for invites, keyed by recipient (across all rooms, all @@ -166,6 +167,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_user, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) # Ratelimiter for invites, keyed by issuer (across all rooms, all @@ -174,6 +176,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_issuer, + ratelimit_callbacks=hs.get_module_api_callbacks().ratelimit, ) self._third_party_invite_limiter = Ratelimiter( diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 3148957229..e36503a526 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -94,6 +94,9 @@ from synapse.module_api.callbacks.media_repository_callbacks import ( GET_MEDIA_CONFIG_FOR_USER_CALLBACK, IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK, ) +from synapse.module_api.callbacks.ratelimit_callbacks import ( + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( CHECK_EVENT_FOR_SPAM_CALLBACK, CHECK_LOGIN_FOR_SPAM_CALLBACK, @@ -367,6 +370,20 @@ class ModuleApi: on_legacy_admin_request=on_legacy_admin_request, ) + def register_ratelimit_callbacks( + self, + *, + get_ratelimit_override_for_user: Optional[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = None, + ) -> None: + """Registers callbacks for ratelimit capabilities. + Added in Synapse v1.132.0. + """ + return self._callbacks.ratelimit.register_callbacks( + get_ratelimit_override_for_user=get_ratelimit_override_for_user, + ) + def register_media_repository_callbacks( self, *, @@ -376,7 +393,7 @@ class ModuleApi: ] = None, ) -> None: """Registers callbacks for media repository capabilities. - Added in Synapse v1.x.x. + Added in Synapse v1.132.0. """ return self._callbacks.media_repository.register_callbacks( get_media_config_for_user=get_media_config_for_user, diff --git a/synapse/module_api/callbacks/__init__.py b/synapse/module_api/callbacks/__init__.py index a36c0fc7c6..16ef7a4b47 100644 --- a/synapse/module_api/callbacks/__init__.py +++ b/synapse/module_api/callbacks/__init__.py @@ -30,6 +30,9 @@ from synapse.module_api.callbacks.account_validity_callbacks import ( from synapse.module_api.callbacks.media_repository_callbacks import ( MediaRepositoryModuleApiCallbacks, ) +from synapse.module_api.callbacks.ratelimit_callbacks import ( + RatelimitModuleApiCallbacks, +) from synapse.module_api.callbacks.spamchecker_callbacks import ( SpamCheckerModuleApiCallbacks, ) @@ -42,5 +45,6 @@ class ModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: self.account_validity = AccountValidityModuleApiCallbacks() self.media_repository = MediaRepositoryModuleApiCallbacks(hs) + self.ratelimit = RatelimitModuleApiCallbacks(hs) self.spam_checker = SpamCheckerModuleApiCallbacks(hs) self.third_party_event_rules = ThirdPartyEventRulesModuleApiCallbacks(hs) diff --git a/synapse/module_api/callbacks/ratelimit_callbacks.py b/synapse/module_api/callbacks/ratelimit_callbacks.py new file mode 100644 index 0000000000..ced721b407 --- /dev/null +++ b/synapse/module_api/callbacks/ratelimit_callbacks.py @@ -0,0 +1,62 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional + +from synapse.storage.databases.main.room import RatelimitOverride +from synapse.util.async_helpers import delay_cancellation +from synapse.util.metrics import Measure + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + +GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK = Callable[ + [str, str], Awaitable[Optional[RatelimitOverride]] +] + + +class RatelimitModuleApiCallbacks: + def __init__(self, hs: "HomeServer") -> None: + self.clock = hs.get_clock() + self._get_ratelimit_override_for_user_callbacks: List[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = [] + + def register_callbacks( + self, + get_ratelimit_override_for_user: Optional[ + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK + ] = None, + ) -> None: + """Register callbacks from module for each hook.""" + if get_ratelimit_override_for_user is not None: + self._get_ratelimit_override_for_user_callbacks.append( + get_ratelimit_override_for_user + ) + + async def get_ratelimit_override_for_user( + self, user_id: str, limiter_name: str + ) -> Optional[RatelimitOverride]: + for callback in self._get_ratelimit_override_for_user_callbacks: + with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + res: Optional[RatelimitOverride] = await delay_cancellation( + callback(user_id, limiter_name) + ) + if res: + return res + + return None diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 56217fccdf..46491cc0fe 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -77,7 +77,7 @@ logger = logging.getLogger(__name__) @attr.s(slots=True, frozen=True, auto_attribs=True) class RatelimitOverride: - messages_per_second: int + messages_per_second: float burst_count: int diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 1a1cbde74e..5e73f5d5ec 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -1,6 +1,10 @@ +from typing import Optional + from synapse.api.ratelimiting import LimitExceededError, Ratelimiter from synapse.appservice import ApplicationService from synapse.config.ratelimiting import RatelimitSettings +from synapse.module_api.callbacks.ratelimit_callbacks import RatelimitModuleApiCallbacks +from synapse.storage.databases.main.room import RatelimitOverride from synapse.types import create_requester from tests import unittest @@ -440,3 +444,49 @@ class TestRatelimiter(unittest.HomeserverTestCase): limiter.can_do_action(requester=None, key="a", _time_now_s=20.0) ) self.assertTrue(success) + + def test_get_ratelimit_override_for_user_callback(self) -> None: + test_user_id = "@user:test" + test_limiter_name = "name" + callbacks = RatelimitModuleApiCallbacks(self.hs) + requester = create_requester(test_user_id) + limiter = Ratelimiter( + store=self.hs.get_datastores().main, + clock=self.clock, + cfg=RatelimitSettings( + test_limiter_name, + per_second=0.1, + burst_count=3, + ), + ratelimit_callbacks=callbacks, + ) + + # Observe four actions, exceeding the burst_count. + limiter.record_action(requester=requester, n_actions=4, _time_now_s=0.0) + + # We should be prevented from taking a new action now. + success, _ = self.get_success_or_raise( + limiter.can_do_action(requester=requester, _time_now_s=0.0) + ) + self.assertFalse(success) + + # Now register a callback that overrides the ratelimit for this user + # and limiter name. + async def get_ratelimit_override_for_user( + user_id: str, limiter_name: str + ) -> Optional[RatelimitOverride]: + if user_id == test_user_id: + return RatelimitOverride( + messages_per_second=0.1, + burst_count=10, + ) + return None + + callbacks.register_callbacks( + get_ratelimit_override_for_user=get_ratelimit_override_for_user + ) + + success, _ = self.get_success_or_raise( + limiter.can_do_action(requester=requester, _time_now_s=0.0) + ) + self.assertTrue(success) From 586b82e5800d8a4811b44cef49298cfacacb2ba7 Mon Sep 17 00:00:00 2001 From: Mateusz Reszka Date: Wed, 4 Jun 2025 22:44:25 +0200 Subject: [PATCH 353/691] Propose `CAP_NET_BIND_SERVICE` instead running Synapse with root (#18408) There are alternative ways to use low numbered ports besides root. Users might be mislead into thinking they should run Synapse with root privileges. --- changelog.d/18408.doc | 1 + docs/reverse_proxy.md | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18408.doc diff --git a/changelog.d/18408.doc b/changelog.d/18408.doc new file mode 100644 index 0000000000..d88f429381 --- /dev/null +++ b/changelog.d/18408.doc @@ -0,0 +1 @@ +Mention `CAP_NET_BIND_SERVICE` as an alternative to running Synapse as root in order to bind to a privileged port. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 45de2b1f65..f871a39939 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -5,10 +5,10 @@ It is recommended to put a reverse proxy such as [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy), [HAProxy](https://www.haproxy.org/) or -[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. One advantage -of doing so is that it means that you can expose the default https port -(443) to Matrix clients without needing to run Synapse with root -privileges. +[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. +This has the advantage of being able to expose the default HTTPS port (443) to Matrix +clients without requiring Synapse to bind to a privileged port (port numbers less than +1024), avoiding the need for `CAP_NET_BIND_SERVICE` or running as root. You should configure your reverse proxy to forward requests to `/_matrix` or `/_synapse/client` to Synapse, and have it set the `X-Forwarded-For` and From 8010377a88499c63e5e5268fabdcb0db20d530e9 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 5 Jun 2025 11:49:09 +0100 Subject: [PATCH 354/691] Add support for MSC4155 Invite filtering (#18288) This implements https://github.com/matrix-org/matrix-spec-proposals/pull/4155, which adds support for a new account data type that blocks an invite based on some conditions in the event contents. --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18288.feature | 1 + .../conf/workers-shared-extra.yaml.j2 | 2 + scripts-dev/complement.sh | 1 + synapse/api/constants.py | 4 + synapse/api/errors.py | 3 + synapse/config/experimental.py | 3 + synapse/handlers/federation.py | 15 ++ synapse/handlers/room_member.py | 16 ++ synapse/handlers/sliding_sync/room_lists.py | 11 +- synapse/handlers/sync.py | 11 +- synapse/push/bulk_push_rule_evaluator.py | 13 +- synapse/rest/client/versions.py | 2 + .../storage/databases/main/account_data.py | 20 +++ .../storage/databases/main/user_directory.py | 1 + synapse/storage/invite_rule.py | 110 ++++++++++++ tests/handlers/test_room_member.py | 167 +++++++++++++++++- tests/storage/test_invite_rule.py | 167 ++++++++++++++++++ 17 files changed, 542 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18288.feature create mode 100644 synapse/storage/invite_rule.py create mode 100644 tests/storage/test_invite_rule.py diff --git a/changelog.d/18288.feature b/changelog.d/18288.feature new file mode 100644 index 0000000000..6856167875 --- /dev/null +++ b/changelog.d/18288.feature @@ -0,0 +1 @@ +Add support for [MSC4155](https://github.com/matrix-org/matrix-spec-proposals/pull/4155) Invite Filtering. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9ab8fedcae..48b44ddf90 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -127,6 +127,8 @@ experimental_features: msc3983_appservice_otk_claims: true # Proxy key queries to exclusive ASes msc3984_appservice_key_query: true + # Invite filtering + msc4155_enabled: true server_notices: system_mxid_localpart: _server diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 6be9177f11..08b500ecd6 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -229,6 +229,7 @@ test_packages=( ./tests/msc3902 ./tests/msc3967 ./tests/msc4140 + ./tests/msc4155 ) # Enable dirty runs, so tests will reuse the same container where possible. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 1d0de60b2d..e36461486b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -286,6 +286,10 @@ class AccountDataTypes: IGNORED_USER_LIST: Final = "m.ignored_user_list" TAG: Final = "m.tag" PUSH_RULES: Final = "m.push_rules" + # MSC4155: Invite filtering + MSC4155_INVITE_PERMISSION_CONFIG: Final = ( + "org.matrix.msc4155.invite_permission_config" + ) class HistoryVisibility: diff --git a/synapse/api/errors.py b/synapse/api/errors.py index edd2073384..3eb533f7d5 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -137,6 +137,9 @@ class Codes(str, Enum): PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" KEY_TOO_LARGE = "M_KEY_TOO_LARGE" + # Part of MSC4155 + INVITE_BLOCKED = "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED" + class CodeMessageException(RuntimeError): """An exception with integer code, a message string attributes and optional headers. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 2dc75a778e..259b2c70cb 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -566,3 +566,6 @@ class ExperimentalConfig(Config): "msc4263_limit_key_queries_to_users_who_share_rooms", False, ) + + # MSC4155: Invite filtering + self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b1640e3246..a6de3e824d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -78,6 +78,7 @@ from synapse.replication.http.federation import ( ReplicationStoreRoomOnOutlierMembershipRestServlet, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.invite_rule import InviteRule from synapse.types import JsonDict, StrCollection, get_domain_from_id from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer @@ -1089,6 +1090,20 @@ class FederationHandler: if event.state_key == self._server_notices_mxid: raise SynapseError(HTTPStatus.FORBIDDEN, "Cannot invite this user") + # check the invitee's configuration and apply rules + invite_config = await self.store.get_invite_config_for_user(event.state_key) + rule = invite_config.get_invite_rule(event.sender) + if rule == InviteRule.BLOCK: + logger.info( + f"Automatically rejecting invite from {event.sender} due to the invite filtering rules of {event.state_key}" + ) + raise SynapseError( + 403, + "You are not permitted to invite this user.", + errcode=Codes.INVITE_BLOCKED, + ) + # InviteRule.IGNORE is handled at the sync layer + # We retrieve the room member handler here as to not cause a cyclic dependency member_handler = self.hs.get_room_member_handler() # We don't rate limit based on room ID, as that should be done by diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b0e750c9c7..24ee5e89a6 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -53,6 +53,7 @@ from synapse.metrics import event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta +from synapse.storage.invite_rule import InviteRule from synapse.types import ( JsonDict, Requester, @@ -915,6 +916,21 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): additional_fields=block_invite_result[1], ) + # check the invitee's configuration and apply rules. Admins on the server can bypass. + if not is_requester_admin: + invite_config = await self.store.get_invite_config_for_user(target_id) + rule = invite_config.get_invite_rule(requester.user.to_string()) + if rule == InviteRule.BLOCK: + logger.info( + f"Automatically rejecting invite from {target_id} due to the the invite filtering rules of {requester.user}" + ) + raise SynapseError( + 403, + "You are not permitted to invite this user.", + errcode=Codes.INVITE_BLOCKED, + ) + # InviteRule.IGNORE is handled at the sync layer. + # An empty prev_events list is allowed as long as the auth_event_ids are present if prev_event_ids is not None: return await self._local_membership_update( diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 6d1ac91605..13e69f18a0 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -49,6 +49,7 @@ from synapse.storage.databases.main.state import ( Sentinel as StateSentinel, ) from synapse.storage.databases.main.stream import CurrentStateDeltaMembership +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ( RoomsForUser, RoomsForUserSlidingSync, @@ -278,6 +279,7 @@ class SlidingSyncRoomLists: # Remove invites from ignored users ignored_users = await self.store.ignored_users(user_id) + invite_config = await self.store.get_invite_config_for_user(user_id) if ignored_users: # FIXME: It would be nice to avoid this copy but since # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it @@ -292,7 +294,14 @@ class SlidingSyncRoomLists: room_for_user_sliding_sync = room_membership_for_user_map[room_id] if ( room_for_user_sliding_sync.membership == Membership.INVITE - and room_for_user_sliding_sync.sender in ignored_users + and room_for_user_sliding_sync.sender + and ( + room_for_user_sliding_sync.sender in ignored_users + or invite_config.get_invite_rule( + room_for_user_sliding_sync.sender + ) + == InviteRule.IGNORE + ) ): room_membership_for_user_map.pop(room_id, None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 350c3fa09a..c6f2c38d8d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -66,6 +66,7 @@ from synapse.logging.opentracing import ( from synapse.storage.databases.main.event_push_actions import RoomNotifCounts from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary from synapse.storage.databases.main.stream import PaginateFunction +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import MemberSummary from synapse.types import ( DeviceListUpdates, @@ -2549,6 +2550,7 @@ class SyncHandler: room_entries: List[RoomSyncResultBuilder] = [] invited: List[InvitedSyncResult] = [] knocked: List[KnockedSyncResult] = [] + invite_config = await self.store.get_invite_config_for_user(user_id) for room_id, events in mem_change_events_by_room_id.items(): # The body of this loop will add this room to at least one of the five lists # above. Things get messy if you've e.g. joined, left, joined then left the @@ -2631,7 +2633,11 @@ class SyncHandler: # Only bother if we're still currently invited should_invite = last_non_join.membership == Membership.INVITE if should_invite: - if last_non_join.sender not in ignored_users: + if ( + last_non_join.sender not in ignored_users + and invite_config.get_invite_rule(last_non_join.sender) + != InviteRule.IGNORE + ): invite_room_sync = InvitedSyncResult(room_id, invite=last_non_join) if invite_room_sync: invited.append(invite_room_sync) @@ -2786,6 +2792,7 @@ class SyncHandler: membership_list=Membership.LIST, excluded_rooms=sync_result_builder.excluded_room_ids, ) + invite_config = await self.store.get_invite_config_for_user(user_id) room_entries = [] invited = [] @@ -2811,6 +2818,8 @@ class SyncHandler: elif event.membership == Membership.INVITE: if event.sender in ignored_users: continue + if invite_config.get_invite_rule(event.sender) == InviteRule.IGNORE: + continue invite = await self.store.get_event(event.event_id) invited.append(InvitedSyncResult(room_id=event.room_id, invite=invite)) elif event.membership == Membership.KNOCK: diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 8c106f9649..8249d5e84f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -52,6 +52,7 @@ from synapse.events.snapshot import EventContext from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.state import POWER_KEY from synapse.storage.databases.main.roommember import EventIdMembership +from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator from synapse.types import JsonValue @@ -191,9 +192,17 @@ class BulkPushRuleEvaluator: # if this event is an invite event, we may need to run rules for the user # who's been invited, otherwise they won't get told they've been invited - if event.type == EventTypes.Member and event.membership == Membership.INVITE: + if ( + event.is_state() + and event.type == EventTypes.Member + and event.membership == Membership.INVITE + ): invited = event.state_key - if invited and self.hs.is_mine_id(invited) and invited not in local_users: + invite_config = await self.store.get_invite_config_for_user(invited) + if invite_config.get_invite_rule(event.sender) != InviteRule.ALLOW: + # Invite was blocked or ignored, never notify. + return {} + if self.hs.is_mine_id(invited) and invited not in local_users: local_users.append(invited) if not local_users: diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 266a0b835b..f58f11e5cc 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -174,6 +174,8 @@ class VersionsRestServlet(RestServlet): "org.matrix.simplified_msc3575": msc3575_enabled, # Arbitrary key-value profile fields. "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, + # MSC4155: Invite filtering + "org.matrix.msc4155": self.config.experimental.msc4155_enabled, }, }, ) diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index e583c182ba..d26de1ad16 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -43,6 +43,7 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.push_rule import PushRulesWorkerStore +from synapse.storage.invite_rule import InviteRulesConfig from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonMapping from synapse.util import json_encoder @@ -102,6 +103,8 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) self._delete_account_data_for_deactivated_users, ) + self._msc4155_enabled = hs.config.experimental.msc4155_enabled + def get_max_account_data_stream_id(self) -> int: """Get the current max stream ID for account data stream @@ -557,6 +560,23 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) ) ) + async def get_invite_config_for_user(self, user_id: str) -> InviteRulesConfig: + """ + Get the invite configuration for the current user. + + Args: + user_id: + """ + + if not self._msc4155_enabled: + # This equates to allowing all invites, as if the setting was off. + return InviteRulesConfig(None) + + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG + ) + return InviteRulesConfig(data) + def process_replication_rows( self, stream_name: str, diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 2b867cdb6e..31a8ce6666 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -43,6 +43,7 @@ try: USE_ICU = True except ModuleNotFoundError: + # except ModuleNotFoundError: USE_ICU = False from synapse.api.errors import StoreError diff --git a/synapse/storage/invite_rule.py b/synapse/storage/invite_rule.py new file mode 100644 index 0000000000..b9d9d1eb62 --- /dev/null +++ b/synapse/storage/invite_rule.py @@ -0,0 +1,110 @@ +import logging +from enum import Enum +from typing import Optional, Pattern + +from matrix_common.regex import glob_to_regex + +from synapse.types import JsonMapping, UserID + +logger = logging.getLogger(__name__) + + +class InviteRule(Enum): + """Enum to define the action taken when an invite matches a rule.""" + + ALLOW = "allow" + BLOCK = "block" + IGNORE = "ignore" + + +class InviteRulesConfig: + """Class to determine if a given user permits an invite from another user, and the action to take.""" + + def __init__(self, account_data: Optional[JsonMapping]): + self.allowed_users: list[Pattern[str]] = [] + self.ignored_users: list[Pattern[str]] = [] + self.blocked_users: list[Pattern[str]] = [] + + self.allowed_servers: list[Pattern[str]] = [] + self.ignored_servers: list[Pattern[str]] = [] + self.blocked_servers: list[Pattern[str]] = [] + + def process_field( + values: Optional[list[str]], + ruleset: list[Pattern[str]], + rule: InviteRule, + ) -> None: + if isinstance(values, list): + for value in values: + if isinstance(value, str) and len(value) > 0: + # User IDs cannot exceed 255 bytes. Don't process large, potentially + # expensive glob patterns. + if len(value) > 255: + logger.debug( + "Ignoring invite config glob pattern that is >255 bytes: {value}" + ) + continue + + try: + ruleset.append(glob_to_regex(value)) + except Exception as e: + # If for whatever reason we can't process this, just ignore it. + logger.debug( + f"Could not process '{value}' field of invite rule config, ignoring: {e}" + ) + + if account_data: + process_field( + account_data.get("allowed_users"), self.allowed_users, InviteRule.ALLOW + ) + process_field( + account_data.get("ignored_users"), self.ignored_users, InviteRule.IGNORE + ) + process_field( + account_data.get("blocked_users"), self.blocked_users, InviteRule.BLOCK + ) + process_field( + account_data.get("allowed_servers"), + self.allowed_servers, + InviteRule.ALLOW, + ) + process_field( + account_data.get("ignored_servers"), + self.ignored_servers, + InviteRule.IGNORE, + ) + process_field( + account_data.get("blocked_servers"), + self.blocked_servers, + InviteRule.BLOCK, + ) + + def get_invite_rule(self, user_id: str) -> InviteRule: + """Get the invite rule that matches this user. Will return InviteRule.ALLOW if no rules match + + Args: + user_id: The user ID of the inviting user. + + """ + user = UserID.from_string(user_id) + # The order here is important. We always process user rules before server rules + # and we always process in the order of Allow, Ignore, Block. + for patterns, rule in [ + (self.allowed_users, InviteRule.ALLOW), + (self.ignored_users, InviteRule.IGNORE), + (self.blocked_users, InviteRule.BLOCK), + ]: + for regex in patterns: + if regex.match(user_id): + return rule + + for patterns, rule in [ + (self.allowed_servers, InviteRule.ALLOW), + (self.ignored_servers, InviteRule.IGNORE), + (self.blocked_servers, InviteRule.BLOCK), + ]: + for regex in patterns: + if regex.match(user.domain): + return rule + + return InviteRule.ALLOW diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index f43ce66483..d87fe9d62c 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -5,10 +5,13 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin import synapse.rest.client.login import synapse.rest.client.room -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 +from synapse.federation.federation_base import ( + event_from_pdu_json, +) from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -453,3 +456,165 @@ class RoomMemberMasterHandlerTestCase(HomeserverTestCase): new_count = rows[0][0] self.assertEqual(initial_count, new_count) + + +class TestInviteFiltering(FederatingHomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + synapse.rest.client.login.register_servlets, + synapse.rest.client.room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.handler = hs.get_room_member_handler() + self.fed_handler = hs.get_federation_handler() + self.store = hs.get_datastores().main + + # Create three users. + self.alice = self.register_user("alice", "pass") + self.alice_token = self.login("alice", "pass") + self.bob = self.register_user("bob", "pass") + self.bob_token = self.login("bob", "pass") + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_misc4155_block_invite_local(self) -> None: + """Test that MSC4155 will block a user from being invited to a room""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + { + "blocked_users": [self.alice], + }, + ) + ) + + f = self.get_failure( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + @override_config({"experimental_features": {"msc4155_enabled": False}}) + def test_msc4155_disabled_allow_invite_local(self) -> None: + """Test that MSC4155 will block a user from being invited to a room""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + { + "blocked_users": [self.alice], + }, + ) + ) + + self.get_success( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + ) + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_msc4155_block_invite_remote(self) -> None: + """Test that MSC4155 will block a remote user from being invited to a room""" + # A remote user who sends the invite + remote_server = "otherserver" + remote_user = "@otheruser:" + remote_server + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + {"blocked_users": [remote_user]}, + ) + ) + + room_id = self.helper.create_room_as( + room_creator=self.alice, tok=self.alice_token + ) + room_version = self.get_success(self.store.get_room_version(room_id)) + + invite_event = event_from_pdu_json( + { + "type": EventTypes.Member, + "content": {"membership": "invite"}, + "room_id": room_id, + "sender": remote_user, + "state_key": self.bob, + "depth": 32, + "prev_events": [], + "auth_events": [], + "origin_server_ts": self.clock.time_msec(), + }, + room_version, + ) + + f = self.get_failure( + self.fed_handler.on_invite_request( + remote_server, + invite_event, + invite_event.room_version, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + @override_config({"experimental_features": {"msc4155_enabled": True}}) + def test_msc4155_block_invite_remote_server(self) -> None: + """Test that MSC4155 will block a remote server's user from being invited to a room""" + # A remote user who sends the invite + remote_server = "otherserver" + remote_user = "@otheruser:" + remote_server + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG, + {"blocked_servers": [remote_server]}, + ) + ) + + room_id = self.helper.create_room_as( + room_creator=self.alice, tok=self.alice_token + ) + room_version = self.get_success(self.store.get_room_version(room_id)) + + invite_event = event_from_pdu_json( + { + "type": EventTypes.Member, + "content": {"membership": "invite"}, + "room_id": room_id, + "sender": remote_user, + "state_key": self.bob, + "depth": 32, + "prev_events": [], + "auth_events": [], + "origin_server_ts": self.clock.time_msec(), + }, + room_version, + ) + + f = self.get_failure( + self.fed_handler.on_invite_request( + remote_server, + invite_event, + invite_event.room_version, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") diff --git a/tests/storage/test_invite_rule.py b/tests/storage/test_invite_rule.py new file mode 100644 index 0000000000..38c97ecaa3 --- /dev/null +++ b/tests/storage/test_invite_rule.py @@ -0,0 +1,167 @@ +from synapse.storage.invite_rule import InviteRule, InviteRulesConfig +from synapse.types import UserID + +from tests import unittest + +regular_user = UserID.from_string("@test:example.org") +allowed_user = UserID.from_string("@allowed:allow.example.org") +blocked_user = UserID.from_string("@blocked:block.example.org") +ignored_user = UserID.from_string("@ignored:ignore.example.org") + + +class InviteFilterTestCase(unittest.TestCase): + def test_empty(self) -> None: + """Permit by default""" + config = InviteRulesConfig(None) + self.assertEqual( + config.get_invite_rule(regular_user.to_string()), InviteRule.ALLOW + ) + + def test_ignore_invalid(self) -> None: + """Invalid strings are ignored""" + config = InviteRulesConfig({"blocked_users": ["not a user"]}) + self.assertEqual( + config.get_invite_rule(blocked_user.to_string()), InviteRule.ALLOW + ) + + def test_user_blocked(self) -> None: + """Permit all, except explicitly blocked users""" + config = InviteRulesConfig({"blocked_users": [blocked_user.to_string()]}) + self.assertEqual( + config.get_invite_rule(blocked_user.to_string()), InviteRule.BLOCK + ) + self.assertEqual( + config.get_invite_rule(regular_user.to_string()), InviteRule.ALLOW + ) + + def test_user_ignored(self) -> None: + """Permit all, except explicitly ignored users""" + config = InviteRulesConfig({"ignored_users": [ignored_user.to_string()]}) + self.assertEqual( + config.get_invite_rule(ignored_user.to_string()), InviteRule.IGNORE + ) + self.assertEqual( + config.get_invite_rule(regular_user.to_string()), InviteRule.ALLOW + ) + + def test_user_precedence(self) -> None: + """Always take allowed over ignored, ignored over blocked, and then block.""" + config = InviteRulesConfig( + { + "allowed_users": [allowed_user.to_string()], + "ignored_users": [allowed_user.to_string(), ignored_user.to_string()], + "blocked_users": [ + allowed_user.to_string(), + ignored_user.to_string(), + blocked_user.to_string(), + ], + } + ) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.ALLOW + ) + self.assertEqual( + config.get_invite_rule(ignored_user.to_string()), InviteRule.IGNORE + ) + self.assertEqual( + config.get_invite_rule(blocked_user.to_string()), InviteRule.BLOCK + ) + + def test_server_blocked(self) -> None: + """Block all users on the server except those allowed.""" + user_on_same_server = UserID("blocked", allowed_user.domain) + config = InviteRulesConfig( + { + "allowed_users": [allowed_user.to_string()], + "blocked_servers": [allowed_user.domain], + } + ) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.ALLOW + ) + self.assertEqual( + config.get_invite_rule(user_on_same_server.to_string()), InviteRule.BLOCK + ) + + def test_server_ignored(self) -> None: + """Ignore all users on the server except those allowed.""" + user_on_same_server = UserID("ignored", allowed_user.domain) + config = InviteRulesConfig( + { + "allowed_users": [allowed_user.to_string()], + "ignored_servers": [allowed_user.domain], + } + ) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.ALLOW + ) + self.assertEqual( + config.get_invite_rule(user_on_same_server.to_string()), InviteRule.IGNORE + ) + + def test_server_allow(self) -> None: + """Allow all from a server except explictly blocked or ignored users.""" + blocked_user_on_same_server = UserID("blocked", allowed_user.domain) + ignored_user_on_same_server = UserID("ignored", allowed_user.domain) + allowed_user_on_same_server = UserID("another", allowed_user.domain) + config = InviteRulesConfig( + { + "ignored_users": [ignored_user_on_same_server.to_string()], + "blocked_users": [blocked_user_on_same_server.to_string()], + "allowed_servers": [allowed_user.to_string()], + } + ) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.ALLOW + ) + self.assertEqual( + config.get_invite_rule(allowed_user_on_same_server.to_string()), + InviteRule.ALLOW, + ) + self.assertEqual( + config.get_invite_rule(blocked_user_on_same_server.to_string()), + InviteRule.BLOCK, + ) + self.assertEqual( + config.get_invite_rule(ignored_user_on_same_server.to_string()), + InviteRule.IGNORE, + ) + + def test_server_precedence(self) -> None: + """Always take allowed over ignored, ignored over blocked, and then block.""" + config = InviteRulesConfig( + { + "allowed_servers": [allowed_user.domain], + "ignored_servers": [allowed_user.domain, ignored_user.domain], + "blocked_servers": [ + allowed_user.domain, + ignored_user.domain, + blocked_user.domain, + ], + } + ) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.ALLOW + ) + self.assertEqual( + config.get_invite_rule(ignored_user.to_string()), InviteRule.IGNORE + ) + self.assertEqual( + config.get_invite_rule(blocked_user.to_string()), InviteRule.BLOCK + ) + + def test_server_glob(self) -> None: + """Test that glob patterns match""" + config = InviteRulesConfig({"blocked_servers": ["*.example.org"]}) + self.assertEqual( + config.get_invite_rule(allowed_user.to_string()), InviteRule.BLOCK + ) + self.assertEqual( + config.get_invite_rule(ignored_user.to_string()), InviteRule.BLOCK + ) + self.assertEqual( + config.get_invite_rule(blocked_user.to_string()), InviteRule.BLOCK + ) + self.assertEqual( + config.get_invite_rule(regular_user.to_string()), InviteRule.ALLOW + ) From 0b9f1757a74e627c31ac5937735979601079cb8e Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 5 Jun 2025 11:14:02 +0000 Subject: [PATCH 355/691] Reduce disk wastage by cleaning up `received_transactions` older than 1 day, rather than 30 days. (#18310) Clean up `received_transactions` older than 1 day, rather than 30 days \ Reduces disk waste by homeservers Closes #6437 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18310.misc | 1 + synapse/storage/databases/main/transactions.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18310.misc diff --git a/changelog.d/18310.misc b/changelog.d/18310.misc new file mode 100644 index 0000000000..1837b339f4 --- /dev/null +++ b/changelog.d/18310.misc @@ -0,0 +1 @@ +Reduce disk wastage by cleaning up `received_transactions` older than 1 day, rather than 30 days. \ No newline at end of file diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 770802483c..bfc324b80d 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -86,10 +86,10 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: now = self._clock.time_msec() - month_ago = now - 30 * 24 * 60 * 60 * 1000 + day_ago = now - 24 * 60 * 60 * 1000 def _cleanup_transactions_txn(txn: LoggingTransaction) -> None: - txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,)) + txn.execute("DELETE FROM received_transactions WHERE ts < ?", (day_ago,)) await self.db_pool.runInteraction( "_cleanup_transactions", _cleanup_transactions_txn From 865d43b4b3af0be8ea06e87a8fd7b1b4cb1e764a Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:02:40 +0200 Subject: [PATCH 356/691] docs: render missing docs for scheduled tasks admin api (#18516) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18516.doc | 1 + docs/SUMMARY.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/18516.doc diff --git a/changelog.d/18516.doc b/changelog.d/18516.doc new file mode 100644 index 0000000000..c739d32f5e --- /dev/null +++ b/changelog.d/18516.doc @@ -0,0 +1 @@ +Surface hidden Admin API documentation regarding fetching of scheduled tasks. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 40d70a4485..33289ccaad 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -68,6 +68,7 @@ - [Registration Tokens](usage/administration/admin_api/registration_tokens.md) - [Manipulate Room Membership](admin_api/room_membership.md) - [Rooms](admin_api/rooms.md) + - [Scheduled tasks](admin_api/scheduled_tasks.md) - [Server Notices](admin_api/server_notices.md) - [Statistics](admin_api/statistics.md) - [Users](admin_api/user_admin_api.md) From e80bc4b0621074a395741d1ccb2955c811f1dd7a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 5 Jun 2025 15:30:28 -0500 Subject: [PATCH 357/691] Distinguish all vs local events being persisted in the "Event Send Time Quantiles" graph (#18510) (Applies to the Grafana graphs) As discovered by @devonh, we use `synapse_storage_events_persisted_events_total` (which tracks *all* persisted events) for the "Events" rate in the "Event Send Time Quantiles" graph. This is pretty misleading as I would expect it to be the rate of events being sent given the graph title, "Event Send Time Quantiles". Since the event persistence queues are shared for local and remote events from federation and will block local events being sent, I think it does still make sense to have the event persist rate. I've updated the graph to include the rate of "Local events being persisted" and the rate of "All events being persisted". I think this properly disambiguates and clarifies what the graph is trying to show. --- changelog.d/18510.misc | 1 + contrib/grafana/synapse.json | 96 ++++++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 changelog.d/18510.misc diff --git a/changelog.d/18510.misc b/changelog.d/18510.misc new file mode 100644 index 0000000000..71a3549d31 --- /dev/null +++ b/changelog.d/18510.misc @@ -0,0 +1 @@ +Distinguish all vs local events being persisted in the "Event Send Time Quantiles" graph (Grafana). diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 30d6d87500..62b58a199d 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -220,29 +220,24 @@ "yBucketBound": "auto" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_PROMETHEUS}", + "type": "prometheus" }, - "description": "", + "aliasColors": {}, + "dashLength": 10, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, - "hiddenSeries": false, "id": 152, "legend": { "avg": false, @@ -255,71 +250,81 @@ "values": false }, "lines": true, - "linewidth": 0, - "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", + "pluginVersion": "10.4.3", "pointradius": 5, - "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Avg", "fill": 0, - "linewidth": 3 + "linewidth": 3, + "$$hashKey": "object:48" }, { "alias": "99%", "color": "#C4162A", - "fillBelowTo": "90%" + "fillBelowTo": "90%", + "$$hashKey": "object:49" }, { "alias": "90%", "color": "#FF7383", - "fillBelowTo": "75%" + "fillBelowTo": "75%", + "$$hashKey": "object:50" }, { "alias": "75%", "color": "#FFEE52", - "fillBelowTo": "50%" + "fillBelowTo": "50%", + "$$hashKey": "object:51" }, { "alias": "50%", "color": "#73BF69", - "fillBelowTo": "25%" + "fillBelowTo": "25%", + "$$hashKey": "object:52" }, { "alias": "25%", "color": "#1F60C4", - "fillBelowTo": "5%" + "fillBelowTo": "5%", + "$$hashKey": "object:53" }, { "alias": "5%", - "lines": false + "lines": false, + "$$hashKey": "object:54" }, { "alias": "Average", "color": "rgb(255, 255, 255)", "lines": true, - "linewidth": 3 + "linewidth": 3, + "$$hashKey": "object:55" }, { - "alias": "Events", + "alias": "Local events being persisted", + "color": "#96d98D", + "points": true, + "yaxis": 2, + "zindex": -3, + "$$hashKey": "object:56" + }, + { + "$$hashKey": "object:329", "color": "#B877D9", - "hideTooltip": true, + "alias": "All events being persisted", "points": true, "yaxis": 2, "zindex": -3 } ], "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { @@ -384,7 +389,20 @@ }, "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", "legendFormat": "Average", - "refId": "H" + "refId": "H", + "editorMode": "code", + "range": true + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", + "hide": false, + "instant": false, + "legendFormat": "Local events being persisted", + "refId": "E", + "editorMode": "code" }, { "datasource": { @@ -393,8 +411,9 @@ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", "hide": false, "instant": false, - "legendFormat": "Events", - "refId": "E" + "legendFormat": "All events being persisted", + "refId": "I", + "editorMode": "code" } ], "thresholds": [ @@ -428,7 +447,9 @@ "xaxis": { "mode": "time", "show": true, - "values": [] + "values": [], + "name": null, + "buckets": null }, "yaxes": [ { @@ -450,7 +471,20 @@ ], "yaxis": { "align": false - } + }, + "bars": false, + "dashes": false, + "description": "", + "fill": 0, + "fillGradient": 0, + "hiddenSeries": false, + "linewidth": 0, + "percentage": false, + "points": false, + "stack": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null }, { "aliasColors": {}, From 82189cbde45225d6c8793018343f265894ac8e23 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 6 Jun 2025 11:48:49 +0100 Subject: [PATCH 358/691] Export RatelimitOverride from ModuleApi (#18513) --- changelog.d/18513.bugfix | 1 + docs/modules/ratelimit_callbacks.md | 7 ++++++- synapse/api/ratelimiting.py | 2 +- synapse/module_api/__init__.py | 2 ++ .../module_api/callbacks/ratelimit_callbacks.py | 14 +++++++++++++- synapse/storage/databases/main/room.py | 4 +++- tests/api/test_ratelimiting.py | 4 ++-- 7 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18513.bugfix diff --git a/changelog.d/18513.bugfix b/changelog.d/18513.bugfix new file mode 100644 index 0000000000..b44eaafd15 --- /dev/null +++ b/changelog.d/18513.bugfix @@ -0,0 +1 @@ +Support the import of the `RatelimitOverride` type from `synapse.module_api` in modules and rename `messages_per_second` to `per_second`. \ No newline at end of file diff --git a/docs/modules/ratelimit_callbacks.md b/docs/modules/ratelimit_callbacks.md index bf923c045e..3386220e37 100644 --- a/docs/modules/ratelimit_callbacks.md +++ b/docs/modules/ratelimit_callbacks.md @@ -11,7 +11,7 @@ The available ratelimit callbacks are: _First introduced in Synapse v1.132.0_ ```python -async def get_ratelimit_override_for_user(user: str, limiter_name: str) -> Optional[RatelimitOverride] +async def get_ratelimit_override_for_user(user: str, limiter_name: str) -> Optional[synapse.module_api.RatelimitOverride] ``` Called when constructing a ratelimiter of a particular type for a user. The module can @@ -26,6 +26,11 @@ The limiters that are currently supported are: - `rc_invites.per_user` - `rc_invites.per_issuer` +The `RatelimitOverride` return type has the following fields: + +- `per_second: float`. The number of actions that can be performed in a second. `0.0` means that ratelimiting is disabled. +- `burst_count: int`. The number of actions that can be performed before being limited. + If multiple modules implement this callback, they will be considered in order. If a callback returns `None`, Synapse falls through to the next one. The value of the first callback that does not return `None` will be used. If this happens, Synapse will not call diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 38e5bdaa75..4f3bf8f770 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -184,7 +184,7 @@ class Ratelimiter: ) if module_override: - rate_hz = module_override.messages_per_second + rate_hz = module_override.per_second burst_count = module_override.burst_count # Override default values if set diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index e36503a526..656fe323f3 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -96,6 +96,7 @@ from synapse.module_api.callbacks.media_repository_callbacks import ( ) from synapse.module_api.callbacks.ratelimit_callbacks import ( GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK, + RatelimitOverride, ) from synapse.module_api.callbacks.spamchecker_callbacks import ( CHECK_EVENT_FOR_SPAM_CALLBACK, @@ -197,6 +198,7 @@ __all__ = [ "ProfileInfo", "RoomAlias", "UserProfile", + "RatelimitOverride", ] logger = logging.getLogger(__name__) diff --git a/synapse/module_api/callbacks/ratelimit_callbacks.py b/synapse/module_api/callbacks/ratelimit_callbacks.py index ced721b407..64f9cc81e8 100644 --- a/synapse/module_api/callbacks/ratelimit_callbacks.py +++ b/synapse/module_api/callbacks/ratelimit_callbacks.py @@ -15,7 +15,8 @@ import logging from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional -from synapse.storage.databases.main.room import RatelimitOverride +import attr + from synapse.util.async_helpers import delay_cancellation from synapse.util.metrics import Measure @@ -24,6 +25,17 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) + +@attr.s(auto_attribs=True) +class RatelimitOverride: + """Represents a ratelimit being overridden.""" + + per_second: float + """The number of actions that can be performed in a second. `0.0` means that ratelimiting is disabled.""" + burst_count: int + """How many actions that can be performed before being limited.""" + + GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK = Callable[ [str, str], Awaitable[Optional[RatelimitOverride]] ] diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 46491cc0fe..1df06a5171 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -77,7 +77,9 @@ logger = logging.getLogger(__name__) @attr.s(slots=True, frozen=True, auto_attribs=True) class RatelimitOverride: - messages_per_second: float + # n.b. elsewhere in Synapse messages_per_second is represented as a float, but it is + # an integer in the database + messages_per_second: int burst_count: int diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 5e73f5d5ec..93f4f98916 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -3,8 +3,8 @@ from typing import Optional from synapse.api.ratelimiting import LimitExceededError, Ratelimiter from synapse.appservice import ApplicationService from synapse.config.ratelimiting import RatelimitSettings +from synapse.module_api import RatelimitOverride from synapse.module_api.callbacks.ratelimit_callbacks import RatelimitModuleApiCallbacks -from synapse.storage.databases.main.room import RatelimitOverride from synapse.types import create_requester from tests import unittest @@ -477,7 +477,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): ) -> Optional[RatelimitOverride]: if user_id == test_user_id: return RatelimitOverride( - messages_per_second=0.1, + per_second=0.1, burst_count=10, ) return None From 919c3624668e6e4f1ae11708a04864a542816200 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 6 Jun 2025 11:19:58 +0000 Subject: [PATCH 359/691] Remove destinations from sending if not whitelisted (#18484) Co-authored-by: Andrew Morgan --- changelog.d/18484.bugfix | 1 + synapse/config/federation.py | 16 +++++++ synapse/federation/sender/__init__.py | 62 +++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18484.bugfix diff --git a/changelog.d/18484.bugfix b/changelog.d/18484.bugfix new file mode 100644 index 0000000000..c816410da4 --- /dev/null +++ b/changelog.d/18484.bugfix @@ -0,0 +1 @@ +Remove destinations from sending if not whitelisted. diff --git a/synapse/config/federation.py b/synapse/config/federation.py index cf29fa2562..31f46e420d 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -94,5 +94,21 @@ class FederationConfig(Config): 2**62, ) + def is_domain_allowed_according_to_federation_whitelist(self, domain: str) -> bool: + """ + Returns whether a domain is allowed according to the federation whitelist. If a + federation whitelist is not set, all domains are allowed. + + Args: + domain: The domain to test. + + Returns: + True if the domain is allowed or if a whitelist is not set, False otherwise. + """ + if self.federation_domain_whitelist is None: + return True + + return domain in self.federation_domain_whitelist + _METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}} diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index b95b3c629d..2eef7b707d 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -342,6 +342,8 @@ class _DestinationWakeupQueue: destination, _ = self.queue.popitem(last=False) queue = self.sender._get_per_destination_queue(destination) + if queue is None: + continue if not queue._new_data_to_send: # The per destination queue has already been woken up. @@ -436,12 +438,23 @@ class FederationSender(AbstractFederationSender): self._wake_destinations_needing_catchup, ) - def _get_per_destination_queue(self, destination: str) -> PerDestinationQueue: + def _get_per_destination_queue( + self, destination: str + ) -> Optional[PerDestinationQueue]: """Get or create a PerDestinationQueue for the given destination Args: destination: server_name of remote server + + Returns: + None if the destination is not allowed by the federation whitelist. + Otherwise a PerDestinationQueue for this destination. """ + if not self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + destination + ): + return None + queue = self._per_destination_queues.get(destination) if not queue: queue = PerDestinationQueue(self.hs, self._transaction_manager, destination) @@ -718,6 +731,16 @@ class FederationSender(AbstractFederationSender): # track the fact that we have a PDU for these destinations, # to allow us to perform catch-up later on if the remote is unreachable # for a while. + # Filter out any destinations not present in the federation_domain_whitelist, if + # the whitelist exists. These destinations should not be sent to so let's not + # waste time or space keeping track of events destined for them. + destinations = [ + d + for d in destinations + if self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + d + ) + ] await self.store.store_destination_rooms_entries( destinations, pdu.room_id, @@ -732,7 +755,12 @@ class FederationSender(AbstractFederationSender): ) for destination in destinations: - self._get_per_destination_queue(destination).send_pdu(pdu) + queue = self._get_per_destination_queue(destination) + # We expect `queue` to not be None as we already filtered out + # non-whitelisted destinations above. + assert queue is not None + + queue.send_pdu(pdu) async def send_read_receipt(self, receipt: ReadReceipt) -> None: """Send a RR to any other servers in the room @@ -841,12 +869,16 @@ class FederationSender(AbstractFederationSender): for domain in immediate_domains: # Add to destination queue and wake the destination up queue = self._get_per_destination_queue(domain) + if queue is None: + continue queue.queue_read_receipt(receipt) queue.attempt_new_transaction() for domain in delay_domains: # Add to destination queue... queue = self._get_per_destination_queue(domain) + if queue is None: + continue queue.queue_read_receipt(receipt) # ... and schedule the destination to be woken up. @@ -882,9 +914,10 @@ class FederationSender(AbstractFederationSender): if self.is_mine_server_name(destination): continue - self._get_per_destination_queue(destination).send_presence( - states, start_loop=False - ) + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.send_presence(states, start_loop=False) self._destination_wakeup_queue.add_to_queue(destination) @@ -934,6 +967,8 @@ class FederationSender(AbstractFederationSender): return queue = self._get_per_destination_queue(edu.destination) + if queue is None: + return if key: queue.send_keyed_edu(edu, key) else: @@ -958,9 +993,15 @@ class FederationSender(AbstractFederationSender): for destination in destinations: if immediate: - self._get_per_destination_queue(destination).attempt_new_transaction() + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.attempt_new_transaction() else: - self._get_per_destination_queue(destination).mark_new_data() + queue = self._get_per_destination_queue(destination) + if queue is None: + continue + queue.mark_new_data() self._destination_wakeup_queue.add_to_queue(destination) def wake_destination(self, destination: str) -> None: @@ -979,7 +1020,9 @@ class FederationSender(AbstractFederationSender): ): return - self._get_per_destination_queue(destination).attempt_new_transaction() + queue = self._get_per_destination_queue(destination) + if queue is not None: + queue.attempt_new_transaction() @staticmethod def get_current_token() -> int: @@ -1024,6 +1067,9 @@ class FederationSender(AbstractFederationSender): d for d in destinations_to_wake if self._federation_shard_config.should_handle(self._instance_name, d) + and self.hs.config.federation.is_domain_allowed_according_to_federation_whitelist( + d + ) ] for destination in destinations_to_wake: From d285d76185216914baab6ad0bef63b5c95a87e5f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jun 2025 12:59:38 +0100 Subject: [PATCH 360/691] Recover an appservice if a successful ping occurs. (#18521) Fixes https://github.com/element-hq/synapse/issues/14240 This scratches an itch that i've had for years. We regularly run into the issue where (especially in development) appservices can go down for a period and them come back up. The ping endpoint was introduced some time ago which means Synapse can determine if an AS is up more or less immediately, so we might as well use that to schedule transaction redelivery. I believe transaction scheduling logic is largely implementation specific, so we should be in the clear to do this without any spec changes. --- changelog.d/18521.feature | 1 + synapse/appservice/scheduler.py | 36 +++++++++++++++++++++++-- synapse/rest/client/appservice_ping.py | 7 ++++- tests/appservice/test_scheduler.py | 37 +++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18521.feature diff --git a/changelog.d/18521.feature b/changelog.d/18521.feature new file mode 100644 index 0000000000..29da172e91 --- /dev/null +++ b/changelog.d/18521.feature @@ -0,0 +1 @@ +Successful requests to `/_matrix/app/v1/ping` will now force Synapse to reattempt delivering transactions to appservices. diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 7994da0868..cba08dde85 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2015, 2016 OpenMarket Ltd -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -70,6 +70,8 @@ from typing import ( Tuple, ) +from twisted.internet.interfaces import IDelayedCall + from synapse.appservice import ( ApplicationService, ApplicationServiceState, @@ -450,6 +452,20 @@ class _TransactionController: recoverer.recover() logger.info("Now %i active recoverers", len(self.recoverers)) + def force_retry(self, service: ApplicationService) -> None: + """Forces a Recoverer to attempt delivery of transations immediately. + + Args: + service: + """ + recoverer = self.recoverers.get(service.id) + if not recoverer: + # No need to force a retry on a happy AS. + logger.info(f"{service.id} is not in recovery, not forcing retry") + return + + recoverer.force_retry() + async def _is_service_up(self, service: ApplicationService) -> bool: state = await self.store.get_appservice_state(service) return state == ApplicationServiceState.UP or state is None @@ -482,11 +498,12 @@ class _Recoverer: self.service = service self.callback = callback self.backoff_counter = 1 + self.scheduled_recovery: Optional[IDelayedCall] = None def recover(self) -> None: delay = 2**self.backoff_counter logger.info("Scheduling retries on %s in %fs", self.service.id, delay) - self.clock.call_later( + self.scheduled_recovery = self.clock.call_later( delay, run_as_background_process, "as-recoverer", self.retry ) @@ -496,6 +513,21 @@ class _Recoverer: self.backoff_counter += 1 self.recover() + def force_retry(self) -> None: + """Cancels the existing timer and forces an immediate retry in the background. + + Args: + service: + """ + # Prevent the existing backoff from occuring + if self.scheduled_recovery: + self.clock.cancel_call_later(self.scheduled_recovery) + # Run a retry, which will resechedule a recovery if it fails. + run_as_background_process( + "retry", + self.retry, + ) + async def retry(self) -> None: logger.info("Starting retries on %s", self.service.id) try: diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index d6b4e32453..1f9662a95a 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2023 Tulir Asokan -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -53,6 +53,7 @@ class AppservicePingRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.as_api = hs.get_application_service_api() + self.scheduler = hs.get_application_service_scheduler() self.auth = hs.get_auth() async def on_POST( @@ -85,6 +86,10 @@ class AppservicePingRestServlet(RestServlet): start = time.monotonic() try: await self.as_api.ping(requester.app_service, txn_id) + + # We got a OK response, so if the AS needs to be recovered then lets recover it now. + # This sets off a task in the background and so is safe to execute and forget. + self.scheduler.txn_ctrl.force_retry(requester.app_service) except RequestTimedOutError as e: raise SynapseError( HTTPStatus.GATEWAY_TIMEOUT, diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 730b00a9fb..a5bf7e0635 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -2,7 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright 2015, 2016 OpenMarket Ltd -# Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2023, 2025 New Vector, Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -234,6 +234,41 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEqual(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) + def test_recover_force_retry(self) -> None: + txn = Mock() + txns = [txn, None] + pop_txn = False + + def take_txn( + *args: object, **kwargs: object + ) -> "defer.Deferred[Optional[Mock]]": + if pop_txn: + return defer.succeed(txns.pop(0)) + else: + return defer.succeed(txn) + + self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn) + + # Start the recovery, and then fail the first attempt. + self.recoverer.recover() + self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) + txn.send = AsyncMock(return_value=False) + txn.complete = AsyncMock(return_value=None) + self.clock.advance_time(2) + self.assertEqual(1, txn.send.call_count) + self.assertEqual(0, txn.complete.call_count) + self.assertEqual(0, self.callback.call_count) + + # Now allow the send to succeed, and force a retry. + pop_txn = True # returns the txn the first time, then no more. + txn.send = AsyncMock(return_value=True) # successfully send the txn + self.recoverer.force_retry() + self.assertEqual(1, txn.send.call_count) # new mock reset call count + self.assertEqual(1, txn.complete.call_count) + + # Ensure we call the callback to say we're done! + self.callback.assert_called_once_with(self.recoverer) + # Corresponds to synapse.appservice.scheduler._TransactionController.send TxnCtrlArgs: TypeAlias = """ From 6e600c986eba664c1ba60b8b78a29c1878f96747 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jun 2025 15:37:15 +0100 Subject: [PATCH 361/691] Don't allow users to ignore themselves. (#18508) Fixes the self-ignore issues we've being seeing of reports of by ignoring bad requests from clients. Fixes https://github.com/element-hq/synapse/issues/11963 Fix https://github.com/element-hq/element-web/issues/29969 although this should also be fixed on the client to avoid confusing errors popping up while rejecting invites. Related to https://github.com/matrix-org/matrix-rust-sdk/issues/5073 --- changelog.d/18508.bugfix | 1 + synapse/storage/databases/main/account_data.py | 4 ++++ tests/storage/test_account_data.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 changelog.d/18508.bugfix diff --git a/changelog.d/18508.bugfix b/changelog.d/18508.bugfix new file mode 100644 index 0000000000..e8d5228ac1 --- /dev/null +++ b/changelog.d/18508.bugfix @@ -0,0 +1 @@ +Prevent users from adding themselves to their own ignore list. diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index d26de1ad16..715815cc09 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -34,6 +34,7 @@ from typing import ( ) from synapse.api.constants import AccountDataTypes +from synapse.api.errors import Codes, SynapseError from synapse.replication.tcp.streams import AccountDataStream from synapse.storage._base import db_to_json from synapse.storage.database import ( @@ -780,6 +781,9 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) else: currently_ignored_users = set() + if user_id in currently_ignored_users: + raise SynapseError(400, "You cannot ignore yourself", Codes.INVALID_PARAM) + # If the data has not changed, nothing to do. if previously_ignored_users == currently_ignored_users: return diff --git a/tests/storage/test_account_data.py b/tests/storage/test_account_data.py index 2859bcf4bd..0e52dd26ce 100644 --- a/tests/storage/test_account_data.py +++ b/tests/storage/test_account_data.py @@ -24,6 +24,7 @@ from typing import Iterable, Optional, Set from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import AccountDataTypes +from synapse.api.errors import Codes, SynapseError from synapse.server import HomeServer from synapse.util import Clock @@ -93,6 +94,20 @@ class IgnoredUsersTestCase(unittest.HomeserverTestCase): # Check the removed user. self.assert_ignorers("@another:remote", {self.user}) + def test_ignoring_self_fails(self) -> None: + """Ensure users cannot add themselves to the ignored list.""" + + f = self.get_failure( + self.store.add_account_data_for_user( + self.user, + AccountDataTypes.IGNORED_USER_LIST, + {"ignored_users": {self.user: {}}}, + ), + SynapseError, + ).value + self.assertEqual(f.code, 400) + self.assertEqual(f.errcode, Codes.INVALID_PARAM) + def test_caching(self) -> None: """Ensure that caching works properly between different users.""" # The first user ignores a user. From 6521406a37a40a490a1a435f3616c02cfcbbac16 Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Fri, 6 Jun 2025 18:14:09 +0300 Subject: [PATCH 362/691] Migrate to assertEqual (#18488) This small PR migrates from `unittest.assertEquals` to `unittest.assertEqual` which is deprecated from Python2.7: ```python DeprecationWarning: Please use assertEqual instead. ``` Signed-off-by: Emmanuel Ferdman --- tests/util/test_task_scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index 9e403b948b..7f6e63bd49 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -112,11 +112,11 @@ class TestTaskScheduler(HomeserverTestCase): # At this point, there should be MAX_CONCURRENT_RUNNING_TASKS active tasks and # one scheduled task. - self.assertEquals( + self.assertEqual( len(get_tasks_of_status(TaskStatus.ACTIVE)), TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, ) - self.assertEquals( + self.assertEqual( len(get_tasks_of_status(TaskStatus.SCHEDULED)), 1, ) @@ -126,17 +126,17 @@ class TestTaskScheduler(HomeserverTestCase): # Check that MAX_CONCURRENT_RUNNING_TASKS tasks have run and that one # is still scheduled. - self.assertEquals( + self.assertEqual( len(get_tasks_of_status(TaskStatus.COMPLETE)), TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS, ) scheduled_tasks = get_tasks_of_status(TaskStatus.SCHEDULED) - self.assertEquals(len(scheduled_tasks), 1) + self.assertEqual(len(scheduled_tasks), 1) # The scheduled task should start 0.1s after the first of the active tasks # finishes self.reactor.advance(0.1) - self.assertEquals(len(get_tasks_of_status(TaskStatus.ACTIVE)), 1) + self.assertEqual(len(get_tasks_of_status(TaskStatus.ACTIVE)), 1) # ... and should finally complete after another second self.reactor.advance(1) @@ -144,7 +144,7 @@ class TestTaskScheduler(HomeserverTestCase): self.task_scheduler.get_task(scheduled_tasks[0].id) ) assert prev_scheduled_task is not None - self.assertEquals( + self.assertEqual( prev_scheduled_task.status, TaskStatus.COMPLETE, ) From 341d956ee643567ab3a53f5626b38ace4826bf67 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Mon, 9 Jun 2025 11:59:49 +0100 Subject: [PATCH 363/691] Default to `public` join rule in remote summary (#18493) See: https://github.com/element-hq/synapse/issues/18358#issuecomment-2866119550 --- changelog.d/18493.bugfix | 1 + synapse/handlers/room_summary.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18493.bugfix diff --git a/changelog.d/18493.bugfix b/changelog.d/18493.bugfix new file mode 100644 index 0000000000..a4fab75e69 --- /dev/null +++ b/changelog.d/18493.bugfix @@ -0,0 +1 @@ +Fixed room summary API incorrectly returning that a room is private in the room summary response when the join rule is omitted by the remote server. Contributed by @nexy7574. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 64f5bea014..91b131d09b 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -701,7 +701,7 @@ class RoomSummaryHandler: # The API doesn't return the room version so assume that a # join rule of knock is valid. if ( - room.get("join_rule") + room.get("join_rule", JoinRules.PUBLIC) in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) or room.get("world_readable") is True ): From 1ab35a0a7825199a2d65e1c5a16524cd45a98721 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 10 Jun 2025 11:13:47 +0100 Subject: [PATCH 364/691] Mark new module APIs as experimental (#18536) --- changelog.d/18536.doc | 1 + docs/modules/media_repository_callbacks.md | 10 ++++++++++ docs/modules/ratelimit_callbacks.md | 5 +++++ docs/modules/spam_checker_callbacks.md | 5 +++++ 4 files changed, 21 insertions(+) create mode 100644 changelog.d/18536.doc diff --git a/changelog.d/18536.doc b/changelog.d/18536.doc new file mode 100644 index 0000000000..db57be4a83 --- /dev/null +++ b/changelog.d/18536.doc @@ -0,0 +1 @@ +Mark the new module APIs in this release as experimental. \ No newline at end of file diff --git a/docs/modules/media_repository_callbacks.md b/docs/modules/media_repository_callbacks.md index 0cee3384bb..fc37130439 100644 --- a/docs/modules/media_repository_callbacks.md +++ b/docs/modules/media_repository_callbacks.md @@ -14,6 +14,11 @@ _First introduced in Synapse v1.132.0_ async def get_media_config_for_user(user_id: str) -> Optional[JsonDict] ``` +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + Called when processing a request from a client for the [media config endpoint](https://spec.matrix.org/latest/client-server-api/#get_matrixclientv1mediaconfig). @@ -39,6 +44,11 @@ _First introduced in Synapse v1.132.0_ async def is_user_allowed_to_upload_media_of_size(user_id: str, size: int) -> bool ``` +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + Called before media is accepted for upload from a user, in case the module needs to enforce a different limit for the particular user. diff --git a/docs/modules/ratelimit_callbacks.md b/docs/modules/ratelimit_callbacks.md index 3386220e37..30d94024fa 100644 --- a/docs/modules/ratelimit_callbacks.md +++ b/docs/modules/ratelimit_callbacks.md @@ -14,6 +14,11 @@ _First introduced in Synapse v1.132.0_ async def get_ratelimit_override_for_user(user: str, limiter_name: str) -> Optional[synapse.module_api.RatelimitOverride] ``` +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + Called when constructing a ratelimiter of a particular type for a user. The module can return a `messages_per_second` and `burst_count` to be used, or `None` if the default settings are adequate. The user is represented by their Matrix user ID diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 6dfa2a7b25..39d7cbc000 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -254,6 +254,11 @@ _First introduced in Synapse v1.132.0_ async def user_may_send_state_event(user_id: str, room_id: str, event_type: str, state_key: str, content: JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"] ``` +** +Caution: This callback is currently experimental . The method signature or behaviour +may change without notice. +** + Called when processing a request to [send state events](https://spec.matrix.org/latest/client-server-api/#put_matrixclientv3roomsroomidstateeventtypestatekey) to a room. The arguments passed to this callback are: From 5581fbb9065928b4fc642c7a83aaacdaefdd4f31 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 10 Jun 2025 11:18:17 +0100 Subject: [PATCH 365/691] 1.132.0rc1 --- CHANGES.md | 35 +++++++++++++++++++++++++++++++ changelog.d/17892.doc | 1 - changelog.d/18252.misc | 1 - changelog.d/18288.feature | 1 - changelog.d/18310.misc | 1 - changelog.d/18408.doc | 1 - changelog.d/18455.feature | 1 - changelog.d/18456.feature | 1 - changelog.d/18457.feature | 1 - changelog.d/18458.feature | 1 - changelog.d/18484.bugfix | 1 - changelog.d/18486.feature | 1 - changelog.d/18493.bugfix | 1 - changelog.d/18508.bugfix | 1 - changelog.d/18510.misc | 1 - changelog.d/18513.bugfix | 1 - changelog.d/18516.doc | 1 - changelog.d/18521.feature | 1 - changelog.d/18536.doc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 43 insertions(+), 20 deletions(-) delete mode 100644 changelog.d/17892.doc delete mode 100644 changelog.d/18252.misc delete mode 100644 changelog.d/18288.feature delete mode 100644 changelog.d/18310.misc delete mode 100644 changelog.d/18408.doc delete mode 100644 changelog.d/18455.feature delete mode 100644 changelog.d/18456.feature delete mode 100644 changelog.d/18457.feature delete mode 100644 changelog.d/18458.feature delete mode 100644 changelog.d/18484.bugfix delete mode 100644 changelog.d/18486.feature delete mode 100644 changelog.d/18493.bugfix delete mode 100644 changelog.d/18508.bugfix delete mode 100644 changelog.d/18510.misc delete mode 100644 changelog.d/18513.bugfix delete mode 100644 changelog.d/18516.doc delete mode 100644 changelog.d/18521.feature delete mode 100644 changelog.d/18536.doc diff --git a/CHANGES.md b/CHANGES.md index 72091d605c..c7d6c1f129 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,38 @@ +# Synapse 1.132.0rc1 (2025-06-10) + +### Features + +- Add support for [MSC4155](https://github.com/matrix-org/matrix-spec-proposals/pull/4155) Invite Filtering. ([\#18288](https://github.com/element-hq/synapse/issues/18288)) +- Add experimental `user_may_send_state_event` module API callback. ([\#18455](https://github.com/element-hq/synapse/issues/18455)) +- Add experimental `get_media_config_for_user` and `is_user_allowed_to_upload_media_of_size` module API callbacks that allow overriding of media repository maximum upload size. ([\#18457](https://github.com/element-hq/synapse/issues/18457)) +- Add experimental `get_ratelimit_override_for_user` module API callback that allows overriding of per-user ratelimits. ([\#18458](https://github.com/element-hq/synapse/issues/18458)) +- Pass `room_config` argument to `user_may_create_room` spam checker module callback. ([\#18486](https://github.com/element-hq/synapse/issues/18486)) +- Support configuration of default and extra user types. ([\#18456](https://github.com/element-hq/synapse/issues/18456)) +- Successful requests to `/_matrix/app/v1/ping` will now force Synapse to reattempt delivering transactions to appservices. ([\#18521](https://github.com/element-hq/synapse/issues/18521)) +- Support the import of the `RatelimitOverride` type from `synapse.module_api` in modules and rename `messages_per_second` to `per_second`. ([\#18513](https://github.com/element-hq/synapse/issues/18513)) + +### Bugfixes + +- Remove destinations from sending if not whitelisted. ([\#18484](https://github.com/element-hq/synapse/issues/18484)) +- Fixed room summary API incorrectly returning that a room is private in the room summary response when the join rule is omitted by the remote server. Contributed by @nexy7574. ([\#18493](https://github.com/element-hq/synapse/issues/18493)) +- Prevent users from adding themselves to their own user ignore list. ([\#18508](https://github.com/element-hq/synapse/issues/18508)) + +### Improved Documentation + +- Generate config documentation from JSON Schema file. ([\#17892](https://github.com/element-hq/synapse/issues/17892)) +- Mention `CAP_NET_BIND_SERVICE` as an alternative to running Synapse as root in order to bind to a privileged port. ([\#18408](https://github.com/element-hq/synapse/issues/18408)) +- Surface hidden Admin API documentation regarding fetching of scheduled tasks. ([\#18516](https://github.com/element-hq/synapse/issues/18516)) +- Mark the new module APIs in this release as experimental. ([\#18536](https://github.com/element-hq/synapse/issues/18536)) + +### Internal Changes + +- Mark dehydrated devices in the [List All User Devices Admin API](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#list-all-devices). ([\#18252](https://github.com/element-hq/synapse/issues/18252)) +- Reduce disk wastage by cleaning up `received_transactions` older than 1 day, rather than 30 days. ([\#18310](https://github.com/element-hq/synapse/issues/18310)) +- Distinguish all vs local events being persisted in the "Event Send Time Quantiles" graph (Grafana). ([\#18510](https://github.com/element-hq/synapse/issues/18510)) + + + + # Synapse 1.131.0 (2025-06-03) No significant changes since 1.131.0rc1. diff --git a/changelog.d/17892.doc b/changelog.d/17892.doc deleted file mode 100644 index 83c03190dd..0000000000 --- a/changelog.d/17892.doc +++ /dev/null @@ -1 +0,0 @@ -Generate config documentation from JSON Schema file. diff --git a/changelog.d/18252.misc b/changelog.d/18252.misc deleted file mode 100644 index 6c76aca29c..0000000000 --- a/changelog.d/18252.misc +++ /dev/null @@ -1 +0,0 @@ -Mark dehydrated devices in the [List All User Devices Admin API](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#list-all-devices). \ No newline at end of file diff --git a/changelog.d/18288.feature b/changelog.d/18288.feature deleted file mode 100644 index 6856167875..0000000000 --- a/changelog.d/18288.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for [MSC4155](https://github.com/matrix-org/matrix-spec-proposals/pull/4155) Invite Filtering. diff --git a/changelog.d/18310.misc b/changelog.d/18310.misc deleted file mode 100644 index 1837b339f4..0000000000 --- a/changelog.d/18310.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce disk wastage by cleaning up `received_transactions` older than 1 day, rather than 30 days. \ No newline at end of file diff --git a/changelog.d/18408.doc b/changelog.d/18408.doc deleted file mode 100644 index d88f429381..0000000000 --- a/changelog.d/18408.doc +++ /dev/null @@ -1 +0,0 @@ -Mention `CAP_NET_BIND_SERVICE` as an alternative to running Synapse as root in order to bind to a privileged port. diff --git a/changelog.d/18455.feature b/changelog.d/18455.feature deleted file mode 100644 index f1b90e4a60..0000000000 --- a/changelog.d/18455.feature +++ /dev/null @@ -1 +0,0 @@ -Add user_may_send_state_event callback to spam checker module API. \ No newline at end of file diff --git a/changelog.d/18456.feature b/changelog.d/18456.feature deleted file mode 100644 index 706417e341..0000000000 --- a/changelog.d/18456.feature +++ /dev/null @@ -1 +0,0 @@ -Support configuration of default and extra user types. \ No newline at end of file diff --git a/changelog.d/18457.feature b/changelog.d/18457.feature deleted file mode 100644 index 76374dc3cd..0000000000 --- a/changelog.d/18457.feature +++ /dev/null @@ -1 +0,0 @@ -Add new module API callbacks that allows overriding of media repository maximum upload size. \ No newline at end of file diff --git a/changelog.d/18458.feature b/changelog.d/18458.feature deleted file mode 100644 index 92f11a8a2c..0000000000 --- a/changelog.d/18458.feature +++ /dev/null @@ -1 +0,0 @@ -Add a new module API callback that allows overriding of per user ratelimits. \ No newline at end of file diff --git a/changelog.d/18484.bugfix b/changelog.d/18484.bugfix deleted file mode 100644 index c816410da4..0000000000 --- a/changelog.d/18484.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove destinations from sending if not whitelisted. diff --git a/changelog.d/18486.feature b/changelog.d/18486.feature deleted file mode 100644 index 7e1c713081..0000000000 --- a/changelog.d/18486.feature +++ /dev/null @@ -1 +0,0 @@ -Pass room_config argument to user_may_create_room spam checker module callback. diff --git a/changelog.d/18493.bugfix b/changelog.d/18493.bugfix deleted file mode 100644 index a4fab75e69..0000000000 --- a/changelog.d/18493.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed room summary API incorrectly returning that a room is private in the room summary response when the join rule is omitted by the remote server. Contributed by @nexy7574. diff --git a/changelog.d/18508.bugfix b/changelog.d/18508.bugfix deleted file mode 100644 index e8d5228ac1..0000000000 --- a/changelog.d/18508.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent users from adding themselves to their own ignore list. diff --git a/changelog.d/18510.misc b/changelog.d/18510.misc deleted file mode 100644 index 71a3549d31..0000000000 --- a/changelog.d/18510.misc +++ /dev/null @@ -1 +0,0 @@ -Distinguish all vs local events being persisted in the "Event Send Time Quantiles" graph (Grafana). diff --git a/changelog.d/18513.bugfix b/changelog.d/18513.bugfix deleted file mode 100644 index b44eaafd15..0000000000 --- a/changelog.d/18513.bugfix +++ /dev/null @@ -1 +0,0 @@ -Support the import of the `RatelimitOverride` type from `synapse.module_api` in modules and rename `messages_per_second` to `per_second`. \ No newline at end of file diff --git a/changelog.d/18516.doc b/changelog.d/18516.doc deleted file mode 100644 index c739d32f5e..0000000000 --- a/changelog.d/18516.doc +++ /dev/null @@ -1 +0,0 @@ -Surface hidden Admin API documentation regarding fetching of scheduled tasks. \ No newline at end of file diff --git a/changelog.d/18521.feature b/changelog.d/18521.feature deleted file mode 100644 index 29da172e91..0000000000 --- a/changelog.d/18521.feature +++ /dev/null @@ -1 +0,0 @@ -Successful requests to `/_matrix/app/v1/ping` will now force Synapse to reattempt delivering transactions to appservices. diff --git a/changelog.d/18536.doc b/changelog.d/18536.doc deleted file mode 100644 index db57be4a83..0000000000 --- a/changelog.d/18536.doc +++ /dev/null @@ -1 +0,0 @@ -Mark the new module APIs in this release as experimental. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 8ad7f3184a..e0d7ea20fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.132.0~rc1) stable; urgency=medium + + * New Synapse release 1.132.0rc1. + + -- Synapse Packaging team Tue, 10 Jun 2025 11:15:18 +0100 + matrix-synapse-py3 (1.131.0) stable; urgency=medium * New Synapse release 1.131.0. diff --git a/pyproject.toml b/pyproject.toml index d95881b53a..2359c74032 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.131.0" +version = "1.132.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 5ebe80f51c..88804cab8d 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/v1.131/schema/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.132/synapse-config.schema.json type: object properties: modules: From 96c556081a5100a020082eea63982774375d2ea2 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 10 Jun 2025 11:43:58 +0000 Subject: [PATCH 366/691] Add config doc generation command to lint.sh and add missing config schema. (#18522) Follows: #17892, #18456
  1. Add config doc generation command to lint.sh
  2. Add missing `user_types` config schema
--------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18522.doc | 1 + .../configuration/config_documentation.md | 19 ++++++++++--------- schema/synapse-config.schema.yaml | 18 ++++++++++++++++++ scripts-dev/lint.sh | 3 +++ 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 changelog.d/18522.doc diff --git a/changelog.d/18522.doc b/changelog.d/18522.doc new file mode 100644 index 0000000000..83c03190dd --- /dev/null +++ b/changelog.d/18522.doc @@ -0,0 +1 @@ +Generate config documentation from JSON Schema file. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index c014de794d..1e8953ae37 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -764,22 +764,23 @@ max_event_delay_duration: 24h --- ### `user_types` -Configuration settings related to the user types feature. +*(object)* Configuration settings related to the user types feature. This setting has the following sub-options: -* `default_user_type`: The default user type to use for registering new users when no value has been specified. - Defaults to none. -* `extra_user_types`: Array of additional user types to allow. These are treated as real users. Defaults to []. + +* `default_user_type` (string|null): The default user type to use for registering new users when no value has been specified. Defaults to none. Defaults to `null`. + +* `extra_user_types` (list): Array of additional user types to allow. These are treated as real users. Defaults to `[]`. Example configuration: ```yaml user_types: - default_user_type: "custom" - extra_user_types: - - "custom" - - "custom2" + default_user_type: custom + extra_user_types: + - custom + - custom2 ``` - +--- ## Homeserver blocking Useful options for Synapse admins. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 5ebe80f51c..b04f149962 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -912,6 +912,24 @@ properties: default: null examples: - 24h + user_types: + type: object + description: >- + Configuration settings related to the user types feature. + properties: + default_user_type: + type: ["string", "null"] + description: "The default user type to use for registering new users when no value has been specified. Defaults to none." + default: null + extra_user_types: + type: array + description: "Array of additional user types to allow. These are treated as real users." + items: + type: string + default: [] + examples: + - default_user_type: "custom" + extra_user_types: ["custom", "custom2"] admin_contact: type: ["string", "null"] description: How to reach the server admin, used in `ResourceLimitError`. diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index c656047729..7096100a3e 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -139,3 +139,6 @@ cargo-fmt # Ensure type hints are correct. mypy + +# Generate configuration documentation from the JSON Schema +./scripts-dev/gen_config_documentation.py schema/synapse-config.schema.yaml > docs/usage/configuration/config_documentation.md From d5da07703d43171cb6d5d33c97b86d61bfa0427f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 10 Jun 2025 11:44:04 +0000 Subject: [PATCH 367/691] Config schema documentation CI: fix not failing when it should (#18528) Follows: #17892
  1. Config documentation CI: fix not failing if changes are outstanding
Shown to work at : https://github.com/element-hq/synapse/actions/runs/15532406886/job/43724019104?pr=18528 --------- Signed-off-by: Olivier 'reivilibre --- .github/workflows/schema.yaml | 2 +- changelog.d/18528.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18528.doc diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 41f3cd01be..1e13c1fef0 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -51,4 +51,4 @@ jobs: > docs/usage/configuration/config_documentation.md - name: Error in case of any differences # Errors if there are now any modified files (untracked files are ignored). - run: 'git diff || ! git status --porcelain=1 | grep "^ M"' + run: 'git diff --exit-code' diff --git a/changelog.d/18528.doc b/changelog.d/18528.doc new file mode 100644 index 0000000000..83c03190dd --- /dev/null +++ b/changelog.d/18528.doc @@ -0,0 +1 @@ +Generate config documentation from JSON Schema file. From cc6b4980efb281185a0f7dd958d121cf2d268855 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 10 Jun 2025 11:43:58 +0000 Subject: [PATCH 368/691] Add config doc generation command to lint.sh and add missing config schema. (#18522) Follows: #17892, #18456
  1. Add config doc generation command to lint.sh
  2. Add missing `user_types` config schema
--------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18522.doc | 1 + .../configuration/config_documentation.md | 19 ++++++++++--------- schema/synapse-config.schema.yaml | 18 ++++++++++++++++++ scripts-dev/lint.sh | 3 +++ 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 changelog.d/18522.doc diff --git a/changelog.d/18522.doc b/changelog.d/18522.doc new file mode 100644 index 0000000000..83c03190dd --- /dev/null +++ b/changelog.d/18522.doc @@ -0,0 +1 @@ +Generate config documentation from JSON Schema file. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index c014de794d..1e8953ae37 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -764,22 +764,23 @@ max_event_delay_duration: 24h --- ### `user_types` -Configuration settings related to the user types feature. +*(object)* Configuration settings related to the user types feature. This setting has the following sub-options: -* `default_user_type`: The default user type to use for registering new users when no value has been specified. - Defaults to none. -* `extra_user_types`: Array of additional user types to allow. These are treated as real users. Defaults to []. + +* `default_user_type` (string|null): The default user type to use for registering new users when no value has been specified. Defaults to none. Defaults to `null`. + +* `extra_user_types` (list): Array of additional user types to allow. These are treated as real users. Defaults to `[]`. Example configuration: ```yaml user_types: - default_user_type: "custom" - extra_user_types: - - "custom" - - "custom2" + default_user_type: custom + extra_user_types: + - custom + - custom2 ``` - +--- ## Homeserver blocking Useful options for Synapse admins. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 88804cab8d..52a6d5cf71 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -912,6 +912,24 @@ properties: default: null examples: - 24h + user_types: + type: object + description: >- + Configuration settings related to the user types feature. + properties: + default_user_type: + type: ["string", "null"] + description: "The default user type to use for registering new users when no value has been specified. Defaults to none." + default: null + extra_user_types: + type: array + description: "Array of additional user types to allow. These are treated as real users." + items: + type: string + default: [] + examples: + - default_user_type: "custom" + extra_user_types: ["custom", "custom2"] admin_contact: type: ["string", "null"] description: How to reach the server admin, used in `ResourceLimitError`. diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index c656047729..7096100a3e 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -139,3 +139,6 @@ cargo-fmt # Ensure type hints are correct. mypy + +# Generate configuration documentation from the JSON Schema +./scripts-dev/gen_config_documentation.py schema/synapse-config.schema.yaml > docs/usage/configuration/config_documentation.md From 142ba5df89045f94291e3a7fdf2cb797d24d15b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 14:38:54 +0100 Subject: [PATCH 369/691] Bump headers from 0.4.0 to 0.4.1 (#18529) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 980dff6987..fa68713ccb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.8.0" @@ -137,11 +143,11 @@ dependencies = [ [[package]] name = "headers" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", "http", @@ -511,7 +517,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "blake2", "bytes", "headers", From 978032141b0255b08116bdcaadd903eda56f782e Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Tue, 10 Jun 2025 15:11:49 +0100 Subject: [PATCH 370/691] bugfix: ensure _get_power_level_for_sender works when there is no PL event (#18534) --- changelog.d/18534.bugfix | 1 + synapse/state/v2.py | 14 ++++- tests/state/test_v2.py | 118 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18534.bugfix diff --git a/changelog.d/18534.bugfix b/changelog.d/18534.bugfix new file mode 100644 index 0000000000..ee1bfc8f27 --- /dev/null +++ b/changelog.d/18534.bugfix @@ -0,0 +1 @@ +Fix an issue where during state resolution for v11 rooms Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. diff --git a/synapse/state/v2.py b/synapse/state/v2.py index d0c0a9fc96..11e80df425 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -254,7 +254,19 @@ async def _get_power_level_for_sender( room_id, aid, event_map, state_res_store, allow_none=True ) if aev and (aev.type, aev.state_key) == (EventTypes.Create, ""): - if aev.content.get("creator") == event.sender: + creator = ( + event.sender + if event.room_version.implicit_room_creator + else aev.content.get("creator") + ) + if not creator: + logger.warning( + "_get_power_level_for_sender: event %s has no PL in auth_events and " + "creator is missing from create event %s", + event_id, + aev.event_id, + ) + if creator == event.sender: return 100 break return 0 diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 0b70f779d9..968c1ff64c 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -41,6 +41,7 @@ from synapse.event_auth import auth_types_for_event from synapse.events import EventBase, make_event_from_dict from synapse.state.v2 import ( _get_auth_chain_difference, + _get_power_level_for_sender, lexicographical_topological_sort, resolve_events_with_store, ) @@ -863,6 +864,123 @@ class AuthChainDifferenceTestCase(unittest.TestCase): self.assertEqual(difference, {d.event_id, e.event_id}) + def test_get_power_level_for_sender(self) -> None: + """Test that we use the correct definition of `creator` depending + on room version""" + store = TestStateResolutionStore({}) + for room_version in [RoomVersions.V10, RoomVersions.V11]: + create_event = make_event_from_dict( + { + "room_id": ROOM_ID, + "sender": ALICE, + "type": EventTypes.Create, + "state_key": "", + "content": { + "room_version": room_version.identifier, + } + # conditionally add 'creator' if the version doesn't use implicit room creators + | ( + {"creator": ALICE} + if not room_version.implicit_room_creator + else {} + ), + }, + room_version, + ) + member_event = make_event_from_dict( + { + "room_id": ROOM_ID, + "sender": ALICE, + "type": EventTypes.Member, + "state_key": ALICE, + "content": { + "membership": "join", + }, + "auth_events": [create_event.event_id], + "prev_events": [create_event.event_id], + }, + room_version, + ) + pl_event = make_event_from_dict( + { + "room_id": ROOM_ID, + "sender": ALICE, + "type": EventTypes.PowerLevels, + "state_key": "", + "content": { + "users": { + ALICE: 100, + BOB: 50, + }, + "users_default": 10, + }, + "auth_events": [create_event.event_id, member_event.event_id], + "prev_events": [member_event.event_id], + }, + room_version, + ) + + event_map = { + create_event.event_id: create_event, + member_event.event_id: member_event, + pl_event.event_id: pl_event, + } + want_pls = { + ALICE: 100, + BOB: 50, + CHARLIE: 10, + } + for user_id, want_pl in want_pls.items(): + test_event = make_event_from_dict( + { + "room_id": ROOM_ID, + "sender": user_id, + "type": EventTypes.Topic, + "state_key": "", + "content": {"topic": "Test"}, + "auth_events": [ + create_event.event_id, + member_event.event_id, + pl_event.event_id, + ], + "prev_events": [pl_event.event_id], + }, + room_version, + ) + event_map[test_event.event_id] = test_event + got_pl = self.successResultOf( + defer.ensureDeferred( + _get_power_level_for_sender( + ROOM_ID, test_event.event_id, event_map, store + ) + ) + ) + self.assertEqual( + got_pl, + want_pl, + f"wrong pl for {user_id} on v{room_version.identifier}", + ) + + # the creator alone without PL is 100 + got_creator_pl = self.successResultOf( + defer.ensureDeferred( + _get_power_level_for_sender( + ROOM_ID, + member_event.event_id, + { + member_event.event_id: member_event, + create_event.event_id: create_event, + }, + store, + ) + ) + ) + self.assertEqual( + got_creator_pl, + 100, + f"wrong pl for creator with no PL event on v{room_version.identifier}", + ) + T = TypeVar("T") From e4ca593eb6c3ddd4ee98091553df5f92344fc587 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 12 Jun 2025 11:21:39 +0100 Subject: [PATCH 371/691] Log user deactivations (#18541) One liner to give us more clarity when auditing deactivations of user accounts. ### Pull Request Checklist * [ ] Pull request is based on the develop branch * [ ] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [ ] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18541.misc | 1 + synapse/handlers/deactivate_account.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/18541.misc diff --git a/changelog.d/18541.misc b/changelog.d/18541.misc new file mode 100644 index 0000000000..2c733e3e50 --- /dev/null +++ b/changelog.d/18541.misc @@ -0,0 +1 @@ +Log user deactivations. diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 12a7cace55..42e53d920a 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -96,6 +96,14 @@ class DeactivateAccountHandler: 403, "Deactivation of this user is forbidden", Codes.FORBIDDEN ) + logger.info( + "%s requested deactivation of %s erase_data=%s id_server=%s", + requester.user, + user_id, + erase_data, + id_server, + ) + # FIXME: Theoretically there is a race here wherein user resets # password using threepid. From 0de7aa99533033151dd212e8409697f5a9fdbc6d Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 13 Jun 2025 09:44:18 +0200 Subject: [PATCH 372/691] Enable `flake8-logging` and `flake8-logging-format` rules in Ruff and fix related issues throughout the codebase (#18542) This can be reviewed commit by commit. This enables the `flake8-logging` and `flake8-logging-format` rules in Ruff, as well as logging exception stack traces in a few places where it makes sense - https://docs.astral.sh/ruff/rules/#flake8-logging-log - https://docs.astral.sh/ruff/rules/#flake8-logging-format-g ### Linting to avoid pre-formatting log messages See [`adamchainz/flake8-logging` -> *LOG011 avoid pre-formatting log messages*](https://github.com/adamchainz/flake8-logging/blob/152db2f167355fb23e401bf68046c57cb128a2ae/README.rst#log011-avoid-pre-formatting-log-messages) Practically, this means prefer placeholders (`%s`) over f-strings for logging. This is because placeholders are passed as args to loggers, so they can do special handling of them. For example, Sentry will record the args separately in their logging integration: https://github.com/getsentry/sentry-python/blob/c15b390dfe1ca5c01b30dd56b35d693bb50b413c/sentry_sdk/integrations/logging.py#L280-L284 One theoretical small perf benefit is that log levels that aren't enabled won't get formatted, so it doesn't unnecessarily create formatted strings --- changelog.d/18542.misc | 1 + pyproject.toml | 4 ++++ scripts-dev/check_pydantic_models.py | 2 +- synapse/api/auth/msc3861_delegated.py | 2 +- synapse/app/_base.py | 4 ++-- synapse/app/phone_stats_home.py | 2 +- synapse/appservice/scheduler.py | 2 +- synapse/config/logger.py | 22 ++++++++++--------- synapse/config/room.py | 2 +- synapse/config/server.py | 2 +- synapse/events/auto_accept_invites.py | 13 ++++++----- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 3 ++- synapse/handlers/admin.py | 2 +- synapse/handlers/appservice.py | 8 ++----- synapse/handlers/auth.py | 4 ++-- synapse/handlers/device.py | 2 +- synapse/handlers/directory.py | 2 +- synapse/handlers/federation.py | 4 +++- synapse/handlers/identity.py | 4 ++-- synapse/handlers/message.py | 9 +++++--- synapse/handlers/oidc.py | 18 ++++++++------- synapse/handlers/register.py | 4 ++-- synapse/handlers/room.py | 4 ++-- synapse/handlers/room_member.py | 6 +++-- synapse/handlers/saml.py | 2 +- synapse/handlers/send_email.py | 2 +- synapse/handlers/sso.py | 12 ++++++---- synapse/handlers/sync.py | 6 +++-- synapse/handlers/user_directory.py | 3 +-- synapse/handlers/worker_lock.py | 12 ++++++---- synapse/http/client.py | 10 ++++----- synapse/http/matrixfederationclient.py | 2 +- synapse/media/media_repository.py | 12 +++++----- synapse/media/url_previewer.py | 4 ++-- synapse/metrics/__init__.py | 4 ++-- synapse/push/mailer.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/admin/media.py | 22 ++++++++++--------- synapse/storage/_base.py | 2 +- .../databases/main/monthly_active_users.py | 2 +- .../storage/databases/main/user_directory.py | 5 +++-- synapse/storage/invite_rule.py | 4 +++- .../storage/schema/main/delta/30/as_users.py | 7 ++++-- synapse/util/check_dependencies.py | 4 +++- synapse/util/daemonize.py | 2 +- synapse/util/task_scheduler.py | 9 +++++--- tests/federation/test_federation_server.py | 6 +++-- tests/handlers/test_worker_lock.py | 6 +++-- .../test_matrix_federation_agent.py | 2 +- tests/http/test_proxyagent.py | 2 +- tests/logging/test_remote_handler.py | 12 +++++----- 52 files changed, 164 insertions(+), 122 deletions(-) create mode 100644 changelog.d/18542.misc diff --git a/changelog.d/18542.misc b/changelog.d/18542.misc new file mode 100644 index 0000000000..8a2c2d5bf6 --- /dev/null +++ b/changelog.d/18542.misc @@ -0,0 +1 @@ +Enable [`flake8-logging`](https://docs.astral.sh/ruff/rules/#flake8-logging-log) and [`flake8-logging-format`](https://docs.astral.sh/ruff/rules/#flake8-logging-format-g) rules in Ruff and fix related issues throughout the codebase. diff --git a/pyproject.toml b/pyproject.toml index 2359c74032..e520e425d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ select = [ "PIE", # flake8-executable "EXE", + # flake8-logging + "LOG", + # flake8-logging-format + "G", ] [tool.ruff.lint.isort] diff --git a/scripts-dev/check_pydantic_models.py b/scripts-dev/check_pydantic_models.py index 5eb1f0a9df..26a473a61b 100755 --- a/scripts-dev/check_pydantic_models.py +++ b/scripts-dev/check_pydantic_models.py @@ -243,7 +243,7 @@ def do_lint() -> Set[str]: importlib.import_module(module_info.name) except ModelCheckerException as e: logger.warning( - f"Bad annotation found when importing {module_info.name}" + "Bad annotation found when importing %s", module_info.name ) failures.add(format_model_checker_exception(e)) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index e500a06afe..ad5d59eef1 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -475,7 +475,7 @@ class MSC3861DelegatedAuth(BaseAuth): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. - logging.info("Admin toked used") + logger.info("Admin toked used") # XXX: that user doesn't exist and won't be provisioned. # This is mostly fine for admin calls, but we should also think about doing # requesters without a user_id. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4cc260d551..49ab5d680d 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -445,8 +445,8 @@ def listen_http( # getHost() returns a UNIXAddress which contains an instance variable of 'name' # encoded as a byte string. Decode as utf-8 so pretty. logger.info( - "Synapse now listening on Unix Socket at: " - f"{ports[0].getHost().name.decode('utf-8')}" + "Synapse now listening on Unix Socket at: %s", + ports[0].getHost().name.decode("utf-8"), ) return ports diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index bb450a235c..9cd2e1b71e 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -173,7 +173,7 @@ async def phone_stats_home( stats["log_level"] = logging.getLevelName(log_level) logger.info( - "Reporting stats to %s: %s" % (hs.config.metrics.report_stats_endpoint, stats) + "Reporting stats to %s: %s", hs.config.metrics.report_stats_endpoint, stats ) try: await hs.get_proxied_http_client().put_json( diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index cba08dde85..ab3f4e15fe 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -461,7 +461,7 @@ class _TransactionController: recoverer = self.recoverers.get(service.id) if not recoverer: # No need to force a retry on a happy AS. - logger.info(f"{service.id} is not in recovery, not forcing retry") + logger.info("%s is not in recovery, not forcing retry", service.id) return recoverer.force_retry() diff --git a/synapse/config/logger.py b/synapse/config/logger.py index e5aca36b75..3f86ec1169 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -51,6 +51,8 @@ if TYPE_CHECKING: from synapse.config.homeserver import HomeServerConfig from synapse.server import HomeServer +logger = logging.getLogger(__name__) + DEFAULT_LOG_CONFIG = Template( """\ # Log configuration for Synapse. @@ -291,7 +293,7 @@ def _load_logging_config(log_config_path: str) -> None: log_config = yaml.safe_load(f.read()) if not log_config: - logging.warning("Loaded a blank logging config?") + logger.warning("Loaded a blank logging config?") # If the old structured logging configuration is being used, raise an error. if "structured" in log_config and log_config.get("structured"): @@ -312,7 +314,7 @@ def _reload_logging_config(log_config_path: Optional[str]) -> None: return _load_logging_config(log_config_path) - logging.info("Reloaded log config from %s due to SIGHUP", log_config_path) + logger.info("Reloaded log config from %s due to SIGHUP", log_config_path) def setup_logging( @@ -349,17 +351,17 @@ def setup_logging( appbase.register_sighup(_reload_logging_config, log_config_path) # Log immediately so we can grep backwards. - logging.warning("***** STARTING SERVER *****") - logging.warning( + logger.warning("***** STARTING SERVER *****") + logger.warning( "Server %s version %s", sys.argv[0], SYNAPSE_VERSION, ) - logging.warning("Copyright (c) 2023 New Vector, Inc") - logging.warning( + logger.warning("Copyright (c) 2023 New Vector, Inc") + logger.warning( "Licensed under the AGPL 3.0 license. Website: https://github.com/element-hq/synapse" ) - logging.info("Server hostname: %s", config.server.server_name) - logging.info("Public Base URL: %s", config.server.public_baseurl) - logging.info("Instance name: %s", hs.get_instance_name()) - logging.info("Twisted reactor: %s", type(reactor).__name__) + logger.info("Server hostname: %s", config.server.server_name) + logger.info("Public Base URL: %s", config.server.public_baseurl) + logger.info("Instance name: %s", hs.get_instance_name()) + logger.info("Twisted reactor: %s", type(reactor).__name__) diff --git a/synapse/config/room.py b/synapse/config/room.py index ec8cf5be36..bad387ecbf 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -27,7 +27,7 @@ from synapse.types import JsonDict from ._base import Config, ConfigError -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) class RoomDefaultEncryptionTypes: diff --git a/synapse/config/server.py b/synapse/config/server.py index 6b29983617..6893450989 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -41,7 +41,7 @@ from synapse.util.stringutils import parse_and_validate_server_name from ._base import Config, ConfigError from ._util import validate_config -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) DIRECT_TCP_ERROR = """ Using direct TCP replication for workers is no longer supported. diff --git a/synapse/events/auto_accept_invites.py b/synapse/events/auto_accept_invites.py index 4295107c47..9e5f76f33f 100644 --- a/synapse/events/auto_accept_invites.py +++ b/synapse/events/auto_accept_invites.py @@ -195,15 +195,18 @@ class InviteAutoAccepter: except SynapseError as e: if e.code == HTTPStatus.FORBIDDEN: logger.debug( - f"Update_room_membership was forbidden. This can sometimes be expected for remote invites. Exception: {e}" + "Update_room_membership was forbidden. This can sometimes be expected for remote invites. Exception: %s", + e, ) else: - logger.warn( - f"Update_room_membership raised the following unexpected (SynapseError) exception: {e}" + logger.warning( + "Update_room_membership raised the following unexpected (SynapseError) exception: %s", + e, ) except Exception as e: - logger.warn( - f"Update_room_membership raised the following unexpected exception: {e}" + logger.warning( + "Update_room_membership raised the following unexpected exception: %s", + e, ) sleep = 2**retries diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7c485aa7e0..2d1da70793 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1818,7 +1818,7 @@ class FederationClient(FederationBase): ) return timestamp_to_event_response except SynapseError as e: - logger.warn( + logger.warning( "timestamp_to_event(room_id=%s, timestamp=%s, direction=%s): encountered error when trying to fetch from destinations: %s", room_id, timestamp, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2f2c78babc..8718b70401 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -928,7 +928,8 @@ class FederationServer(FederationBase): # joins) or the full state (for full joins). # Return a 404 as we would if we weren't in the room at all. logger.info( - f"Rejecting /send_{membership_type} to %s because it's a partial state room", + "Rejecting /send_%s to %s because it's a partial state room", + membership_type, room_id, ) raise SynapseError( diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 971a74244f..262437f04f 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -495,7 +495,7 @@ class AdminHandler: ) except Exception as ex: logger.info( - f"Redaction of event {event.event_id} failed due to: {ex}" + "Redaction of event %s failed due to: %s", event.event_id, ex ) result["failed_redactions"][event.event_id] = str(ex) await self._task_scheduler.update_task(task.id, result=result) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index b7d1033351..f3bbdb5a05 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -465,9 +465,7 @@ class ApplicationServicesHandler: service, "read_receipt" ) if new_token is not None and new_token.stream <= from_key: - logger.debug( - "Rejecting token lower than or equal to stored: %s" % (new_token,) - ) + logger.debug("Rejecting token lower than or equal to stored: %s", new_token) return [] from_token = MultiWriterStreamToken(stream=from_key) @@ -509,9 +507,7 @@ class ApplicationServicesHandler: service, "presence" ) if new_token is not None and new_token <= from_key: - logger.debug( - "Rejecting token lower than or equal to stored: %s" % (new_token,) - ) + logger.debug("Rejecting token lower than or equal to stored: %s", new_token) return [] for user in users: diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e96922c08d..de5b38caca 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1895,7 +1895,7 @@ def load_single_legacy_password_auth_provider( try: provider = module(config=config, account_handler=api) except Exception as e: - logger.error("Error while initializing %r: %s", module, e) + logger.exception("Error while initializing %r: %s", module, e) raise # All methods that the module provides should be async, but this wasn't enforced @@ -2428,7 +2428,7 @@ class PasswordAuthProvider: except CancelledError: raise except Exception as e: - logger.error("Module raised an exception in is_3pid_allowed: %s", e) + logger.exception("Module raised an exception in is_3pid_allowed: %s", e) raise SynapseError(code=500, msg="Internal Server Error") return True diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index f8b547bbed..8f9bf92fda 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1600,7 +1600,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): if prev_stream_id is not None and cached_devices == { d["device_id"]: d for d in devices }: - logging.info( + logger.info( "Skipping device list resync for %s, as our cache matches already", user_id, ) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 48c7d411d5..74c697960f 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -282,7 +282,7 @@ class DirectoryHandler: except RequestSendFailed: raise SynapseError(502, "Failed to fetch alias") except CodeMessageException as e: - logging.warning( + logger.warning( "Error retrieving alias %s -> %s %s", room_alias, e.code, e.msg ) if e.code == 404: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a6de3e824d..729b676163 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1095,7 +1095,9 @@ class FederationHandler: rule = invite_config.get_invite_rule(event.sender) if rule == InviteRule.BLOCK: logger.info( - f"Automatically rejecting invite from {event.sender} due to the invite filtering rules of {event.state_key}" + "Automatically rejecting invite from %s due to the invite filtering rules of %s", + event.sender, + event.state_key, ) raise SynapseError( 403, diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 89191217d6..d96b585308 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -218,7 +218,7 @@ class IdentityHandler: return data except HttpResponseException as e: - logger.error("3PID bind failed with Matrix error: %r", e) + logger.exception("3PID bind failed with Matrix error: %r", e) raise e.to_synapse_error() except RequestTimedOutError: raise SynapseError(500, "Timed out contacting identity server") @@ -323,7 +323,7 @@ class IdentityHandler: # The remote server probably doesn't support unbinding (yet) logger.warning("Received %d response while unbinding threepid", e.code) else: - logger.error("Failed to unbind threepid on identity server: %s", e) + logger.exception("Failed to unbind threepid on identity server: %s", e) raise SynapseError(500, "Failed to contact identity server") except RequestTimedOutError: raise SynapseError(500, "Timed out contacting identity server") diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index cb6de02309..5d6ee6996f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -460,7 +460,7 @@ class MessageHandler: # date from the database in the same database transaction. await self.store.expire_event(event_id) except Exception as e: - logger.error("Could not expire event %s: %r", event_id, e) + logger.exception("Could not expire event %s: %r", event_id, e) # Schedule the expiry of the next event to expire. await self._schedule_next_expiry() @@ -2061,7 +2061,8 @@ class EventCreationHandler: # dependent on _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY logger.info( "Failed to send dummy event into room %s. Will exclude it from " - "future attempts until cache expires" % (room_id,) + "future attempts until cache expires", + room_id, ) now = self.clock.time_msec() self._rooms_to_exclude_from_dummy_event_insertion[room_id] = now @@ -2120,7 +2121,9 @@ class EventCreationHandler: except AuthError: logger.info( "Failed to send dummy event into room %s for user %s due to " - "lack of power. Will try another user" % (room_id, user_id) + "lack of power. Will try another user", + room_id, + user_id, ) return False diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index acf2d4bc8b..8f3e717fb4 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -563,12 +563,13 @@ class OidcProvider: raise ValueError("Unexpected subject") except Exception: logger.warning( - f"OIDC Back-Channel Logout is enabled for issuer {self.issuer!r} " + "OIDC Back-Channel Logout is enabled for issuer %r " "but it looks like the configured `user_mapping_provider` " "does not use the `sub` claim as subject. If it is the case, " "and you want Synapse to ignore the `sub` claim in OIDC " "Back-Channel Logouts, set `backchannel_logout_ignore_sub` " - "to `true` in the issuer config." + "to `true` in the issuer config.", + self.issuer, ) @property @@ -826,10 +827,10 @@ class OidcProvider: if response.code < 400: logger.debug( "Invalid response from the authorization server: " - 'responded with a "{status}" ' - "but body has an error field: {error!r}".format( - status=status, error=resp["error"] - ) + 'responded with a "%s" ' + "but body has an error field: %r", + status, + resp["error"], ) description = resp.get("error_description", error) @@ -1385,7 +1386,8 @@ class OidcProvider: # support dynamic registration in Synapse at some point. if not self._config.backchannel_logout_enabled: logger.warning( - f"Received an OIDC Back-Channel Logout request from issuer {self.issuer!r} but it is disabled in config" + "Received an OIDC Back-Channel Logout request from issuer %r but it is disabled in config", + self.issuer, ) # TODO: this responds with a 400 status code, which is what the OIDC @@ -1797,5 +1799,5 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): extras[key] = template.render(user=userinfo).strip() except Exception as e: # Log an error and skip this value (don't break login for this). - logger.error("Failed to render OIDC extra attribute %s: %s" % (key, e)) + logger.exception("Failed to render OIDC extra attribute %s: %s", key, e) return extras diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d577039a4c..970013ef20 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -506,7 +506,7 @@ class RegistrationHandler: ratelimit=False, ) except Exception as e: - logger.error("Failed to join new user to %r: %r", r, e) + logger.exception("Failed to join new user to %r: %r", r, e) async def _join_rooms(self, user_id: str) -> None: """ @@ -596,7 +596,7 @@ class RegistrationHandler: # moving away from bare excepts is a good thing to do. logger.error("Failed to join new user to %r: %r", r, e) except Exception as e: - logger.error("Failed to join new user to %r: %r", r, e, exc_info=True) + logger.exception("Failed to join new user to %r: %r", r, e) async def _auto_join_rooms(self, user_id: str) -> None: """Automatically joins users to auto join rooms - creating the room in the first place diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1ccb6f7171..0f004d02d3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -698,7 +698,7 @@ class RoomCreationHandler: except SynapseError as e: # again I'm not really expecting this to fail, but if it does, I'd rather # we returned the new room to the client at this point. - logger.error("Unable to send updated alias events in old room: %s", e) + logger.exception("Unable to send updated alias events in old room: %s", e) try: await self.event_creation_handler.create_and_send_nonmember_event( @@ -715,7 +715,7 @@ class RoomCreationHandler: except SynapseError as e: # again I'm not really expecting this to fail, but if it does, I'd rather # we returned the new room to the client at this point. - logger.error("Unable to send updated alias events in new room: %s", e) + logger.exception("Unable to send updated alias events in new room: %s", e) async def create_room( self, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 24ee5e89a6..cf9db7b018 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -922,7 +922,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): rule = invite_config.get_invite_rule(requester.user.to_string()) if rule == InviteRule.BLOCK: logger.info( - f"Automatically rejecting invite from {target_id} due to the the invite filtering rules of {requester.user}" + "Automatically rejecting invite from %s due to the the invite filtering rules of %s", + target_id, + requester.user, ) raise SynapseError( 403, @@ -1570,7 +1572,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): require_consent=False, ) except Exception as e: - logger.exception("Error kicking guest user: %s" % (e,)) + logger.exception("Error kicking guest user: %s", e) async def lookup_room_alias( self, room_alias: RoomAlias diff --git a/synapse/handlers/saml.py b/synapse/handlers/saml.py index 8ebd3d4ff9..81bec7499c 100644 --- a/synapse/handlers/saml.py +++ b/synapse/handlers/saml.py @@ -124,7 +124,7 @@ class SamlHandler: ) # Since SAML sessions timeout it is useful to log when they were created. - logger.info("Initiating a new SAML session: %s" % (reqid,)) + logger.info("Initiating a new SAML session: %s", reqid) now = self.clock.time_msec() self._outstanding_requests_dict[reqid] = Saml2SessionData( diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 8cf8d2bada..353991573f 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -238,7 +238,7 @@ class SendEmailHandler: multipart_msg.attach(text_part) multipart_msg.attach(html_part) - logger.info("Sending email to %s" % email_address) + logger.info("Sending email to %s", email_address) await self._sendmail( self._reactor, diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 07827cf95b..7c5cf91aba 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -1230,12 +1230,16 @@ class SsoHandler: if expected_user_id is not None and user_id != expected_user_id: logger.error( "Received a logout notification from SSO provider " - f"{auth_provider_id!r} for the user {expected_user_id!r}, but with " - f"a session ID ({auth_provider_session_id!r}) which belongs to " - f"{user_id!r}. This may happen when the SSO provider user mapper " + "%r for the user %r, but with " + "a session ID (%r) which belongs to " + "%r. This may happen when the SSO provider user mapper " "uses something else than the standard attribute as mapping ID. " "For OIDC providers, set `backchannel_logout_ignore_sub` to `true` " - "in the provider config if that is the case." + "in the provider config if that is the case.", + auth_provider_id, + expected_user_id, + auth_provider_session_id, + user_id, ) continue diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c6f2c38d8d..a400e63fd5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -3074,8 +3074,10 @@ class SyncHandler: if batch.limited and since_token: user_id = sync_result_builder.sync_config.user.to_string() logger.debug( - "Incremental gappy sync of %s for user %s with %d state events" - % (room_id, user_id, len(state)) + "Incremental gappy sync of %s for user %s with %d state events", + room_id, + user_id, + len(state), ) elif room_builder.rtype == "archived": archived_room_sync = ArchivedSyncResult( diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 33edef5f14..1f692c79a0 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -749,10 +749,9 @@ class UserDirectoryHandler(StateDeltasHandler): ) continue except Exception: - logger.error( + logger.exception( "Failed to refresh profile for %r due to unhandled exception", user_id, - exc_info=True, ) await self.store.set_remote_user_profile_in_user_dir_stale( user_id, diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index e58a416026..b36f7bd5fd 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -50,6 +50,8 @@ if TYPE_CHECKING: from synapse.server import HomeServer +logger = logging.getLogger(__name__) + # This lock is used to avoid creating an event while we are purging the room. # We take a read lock when creating an event, and a write one when purging a room. # This is because it is fine to create several events concurrently, since referenced events @@ -271,8 +273,9 @@ class WaitingLock: next = self._retry_interval self._retry_interval = max(5, next * 2) if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes - logging.warning( - f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + logger.warning( + "Lock timeout is getting excessive: %ss. There may be a deadlock.", + self._retry_interval, ) return next * random.uniform(0.9, 1.1) @@ -350,7 +353,8 @@ class WaitingMultiLock: next = self._retry_interval self._retry_interval = max(5, next * 2) if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes - logging.warning( - f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + logger.warning( + "Lock timeout is getting excessive: %ss. There may be a deadlock.", + self._retry_interval, ) return next * random.uniform(0.9, 1.1) diff --git a/synapse/http/client.py b/synapse/http/client.py index 84a510fb42..928bfb228a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -213,7 +213,7 @@ class _IPBlockingResolver: if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): logger.info( - "Blocked %s from DNS resolution to %s" % (ip_address, hostname) + "Blocked %s from DNS resolution to %s", ip_address, hostname ) has_bad_ip = True @@ -318,7 +318,7 @@ class BlocklistingAgentWrapper(Agent): pass else: if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): - logger.info("Blocking access to %s" % (ip_address,)) + logger.info("Blocking access to %s", ip_address) e = SynapseError(HTTPStatus.FORBIDDEN, "IP address blocked") return defer.fail(Failure(e)) @@ -723,7 +723,7 @@ class BaseHttpClient: resp_headers = dict(response.headers.getAllRawHeaders()) if response.code > 299: - logger.warning("Got %d when downloading %s" % (response.code, url)) + logger.warning("Got %d when downloading %s", response.code, url) raise SynapseError( HTTPStatus.BAD_GATEWAY, "Got error %d" % (response.code,), Codes.UNKNOWN ) @@ -1106,7 +1106,7 @@ class _MultipartParserProtocol(protocol.Protocol): self.stream.write(data[start:end]) except Exception as e: logger.warning( - f"Exception encountered writing file data to stream: {e}" + "Exception encountered writing file data to stream: %s", e ) self.deferred.errback() self.file_length += end - start @@ -1129,7 +1129,7 @@ class _MultipartParserProtocol(protocol.Protocol): try: self.parser.write(incoming_data) except Exception as e: - logger.warning(f"Exception writing to multipart parser: {e}") + logger.warning("Exception writing to multipart parser: %s", e) self.deferred.errback() return diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 88bf98045c..97a863a118 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -602,7 +602,7 @@ class MatrixFederationHttpClient: try: parse_and_validate_server_name(request.destination) except ValueError: - logger.exception(f"Invalid destination: {request.destination}.") + logger.exception("Invalid destination: %s.", request.destination) raise FederationDeniedError(request.destination) if timeout is not None: diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 18c5a8ecec..1d05c536af 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -313,7 +313,7 @@ class MediaRepository: logger.info("Stored local media in file %r", fname) if should_quarantine: - logger.warn( + logger.warning( "Media has been automatically quarantined as it matched existing quarantined media" ) @@ -366,7 +366,7 @@ class MediaRepository: logger.info("Stored local media in file %r", fname) if should_quarantine: - logger.warn( + logger.warning( "Media has been automatically quarantined as it matched existing quarantined media" ) @@ -1393,8 +1393,8 @@ class MediaRepository: ) logger.info( - "Purging remote media last accessed before" - f" {remote_media_threshold_timestamp_ms}" + "Purging remote media last accessed before %s", + remote_media_threshold_timestamp_ms, ) await self.delete_old_remote_media( @@ -1409,8 +1409,8 @@ class MediaRepository: ) logger.info( - "Purging local media last accessed before" - f" {local_media_threshold_timestamp_ms}" + "Purging local media last accessed before %s", + local_media_threshold_timestamp_ms, ) await self.delete_old_local_media( diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 8ef2b3f0c0..0c665e1942 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -287,7 +287,7 @@ class UrlPreviewer: og["og:image:width"] = dims["width"] og["og:image:height"] = dims["height"] else: - logger.warning("Couldn't get dims for %s" % url) + logger.warning("Couldn't get dims for %s", url) # define our OG response for this media elif _is_html(media_info.media_type): @@ -609,7 +609,7 @@ class UrlPreviewer: should_quarantine = await self.store.get_is_hash_quarantined(sha256) if should_quarantine: - logger.warn( + logger.warning( "Media has been automatically quarantined as it matched existing quarantined media" ) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 3051b623d0..86ac2c2395 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -118,7 +118,7 @@ class LaterGauge(Collector): def _register(self) -> None: if self.name in all_gauges.keys(): - logger.warning("%s already registered, reregistering" % (self.name,)) + logger.warning("%s already registered, reregistering", self.name) REGISTRY.unregister(all_gauges.pop(self.name)) REGISTRY.register(self) @@ -244,7 +244,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): def _register_with_collector(self) -> None: if self.name in all_gauges.keys(): - logger.warning("%s already registered, reregistering" % (self.name,)) + logger.warning("%s already registered, reregistering", self.name) REGISTRY.unregister(all_gauges.pop(self.name)) REGISTRY.register(self) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index cf611bd90b..fadba480dd 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -135,7 +135,7 @@ class Mailer: self.app_name = app_name self.email_subjects: EmailSubjectConfig = hs.config.email.email_subjects - logger.info("Created Mailer for app_name %s" % app_name) + logger.info("Created Mailer for app_name %s", app_name) emails_sent_counter.labels("password_reset") diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 00f108de08..aeb4267bb7 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -165,7 +165,7 @@ class ClientRestResource(JsonResource): # Fail on unknown servlet groups. if servlet_group not in SERVLET_GROUPS: if servlet_group == "media": - logger.warn( + logger.warning( "media.can_load_media_repo needs to be configured for the media servlet to be available" ) raise RuntimeError( diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index ee6a681285..195f22a4c2 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -71,7 +71,7 @@ class QuarantineMediaInRoom(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining room: %s", room_id) + logger.info("Quarantining room: %s", room_id) # Quarantine all media in this room num_quarantined = await self.store.quarantine_media_ids_in_room( @@ -98,7 +98,7 @@ class QuarantineMediaByUser(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining media by user: %s", user_id) + logger.info("Quarantining media by user: %s", user_id) # Quarantine all media this user has uploaded num_quarantined = await self.store.quarantine_media_ids_by_user( @@ -127,7 +127,7 @@ class QuarantineMediaByID(RestServlet): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - logging.info("Quarantining media by ID: %s/%s", server_name, media_id) + logger.info("Quarantining media by ID: %s/%s", server_name, media_id) # Quarantine this media id await self.store.quarantine_media_by_id( @@ -155,7 +155,7 @@ class UnquarantineMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Remove from quarantine media by ID: %s/%s", server_name, media_id) + logger.info("Remove from quarantine media by ID: %s/%s", server_name, media_id) # Remove from quarantine this media id await self.store.quarantine_media_by_id(server_name, media_id, None) @@ -177,7 +177,7 @@ class ProtectMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Protecting local media by ID: %s", media_id) + logger.info("Protecting local media by ID: %s", media_id) # Protect this media id await self.store.mark_local_media_as_safe(media_id, safe=True) @@ -199,7 +199,7 @@ class UnprotectMediaByID(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - logging.info("Unprotecting local media by ID: %s", media_id) + logger.info("Unprotecting local media by ID: %s", media_id) # Unprotect this media id await self.store.mark_local_media_as_safe(media_id, safe=False) @@ -280,7 +280,7 @@ class DeleteMediaByID(RestServlet): if await self.store.get_local_media(media_id) is None: raise NotFoundError("Unknown media") - logging.info("Deleting local media by ID: %s", media_id) + logger.info("Deleting local media by ID: %s", media_id) deleted_media, total = await self.media_repository.delete_local_media_ids( [media_id] @@ -327,9 +327,11 @@ class DeleteMediaByDateSize(RestServlet): if server_name is not None and self.server_name != server_name: raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only delete local media") - logging.info( - "Deleting local media by timestamp: %s, size larger than: %s, keep profile media: %s" - % (before_ts, size_gt, keep_profiles) + logger.info( + "Deleting local media by timestamp: %s, size larger than: %s, keep profile media: %s", + before_ts, + size_gt, + keep_profiles, ) deleted_media, total = await self.media_repository.delete_old_local_media( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b5fe7dd858..2309b1648e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -240,5 +240,5 @@ def db_to_json(db_content: Union[memoryview, bytes, bytearray, str]) -> Any: try: return json_decoder.decode(db_content) except Exception: - logging.warning("Tried to decode '%r' as JSON and failed", db_content) + logger.warning("Tried to decode '%r' as JSON and failed", db_content) raise diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 659ee13d71..f5a6b98be7 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -331,7 +331,7 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): values={"timestamp": int(self._clock.time_msec())}, ) else: - logger.warning("mau limit reserved threepid %s not found in db" % tp) + logger.warning("mau limit reserved threepid %s not found in db", tp) async def upsert_monthly_active_user(self, user_id: str) -> None: """Updates or inserts the user into the monthly active user table, which diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 31a8ce6666..09671abcad 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -253,8 +253,9 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): return 1 logger.debug( - "Processing the next %d rooms of %d remaining" - % (len(rooms_to_work_on), progress["remaining"]) + "Processing the next %d rooms of %d remaining", + len(rooms_to_work_on), + progress["remaining"], ) processed_event_count = 0 diff --git a/synapse/storage/invite_rule.py b/synapse/storage/invite_rule.py index b9d9d1eb62..f63390871e 100644 --- a/synapse/storage/invite_rule.py +++ b/synapse/storage/invite_rule.py @@ -50,7 +50,9 @@ class InviteRulesConfig: except Exception as e: # If for whatever reason we can't process this, just ignore it. logger.debug( - f"Could not process '{value}' field of invite rule config, ignoring: {e}" + "Could not process '%s' field of invite rule config, ignoring: %s", + value, + e, ) if account_data: diff --git a/synapse/storage/schema/main/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py index 2a3023cd07..060217575b 100644 --- a/synapse/storage/schema/main/delta/30/as_users.py +++ b/synapse/storage/schema/main/delta/30/as_users.py @@ -63,8 +63,11 @@ def run_upgrade( if user_id in owned.keys(): logger.error( "user_id %s was owned by more than one application" - " service (IDs %s and %s); assigning arbitrarily to %s" - % (user_id, owned[user_id], appservice.id, owned[user_id]) + " service (IDs %s and %s); assigning arbitrarily to %s", + user_id, + owned[user_id], + appservice.id, + owned[user_id], ) owned.setdefault(appservice.id, []).append(user_id) diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py index 68336814c0..c790c9667f 100644 --- a/synapse/util/check_dependencies.py +++ b/synapse/util/check_dependencies.py @@ -37,6 +37,8 @@ DISTRIBUTION_NAME = "matrix-synapse" __all__ = ["check_requirements"] +logger = logging.getLogger(__name__) + class DependencyException(Exception): @property @@ -211,6 +213,6 @@ def check_requirements(extra: Optional[str] = None) -> None: if deps_unfulfilled: for err in errors: - logging.error(err) + logger.error(err) raise DependencyException(deps_unfulfilled) diff --git a/synapse/util/daemonize.py b/synapse/util/daemonize.py index 52549f53c4..9fdefc5a76 100644 --- a/synapse/util/daemonize.py +++ b/synapse/util/daemonize.py @@ -133,7 +133,7 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") - # write a log line on SIGTERM. def sigterm(signum: int, frame: Optional[FrameType]) -> NoReturn: - logger.warning("Caught signal %s. Stopping daemon." % signum) + logger.warning("Caught signal %s. Stopping daemon.", signum) sys.exit(0) signal.signal(signal.SIGTERM, sigterm) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 4683d09cd7..5169656c73 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -440,7 +440,8 @@ class TaskScheduler: except Exception: f = Failure() logger.error( - f"scheduled task {task.id} failed", + "scheduled task %s failed", + task.id, exc_info=(f.type, f.value, f.getTracebackObject()), ) status = TaskStatus.FAILED @@ -473,8 +474,10 @@ class TaskScheduler: self._clock.time_msec() > task.timestamp + TaskScheduler.LAST_UPDATE_BEFORE_WARNING_MS ): - logger.warn( - f"Task {task.id} (action {task.action}) has seen no update for more than 24h and may be stuck" + logger.warning( + "Task %s (action %s) has seen no update for more than 24h and may be stuck", + task.id, + task.action, ) if task.id in self._running_tasks: diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 42dc844734..a7b702fbd1 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -45,6 +45,8 @@ from synapse.util import Clock from tests import unittest from tests.unittest import override_config +logger = logging.getLogger(__name__) + class FederationServerTests(unittest.FederatingHomeserverTestCase): servlets = [ @@ -252,7 +254,7 @@ class MessageAcceptTests(unittest.FederatingHomeserverTestCase): class ServerACLsTestCase(unittest.TestCase): def test_blocked_server(self) -> None: e = _create_acl_event({"allow": ["*"], "deny": ["evil.com"]}) - logging.info("ACL event: %s", e.content) + logger.info("ACL event: %s", e.content) server_acl_evalutor = server_acl_evaluator_from_event(e) @@ -266,7 +268,7 @@ class ServerACLsTestCase(unittest.TestCase): def test_block_ip_literals(self) -> None: e = _create_acl_event({"allow_ip_literals": False, "allow": ["*"]}) - logging.info("ACL event: %s", e.content) + logger.info("ACL event: %s", e.content) server_acl_evalutor = server_acl_evaluator_from_event(e) diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 0691d3f99c..968d119a50 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -65,8 +65,10 @@ class WorkerLockTestCase(unittest.HomeserverTestCase): timeout_seconds = 15 # Increased timeout for RISC-V # add a print or log statement here for visibility in CI logs logger.info( # use logger.info - f"Detected RISC-V architecture ({current_machine}). " - f"Adjusting test_lock_contention: timeout={timeout_seconds}s" + "Detected RISC-V architecture (%s). " + "Adjusting test_lock_contention: timeout=%ss", + current_machine, + timeout_seconds, ) else: # Settings for other architectures diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 0fbb4db2f7..6b25e53c28 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -1822,7 +1822,7 @@ def _get_test_protocol_factory() -> IProtocolFactory: def _log_request(request: str) -> None: """Implements Factory.log, which is expected by Request.finish""" - logger.info(f"Completed request {request}") + logger.info("Completed request %s", request) @implementer(IPolicyForHTTPS) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index 80b0856a56..2ef8a95c45 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -893,4 +893,4 @@ def _get_test_protocol_factory() -> IProtocolFactory: def _log_request(request: str) -> None: """Implements Factory.log, which is expected by Request.finish""" - logger.info(f"Completed request {request}") + logger.info("Completed request %s", request) diff --git a/tests/logging/test_remote_handler.py b/tests/logging/test_remote_handler.py index f5412ac6e2..4178a8d831 100644 --- a/tests/logging/test_remote_handler.py +++ b/tests/logging/test_remote_handler.py @@ -86,11 +86,11 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send some debug messages for i in range(3): - logger.debug("debug %s" % (i,)) + logger.debug("debug %s", i) # Send a bunch of useful messages for i in range(7): - logger.info("info %s" % (i,)) + logger.info("info %s", i) # The last debug message pushes it past the maximum buffer logger.debug("too much debug") @@ -116,15 +116,15 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send some debug messages for i in range(3): - logger.debug("debug %s" % (i,)) + logger.debug("debug %s", i) # Send a bunch of useful messages for i in range(10): - logger.warning("warn %s" % (i,)) + logger.warning("warn %s", i) # Send a bunch of info messages for i in range(3): - logger.info("info %s" % (i,)) + logger.info("info %s", i) # The last debug message pushes it past the maximum buffer logger.debug("too much debug") @@ -152,7 +152,7 @@ class RemoteHandlerTestCase(LoggerCleanupMixin, TestCase): # Send a bunch of useful messages for i in range(20): - logger.warning("warn %s" % (i,)) + logger.warning("warn %s", i) # Allow the reconnection client, server = connect_logging_client(self.reactor, 0) From 1709957395d13b86db6df089824d3ea7b53f2a39 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Jun 2025 05:29:23 -0500 Subject: [PATCH 373/691] Fix bug where sliding sync ignored `room_id_to_include` option (#18535) This was correctly handled for the "fallback" case where the background updates hadn't finished --------- Co-authored-by: Eric Eastwood --- changelog.d/18535.bugfix | 1 + synapse/handlers/sliding_sync/room_lists.py | 50 +---- synapse/util/__init__.py | 83 +++++++- .../client/sliding_sync/test_sliding_sync.py | 52 +++++ tests/util/test_mutable_overlay_mapping.py | 190 ++++++++++++++++++ 5 files changed, 336 insertions(+), 40 deletions(-) create mode 100644 changelog.d/18535.bugfix create mode 100644 tests/util/test_mutable_overlay_mapping.py diff --git a/changelog.d/18535.bugfix b/changelog.d/18535.bugfix new file mode 100644 index 0000000000..246b1bc01f --- /dev/null +++ b/changelog.d/18535.bugfix @@ -0,0 +1 @@ +Fix long-standing bug where sliding sync did not honour the `room_id_to_include` config option. diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 13e69f18a0..e196199f8a 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -23,6 +23,7 @@ from typing import ( List, Literal, Mapping, + MutableMapping, Optional, Set, Tuple, @@ -73,6 +74,7 @@ from synapse.types.handlers.sliding_sync import ( SlidingSyncResult, ) from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping if TYPE_CHECKING: from synapse.server import HomeServer @@ -245,9 +247,11 @@ class SlidingSyncRoomLists: # Note: this won't include rooms the user has left themselves. We add back # `newly_left` rooms below. This is more efficient than fetching all rooms and # then filtering out the old left rooms. - room_membership_for_user_map = ( - await self.store.get_sliding_sync_rooms_for_user_from_membership_snapshots( - user_id + room_membership_for_user_map: MutableMapping[str, RoomsForUserSlidingSync] = ( + MutableOverlayMapping( + await self.store.get_sliding_sync_rooms_for_user_from_membership_snapshots( + user_id + ) ) ) # To play nice with the rewind logic below, we need to go fetch the rooms the @@ -268,26 +272,12 @@ class SlidingSyncRoomLists: ) ) if self_leave_room_membership_for_user_map: - # FIXME: It would be nice to avoid this copy but since - # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it - # can't return a mutable value like a `dict`. We make the copy to get a - # mutable dict that we can change. We try to only make a copy when necessary - # (if we actually need to change something) as in most cases, the logic - # doesn't need to run. - room_membership_for_user_map = dict(room_membership_for_user_map) room_membership_for_user_map.update(self_leave_room_membership_for_user_map) # Remove invites from ignored users ignored_users = await self.store.ignored_users(user_id) invite_config = await self.store.get_invite_config_for_user(user_id) if ignored_users: - # FIXME: It would be nice to avoid this copy but since - # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it - # can't return a mutable value like a `dict`. We make the copy to get a - # mutable dict that we can change. We try to only make a copy when necessary - # (if we actually need to change something) as in most cases, the logic - # doesn't need to run. - room_membership_for_user_map = dict(room_membership_for_user_map) # Make a copy so we don't run into an error: `dictionary changed size during # iteration`, when we remove items for room_id in list(room_membership_for_user_map.keys()): @@ -316,13 +306,6 @@ class SlidingSyncRoomLists: sync_config.user, room_membership_for_user_map, to_token=to_token ) if changes: - # FIXME: It would be nice to avoid this copy but since - # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it - # can't return a mutable value like a `dict`. We make the copy to get a - # mutable dict that we can change. We try to only make a copy when necessary - # (if we actually need to change something) as in most cases, the logic - # doesn't need to run. - room_membership_for_user_map = dict(room_membership_for_user_map) for room_id, change in changes.items(): if change is None: # Remove rooms that the user joined after the `to_token` @@ -364,13 +347,6 @@ class SlidingSyncRoomLists: newly_left_room_map.keys() - room_membership_for_user_map.keys() ) if missing_newly_left_rooms: - # FIXME: It would be nice to avoid this copy but since - # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it - # can't return a mutable value like a `dict`. We make the copy to get a - # mutable dict that we can change. We try to only make a copy when necessary - # (if we actually need to change something) as in most cases, the logic - # doesn't need to run. - room_membership_for_user_map = dict(room_membership_for_user_map) for room_id in missing_newly_left_rooms: newly_left_room_for_user = newly_left_room_map[room_id] # This should be a given @@ -461,6 +437,10 @@ class SlidingSyncRoomLists: else: room_membership_for_user_map.pop(room_id, None) + # Remove any rooms that we globally exclude from sync. + for room_id in self.rooms_to_exclude_globally: + room_membership_for_user_map.pop(room_id, None) + dm_room_ids = await self._get_dm_rooms_for_user(user_id) if sync_config.lists: @@ -577,14 +557,6 @@ class SlidingSyncRoomLists: if sync_config.room_subscriptions: with start_active_span("assemble_room_subscriptions"): - # FIXME: It would be nice to avoid this copy but since - # `get_sliding_sync_rooms_for_user_from_membership_snapshots` is cached, it - # can't return a mutable value like a `dict`. We make the copy to get a - # mutable dict that we can change. We try to only make a copy when necessary - # (if we actually need to change something) as in most cases, the logic - # doesn't need to run. - room_membership_for_user_map = dict(room_membership_for_user_map) - # Find which rooms are partially stated and may need to be filtered out # depending on the `required_state` requested (see below). partial_state_rooms = await self.store.get_partial_rooms() diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index e0d876e84b..17836d21b3 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -19,10 +19,22 @@ # # +import collections.abc import json import logging import typing -from typing import Any, Callable, Dict, Generator, Optional, Sequence +from typing import ( + Any, + Callable, + Dict, + Generator, + Iterator, + Mapping, + Optional, + Sequence, + Set, + TypeVar, +) import attr from immutabledict import immutabledict @@ -251,3 +263,72 @@ class ExceptionBundle(Exception): parts.append(str(e)) super().__init__("\n - ".join(parts)) self.exceptions = exceptions + + +K = TypeVar("K") +V = TypeVar("V") + + +@attr.s(slots=True, auto_attribs=True) +class MutableOverlayMapping(collections.abc.MutableMapping[K, V]): + """A mutable mapping that allows changes to a read-only underlying + mapping. Supports deletions. + + This is useful for cases where you want to allow modifications to a mapping + without changing or copying the original mapping. + + Note: the underlying mapping must not change while this proxy is in use. + """ + + _underlying_map: Mapping[K, V] + _mutable_map: Dict[K, V] = attr.ib(factory=dict) + _deletions: Set[K] = attr.ib(factory=set) + + def __getitem__(self, key: K) -> V: + if key in self._deletions: + raise KeyError(key) + if key in self._mutable_map: + return self._mutable_map[key] + return self._underlying_map[key] + + def __setitem__(self, key: K, value: V) -> None: + self._deletions.discard(key) + self._mutable_map[key] = value + + def __delitem__(self, key: K) -> None: + if key not in self: + raise KeyError(key) + + self._deletions.add(key) + self._mutable_map.pop(key, None) + + def __iter__(self) -> Iterator[K]: + for key in self._mutable_map: + if key not in self._deletions: + yield key + + for key in self._underlying_map: + if key not in self._deletions and key not in self._mutable_map: + # `key` should not be in both _mutable_map and _deletions + assert key not in self._mutable_map + yield key + + def __len__(self) -> int: + count = len(self._underlying_map) + for key in self._deletions: + if key in self._underlying_map: + count -= 1 + + for key in self._mutable_map: + # `key` should not be in both _mutable_map and _deletions + assert key not in self._deletions + + if key not in self._underlying_map: + count += 1 + + return count + + def clear(self) -> None: + self._underlying_map = {} + self._mutable_map.clear() + self._deletions.clear() diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index dcec5b4cf0..412dbcf77b 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -1616,3 +1616,55 @@ class SlidingSyncTestCase(SlidingSyncBase): {space_room_id, space_room_id2}, exact=True, ) + + def test_exclude_rooms_from_sync(self) -> None: + """Tests that sliding sync honours the `exclude_rooms_from_sync` config + option. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id_to_exclude = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + room_id_to_include = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # We cheekily modify the stored config here, as we can't add it to the + # raw config since we don't know the room ID before we start up. + self.hs.get_sliding_sync_handler().rooms_to_exclude_globally.append( + room_id_to_exclude + ) + self.hs.get_sliding_sync_handler().room_lists.rooms_to_exclude_globally.append( + room_id_to_exclude + ) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "required_state": [], + "timeline_limit": 0, + }, + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Make sure response only contains room_id_to_include + self.assertIncludes( + set(response_body["rooms"].keys()), + {room_id_to_include}, + exact=True, + ) + + # Test that the excluded room is not in the list ops + # Make sure the list is sorted in the way we expect + self.assertIncludes( + set(response_body["lists"]["foo-list"]["ops"][0]["room_ids"]), + {room_id_to_include}, + exact=True, + ) diff --git a/tests/util/test_mutable_overlay_mapping.py b/tests/util/test_mutable_overlay_mapping.py new file mode 100644 index 0000000000..a7335fca73 --- /dev/null +++ b/tests/util/test_mutable_overlay_mapping.py @@ -0,0 +1,190 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import unittest +from typing import Dict + +from synapse.util import MutableOverlayMapping + + +class TestMutableOverlayMapping(unittest.TestCase): + """Tests for the MutableOverlayMapping class.""" + + def test_init(self) -> None: + """Test initialization with different input types.""" + # Test with empty dict + empty_dict: Dict[str, int] = {} + mapping = MutableOverlayMapping(empty_dict) + self.assertEqual(len(mapping), 0) + + # Test with populated dict + populated_dict = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(populated_dict) + self.assertEqual(len(mapping), 3) + self.assertEqual(mapping["a"], 1) + + def test_get_item(self) -> None: + """Test getting items from the mapping.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Get from underlying map + self.assertEqual(mapping["a"], 1) + self.assertEqual(mapping["b"], 2) + + # Check KeyError for non-existent key + with self.assertRaises(KeyError): + mapping["d"] + + def test_set_item(self) -> None: + """Test setting items in the mapping.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Set new key + mapping["d"] = 4 + self.assertEqual(mapping["d"], 4) + + # Override existing key + mapping["a"] = 10 + self.assertEqual(mapping["a"], 10) + + # Original map should be unchanged + self.assertEqual(underlying["a"], 1) + self.assertNotIn("d", underlying) + + def test_del_item(self) -> None: + """Test deleting items from the mapping.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Delete a key + del mapping["a"] + with self.assertRaises(KeyError): + mapping["a"] + + # Original map should be unchanged + self.assertEqual(underlying["a"], 1) + + # Delete non-existent key + with self.assertRaises(KeyError): + del mapping["d"] + + def test_len(self) -> None: + """Test the len() function.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + self.assertEqual(len(mapping), 3) + + # Add a new key + mapping["d"] = 4 + self.assertEqual(len(mapping), 4) + + # Override an existing key + mapping["a"] = 10 + self.assertEqual(len(mapping), 4) + + # Delete a key + del mapping["b"] + self.assertEqual(len(mapping), 3) + + # Delete a key in mutable map + del mapping["d"] + self.assertEqual(len(mapping), 2) + + def test_iteration(self) -> None: + """Test iteration over the mapping.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Add a new key and override an existing one + mapping["d"] = 4 + mapping["a"] = 10 + + # Delete a key + del mapping["c"] + + iterated_keys = set() + for k in mapping: + iterated_keys.add(k) + + # Expected keys: a, b, d (c is deleted) + self.assertEqual(iterated_keys, {"a", "b", "d"}) + + iterated_items = dict(mapping.items()) + self.assertDictEqual(iterated_items, {"a": 10, "b": 2, "d": 4}) + + def test_clear(self) -> None: + """Test the clear method.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Add a new key and override an existing one + mapping["d"] = 4 + mapping["a"] = 10 + + # Clear the mapping + mapping.clear() + self.assertEqual(len(mapping), 0) + + # All keys should be gone + with self.assertRaises(KeyError): + mapping["a"] + + with self.assertRaises(KeyError): + mapping["d"] + + # Adding a new key after clearing + mapping["b"] = 2 + self.assertEqual(mapping["b"], 2) + self.assertEqual(len(mapping), 1) + + # The underlying map should remain unchanged + self.assertDictEqual(underlying, {"a": 1, "b": 2, "c": 3}) + + def test_dict_methods(self) -> None: + """Test standard dict methods.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Test keys, values, and items + self.assertEqual(set(mapping.keys()), {"a", "b", "c"}) + self.assertEqual(set(mapping.values()), {1, 2, 3}) + self.assertEqual(set(mapping.items()), {("a", 1), ("b", 2), ("c", 3)}) + + # Modify, then test again + mapping["d"] = 4 + mapping["a"] = 10 + del mapping["c"] + + self.assertEqual(set(mapping.keys()), {"a", "b", "d"}) + self.assertEqual(set(mapping.values()), {10, 2, 4}) + self.assertEqual(set(mapping.items()), {("a", 10), ("b", 2), ("d", 4)}) + + def test_key_presence(self) -> None: + """Test checking if keys exist in the mapping.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + mapping["d"] = 4 + mapping["a"] = 10 + del mapping["c"] + + # Test key presence + self.assertIn("a", mapping) + self.assertIn("b", mapping) + self.assertNotIn("c", mapping) + self.assertIn("d", mapping) + self.assertNotIn("e", mapping) From 6b1e3c9c666161a79059f130e6745b1fed351ebb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:34:38 +0100 Subject: [PATCH 374/691] Bump requests from 2.32.2 to 2.32.4 (#18533) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index cbed01b564..8e15c7eade 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2256,19 +2256,19 @@ rpds-py = ">=0.7.0" [[package]] name = "requests" -version = "2.32.2" +version = "2.32.4" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, + {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, ] [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" +charset_normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<3" From 3b94e40cc882ac2728fa8f053e45ac669e2d7fa2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:36:21 +0100 Subject: [PATCH 375/691] Fix typo of Math.pow, `^` -> `**` (#18543) --- changelog.d/18543.bugfix | 1 + synapse/app/phone_stats_home.py | 4 +--- synapse/handlers/worker_lock.py | 5 +++-- synapse/util/constants.py | 20 ++++++++++++++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 changelog.d/18543.bugfix create mode 100644 synapse/util/constants.py diff --git a/changelog.d/18543.bugfix b/changelog.d/18543.bugfix new file mode 100644 index 0000000000..3db7b04a56 --- /dev/null +++ b/changelog.d/18543.bugfix @@ -0,0 +1 @@ +Fix an issue where "Lock timeout is getting excessive" warnings would be logged even when the lock timeout was <10 minutes. \ No newline at end of file diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 9cd2e1b71e..7e8c7cf37e 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -28,15 +28,13 @@ from prometheus_client import Gauge from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.types import JsonDict +from synapse.util.constants import ONE_HOUR_SECONDS, ONE_MINUTE_SECONDS if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger("synapse.app.homeserver") -ONE_MINUTE_SECONDS = 60 -ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS - MILLISECONDS_PER_SECOND = 1000 INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index b36f7bd5fd..3077e9e463 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -44,6 +44,7 @@ from synapse.logging.opentracing import start_active_span from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore from synapse.util.async_helpers import timeout_deferred +from synapse.util.constants import ONE_MINUTE_SECONDS if TYPE_CHECKING: from synapse.logging.opentracing import opentracing @@ -272,7 +273,7 @@ class WaitingLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) - if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations logger.warning( "Lock timeout is getting excessive: %ss. There may be a deadlock.", self._retry_interval, @@ -352,7 +353,7 @@ class WaitingMultiLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) - if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations logger.warning( "Lock timeout is getting excessive: %ss. There may be a deadlock.", self._retry_interval, diff --git a/synapse/util/constants.py b/synapse/util/constants.py new file mode 100644 index 0000000000..9986017147 --- /dev/null +++ b/synapse/util/constants.py @@ -0,0 +1,20 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +# Time-based constants. +# +# Laying these out incrementally, even if only some are required, helps with +# readability and catching bugs. +ONE_MINUTE_SECONDS = 60 +ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS From db8a8d33fec3d4104de100a2360033ce9676d2d1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:56:39 +0100 Subject: [PATCH 376/691] bugfix: calculate the PL for non-creators correctly in v11 rooms (#18547) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18547.bugfix | 1 + synapse/state/v2.py | 2 +- tests/state/test_v2.py | 56 ++++++++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 changelog.d/18547.bugfix diff --git a/changelog.d/18547.bugfix b/changelog.d/18547.bugfix new file mode 100644 index 0000000000..ee1bfc8f27 --- /dev/null +++ b/changelog.d/18547.bugfix @@ -0,0 +1 @@ +Fix an issue where during state resolution for v11 rooms Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. diff --git a/synapse/state/v2.py b/synapse/state/v2.py index 11e80df425..69df9eb770 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -255,7 +255,7 @@ async def _get_power_level_for_sender( ) if aev and (aev.type, aev.state_key) == (EventTypes.Create, ""): creator = ( - event.sender + aev.sender if event.room_version.implicit_room_creator else aev.content.get("creator") ) diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 968c1ff64c..3e8f9a4358 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -961,25 +961,47 @@ class AuthChainDifferenceTestCase(unittest.TestCase): f"wrong pl for {user_id} on v{room_version.identifier}", ) - # the creator alone without PL is 100 - got_creator_pl = self.successResultOf( - defer.ensureDeferred( - _get_power_level_for_sender( - ROOM_ID, - member_event.event_id, - { - member_event.event_id: member_event, - create_event.event_id: create_event, - }, - store, + # the creator alone without PL is 100, everyone else is 0 + want_pls = { + ALICE: 100, + BOB: 0, + CHARLIE: 0, + } + for user_id, want_pl in want_pls.items(): + test_event = make_event_from_dict( + { + "room_id": ROOM_ID, + "sender": user_id, + "type": EventTypes.Topic, + "state_key": "", + "content": {"topic": "Test"}, + "auth_events": [ + create_event.event_id, + member_event.event_id, + pl_event.event_id, + ], + "prev_events": [pl_event.event_id], + }, + room_version, + ) + got_pl = self.successResultOf( + defer.ensureDeferred( + _get_power_level_for_sender( + ROOM_ID, + test_event.event_id, + { + test_event.event_id: test_event, + create_event.event_id: create_event, + }, + store, + ) ) ) - ) - self.assertEqual( - got_creator_pl, - 100, - f"wrong pl for creator with no PL event on v{room_version.identifier}", - ) + self.assertEqual( + got_pl, + want_pl, + f"wrong pl for {user_id} with no PL event on v{room_version.identifier}", + ) T = TypeVar("T") From f56670515bc402e13ee0bb2dd99ceb6e2ea8ba7d Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:32:24 +0100 Subject: [PATCH 377/691] bugfix: assert we always pass the create event to get_user_power_level (#18545) The create event is required if there is no PL event, in which case the creator gets PL100. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18545.bugfix | 1 + synapse/api/auth/base.py | 22 +++++++++------- synapse/event_auth.py | 33 ++++++++++++++---------- synapse/push/bulk_push_rule_evaluator.py | 23 ++++++++++++++++- synapse/rest/client/room.py | 17 ++++++------ synapse/state/__init__.py | 1 + 6 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 changelog.d/18545.bugfix diff --git a/changelog.d/18545.bugfix b/changelog.d/18545.bugfix new file mode 100644 index 0000000000..3ca6b4d624 --- /dev/null +++ b/changelog.d/18545.bugfix @@ -0,0 +1 @@ +Fix an issue where Synapse could calculate the wrong power level for the creator of the room if there was no power levels event. diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index fc1c0cc903..3126bc9f27 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -37,7 +37,9 @@ from synapse.appservice import ApplicationService from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace +from synapse.state import CREATE_KEY, POWER_KEY from synapse.types import Requester, create_requester +from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable if TYPE_CHECKING: @@ -216,18 +218,20 @@ class BaseAuth: # by checking if they would (theoretically) be able to change the # m.room.canonical_alias events - power_level_event = ( - await self._storage_controllers.state.get_current_state_event( - room_id, EventTypes.PowerLevels, "" - ) + auth_events = await self._storage_controllers.state.get_current_state( + room_id, + StateFilter.from_types( + [ + POWER_KEY, + CREATE_KEY, + ] + ), ) - auth_events = {} - if power_level_event: - auth_events[(EventTypes.PowerLevels, "")] = power_level_event - send_level = event_auth.get_send_level( - EventTypes.CanonicalAlias, "", power_level_event + EventTypes.CanonicalAlias, + "", + auth_events.get(POWER_KEY), ) user_level = event_auth.get_user_power_level( requester.user.to_string(), auth_events diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 5999c264dc..35d02c8294 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -64,6 +64,7 @@ from synapse.api.room_versions import ( RoomVersion, RoomVersions, ) +from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( MutableStateMap, @@ -308,6 +309,13 @@ def check_state_dependent_auth_rules( auth_dict = {(e.type, e.state_key): e for e in auth_events} + # Later code relies on there being a create event e.g _can_federate, _is_membership_change_allowed + # so produce a more intelligible error if we don't have one. + if auth_dict.get(CREATE_KEY) is None: + raise AuthError( + 403, f"Event {event.event_id} is missing a create event in auth_events." + ) + # additional check for m.federate creating_domain = get_domain_from_id(event.room_id) originating_domain = get_domain_from_id(event.sender) @@ -1010,11 +1018,16 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in user_id: user's id to look up in power_levels auth_events: state in force at this point in the room (or rather, a subset of - it including at least the create event and power levels event. + it including at least the create event, and possibly a power levels event). Returns: the user's power level in this room. """ + create_event = auth_events.get(CREATE_KEY) + assert create_event is not None, ( + "A create event in the auth events chain is required to calculate user power level correctly," + " but was not found. This indicates a bug" + ) power_level_event = get_power_level_event(auth_events) if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) @@ -1028,18 +1041,12 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in else: # if there is no power levels event, the creator gets 100 and everyone # else gets 0. - - # some things which call this don't pass the create event: hack around - # that. - key = (EventTypes.Create, "") - create_event = auth_events.get(key) - if create_event is not None: - if create_event.room_version.implicit_room_creator: - creator = create_event.sender - else: - creator = create_event.content[EventContentFields.ROOM_CREATOR] - if creator == user_id: - return 100 + if create_event.room_version.implicit_room_creator: + creator = create_event.sender + else: + creator = create_event.content[EventContentFields.ROOM_CREATOR] + if creator == user_id: + return 100 return 0 diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 8249d5e84f..1f4f5b90c3 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -50,7 +50,7 @@ from synapse.event_auth import auth_types_for_event, get_user_power_level from synapse.events import EventBase, relation_from_event from synapse.events.snapshot import EventContext from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.state import POWER_KEY +from synapse.state import CREATE_KEY, POWER_KEY from synapse.storage.databases.main.roommember import EventIdMembership from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo @@ -246,6 +246,7 @@ class BulkPushRuleEvaluator: StateFilter.from_types(event_types) ) pl_event_id = prev_state_ids.get(POWER_KEY) + create_event_id = prev_state_ids.get(CREATE_KEY) # fastpath: if there's a power level event, that's all we need, and # not having a power level event is an extreme edge case @@ -268,6 +269,26 @@ class BulkPushRuleEvaluator: if auth_event: auth_events_dict[auth_event_id] = auth_event auth_events = {(e.type, e.state_key): e for e in auth_events_dict.values()} + if auth_events.get(CREATE_KEY) is None: + # if the event being checked is the create event, use its own permissions + if event.type == EventTypes.Create and event.get_state_key() == "": + auth_events[CREATE_KEY] = event + else: + auth_events[ + CREATE_KEY + ] = await self.store.get_create_event_for_room(event.room_id) + + # if we are evaluating the create event, then use itself to determine power levels. + if event.type == EventTypes.Create and event.get_state_key() == "": + auth_events[CREATE_KEY] = event + else: + # if we aren't processing the create event, create_event_id should always be set + assert create_event_id is not None + create_event = event_id_to_event.get(create_event_id) + if create_event: + auth_events[CREATE_KEY] = create_event + else: + auth_events[CREATE_KEY] = await self.store.get_event(create_event_id) sender_level = get_user_power_level(event.sender, auth_events) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index bb41575d46..c64ce1f9c7 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -64,6 +64,7 @@ from synapse.logging.opentracing import set_tag from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client._base import client_patterns from synapse.rest.client.transactions import HttpTransactionCache +from synapse.state import CREATE_KEY, POWER_KEY from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID, UserID from synapse.types.state import StateFilter @@ -924,16 +925,16 @@ class RoomEventServlet(RestServlet): if include_unredacted_content and not await self.auth.is_server_admin( requester ): - power_level_event = ( - await self._storage_controllers.state.get_current_state_event( - room_id, EventTypes.PowerLevels, "" - ) + auth_events = await self._storage_controllers.state.get_current_state( + room_id, + StateFilter.from_types( + [ + POWER_KEY, + CREATE_KEY, + ] + ), ) - auth_events = {} - if power_level_event: - auth_events[(EventTypes.PowerLevels, "")] = power_level_event - redact_level = event_auth.get_named_level(auth_events, "redact", 50) user_level = event_auth.get_user_power_level( requester.user.to_string(), auth_events diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 9e48e09270..1c3e5d00a9 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -83,6 +83,7 @@ EVICTION_TIMEOUT_SECONDS = 60 * 60 _NEXT_STATE_ID = 1 +CREATE_KEY = (EventTypes.Create, "") POWER_KEY = (EventTypes.PowerLevels, "") From df04931f0b1a2fede5e7f2ba2489254f75e83333 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 16:33:51 +0100 Subject: [PATCH 378/691] Bump base64 from 0.21.7 to 0.22.1 (#18559) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa68713ccb..97dde00a49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,12 +29,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -147,7 +141,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "headers-core", "http", @@ -517,7 +511,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.7", + "base64", "blake2", "bytes", "headers", From f500c7d982b8d00773a8652affa75800906a570e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Jun 2025 10:41:35 -0500 Subject: [PATCH 379/691] Speed up MAS token introspection (#18357) We do this by shoving it into Rust. We believe our python http client is a bit slow. Also bumps minimum rust version to 1.81.0, released last September (over six months ago) To allow for async Rust, includes some adapters between Tokio in Rust and the Twisted reactor in Python. --- .github/workflows/tests.yml | 22 +- Cargo.lock | 1333 ++++++++++++++++++++++- changelog.d/18357.misc | 1 + rust/Cargo.toml | 12 +- rust/src/errors.rs | 12 + rust/src/http_client.rs | 218 ++++ rust/src/lib.rs | 2 + synapse/api/auth/msc3861_delegated.py | 56 +- synapse/logging/opentracing.py | 7 + synapse/synapse_rust/http_client.pyi | 24 + tests/handlers/test_oauth_delegation.py | 442 ++++---- 11 files changed, 1828 insertions(+), 301 deletions(-) create mode 100644 changelog.d/18357.misc create mode 100644 rust/src/http_client.rs create mode 100644 synapse/synapse_rust/http_client.pyi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 848240f68e..f9156e81ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,7 +85,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -149,7 +149,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Setup Poetry @@ -210,7 +210,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -227,7 +227,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@0d72692bcfbf448b1e2afa01a67f71b455a9dcec # 1.86.0 with: components: clippy - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 @@ -247,7 +247,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: - toolchain: nightly-2022-12-01 + toolchain: nightly-2025-04-23 components: clippy - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 @@ -265,7 +265,7 @@ jobs: uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: # We use nightly so that it correctly groups together imports - toolchain: nightly-2022-12-01 + toolchain: nightly-2025-04-23 components: rustfmt - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 @@ -362,7 +362,7 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -404,7 +404,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 # There aren't wheels for some of the older deps, so we need to install @@ -519,7 +519,7 @@ jobs: run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Run SyTest @@ -663,7 +663,7 @@ jobs: path: synapse - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - name: Prepare Complement's Prerequisites @@ -695,7 +695,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0 + uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 - run: cargo test diff --git a/Cargo.lock b/Cargo.lock index 97dde00a49..71e67bb52a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -23,12 +38,39 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64" version = "0.22.1" @@ -71,12 +113,43 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -107,12 +180,127 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -123,6 +311,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.3.1" @@ -131,17 +330,48 @@ checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", - "wasi", - "windows-targets", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "headers" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "headers-core", "http", @@ -182,18 +412,260 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "httpdate" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + [[package]] name = "itoa" version = "1.0.11" @@ -202,10 +674,11 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -217,9 +690,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "log" @@ -248,12 +727,65 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "portable-atomic" version = "1.6.0" @@ -360,6 +892,55 @@ dependencies = [ "serde", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -369,17 +950,38 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.0", "zerocopy", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -387,7 +989,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.0", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", ] [[package]] @@ -396,7 +1007,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ - "getrandom", + "getrandom 0.3.1", "zerocopy", ] @@ -429,12 +1040,174 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "windows-registry", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustls" +version = "0.23.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "rustls-webpki" +version = "0.103.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.219" @@ -467,6 +1240,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -489,6 +1274,43 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "socket2" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "subtle" version = "2.5.0" @@ -511,12 +1333,14 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "blake2", "bytes", + "futures", "headers", "hex", "http", + "http-body-util", "lazy_static", "log", "mime", @@ -524,18 +1348,175 @@ dependencies = [ "pyo3-log", "pythonize", "regex", + "reqwest", "serde", "serde_json", "sha2", + "tokio", "ulid", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "target-lexicon" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +[[package]] +name = "thiserror" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -548,7 +1529,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand", + "rand 0.9.0", "web-time", ] @@ -564,12 +1545,56 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasi" version = "0.13.3+wasi-0.2.2" @@ -581,23 +1606,24 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -605,10 +1631,23 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" +name = "wasm-bindgen-futures" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -616,9 +1655,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -629,9 +1668,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "web-time" @@ -643,20 +1708,89 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -665,48 +1799,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "wit-bindgen-rt" version = "0.33.0" @@ -716,6 +1898,42 @@ dependencies = [ "bitflags", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.17" @@ -735,3 +1953,52 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/changelog.d/18357.misc b/changelog.d/18357.misc new file mode 100644 index 0000000000..8e4ee71356 --- /dev/null +++ b/changelog.d/18357.misc @@ -0,0 +1 @@ +Increase performance of introspecting access tokens when using delegated auth. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 840988e74e..446db1bcf1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,7 +7,7 @@ name = "synapse" version = "0.1.0" edition = "2021" -rust-version = "1.66.0" +rust-version = "1.81.0" [lib] name = "synapse" @@ -36,13 +36,21 @@ pyo3 = { version = "0.24.2", features = [ "abi3", "abi3-py39", ] } -pyo3-log = "0.12.0" +pyo3-log = "0.12.3" pythonize = "0.24.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } serde_json = "1.0.85" ulid = "1.1.2" +reqwest = { version = "0.12.15", default-features = false, features = [ + "http2", + "stream", + "rustls-tls-native-roots", +] } +http-body-util = "0.1.3" +futures = "0.3.31" +tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/errors.rs b/rust/src/errors.rs index 4e580e3e8c..149019ff4b 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -58,3 +58,15 @@ impl NotFoundError { NotFoundError::new_err(()) } } + +import_exception!(synapse.api.errors, HttpResponseException); + +impl HttpResponseException { + pub fn new(status: StatusCode, bytes: Vec) -> pyo3::PyErr { + HttpResponseException::new_err(( + status.as_u16(), + status.canonical_reason().unwrap_or_default(), + bytes, + )) + } +} diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs new file mode 100644 index 0000000000..eda0197c74 --- /dev/null +++ b/rust/src/http_client.rs @@ -0,0 +1,218 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2025 New Vector, Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +use std::{collections::HashMap, future::Future, panic::AssertUnwindSafe, sync::LazyLock}; + +use anyhow::Context; +use futures::{FutureExt, TryStreamExt}; +use pyo3::{exceptions::PyException, prelude::*, types::PyString}; +use reqwest::RequestBuilder; +use tokio::runtime::Runtime; + +use crate::errors::HttpResponseException; + +/// The tokio runtime that we're using to run async Rust libs. +static RUNTIME: LazyLock = LazyLock::new(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap() +}); + +/// A reference to the `Deferred` python class. +static DEFERRED_CLASS: LazyLock = LazyLock::new(|| { + Python::with_gil(|py| { + py.import("twisted.internet.defer") + .expect("module 'twisted.internet.defer' should be importable") + .getattr("Deferred") + .expect("module 'twisted.internet.defer' should have a 'Deferred' class") + .unbind() + }) +}); + +/// A reference to the twisted `reactor`. +static TWISTED_REACTOR: LazyLock> = LazyLock::new(|| { + Python::with_gil(|py| { + py.import("twisted.internet.reactor") + .expect("module 'twisted.internet.reactor' should be importable") + .unbind() + }) +}); + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; + child_module.add_class::()?; + + // Make sure we fail early if we can't build the lazy statics. + LazyLock::force(&RUNTIME); + LazyLock::force(&DEFERRED_CLASS); + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import acl` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.http_client", child_module)?; + + Ok(()) +} + +#[pyclass] +#[derive(Clone)] +struct HttpClient { + client: reqwest::Client, +} + +#[pymethods] +impl HttpClient { + #[new] + pub fn py_new(user_agent: &str) -> PyResult { + // The twisted reactor can only be imported after Synapse has been + // imported, to allow Synapse to change the twisted reactor. If we try + // and import the reactor too early twisted installs a default reactor, + // which can't be replaced. + LazyLock::force(&TWISTED_REACTOR); + + Ok(HttpClient { + client: reqwest::Client::builder() + .user_agent(user_agent) + .build() + .context("building reqwest client")?, + }) + } + + pub fn get<'a>( + &self, + py: Python<'a>, + url: String, + response_limit: usize, + ) -> PyResult> { + self.send_request(py, self.client.get(url), response_limit) + } + + pub fn post<'a>( + &self, + py: Python<'a>, + url: String, + response_limit: usize, + headers: HashMap, + request_body: String, + ) -> PyResult> { + let mut builder = self.client.post(url); + for (name, value) in headers { + builder = builder.header(name, value); + } + builder = builder.body(request_body); + + self.send_request(py, builder, response_limit) + } +} + +impl HttpClient { + fn send_request<'a>( + &self, + py: Python<'a>, + builder: RequestBuilder, + response_limit: usize, + ) -> PyResult> { + create_deferred(py, async move { + let response = builder.send().await.context("sending request")?; + + let status = response.status(); + + let mut stream = response.bytes_stream(); + let mut buffer = Vec::new(); + while let Some(chunk) = stream.try_next().await.context("reading body")? { + if buffer.len() + chunk.len() > response_limit { + Err(anyhow::anyhow!("Response size too large"))?; + } + + buffer.extend_from_slice(&chunk); + } + + if !status.is_success() { + return Err(HttpResponseException::new(status, buffer)); + } + + let r = Python::with_gil(|py| buffer.into_pyobject(py).map(|o| o.unbind()))?; + + Ok(r) + }) + } +} + +/// Creates a twisted deferred from the given future, spawning the task on the +/// tokio runtime. +/// +/// Does not handle deferred cancellation or contextvars. +fn create_deferred(py: Python, fut: F) -> PyResult> +where + F: Future> + Send + 'static, + for<'a> O: IntoPyObject<'a>, +{ + let deferred = DEFERRED_CLASS.bind(py).call0()?; + let deferred_callback = deferred.getattr("callback")?.unbind(); + let deferred_errback = deferred.getattr("errback")?.unbind(); + + RUNTIME.spawn(async move { + // TODO: Is it safe to assert unwind safety here? I think so, as we + // don't use anything that could be tainted by the panic afterwards. + // Note that `.spawn(..)` asserts unwind safety on the future too. + let res = AssertUnwindSafe(fut).catch_unwind().await; + + Python::with_gil(move |py| { + // Flatten the panic into standard python error + let res = match res { + Ok(r) => r, + Err(panic_err) => { + let panic_message = get_panic_message(&panic_err); + Err(PyException::new_err( + PyString::new(py, panic_message).unbind(), + )) + } + }; + + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` + match res { + Ok(obj) => { + TWISTED_REACTOR + .call_method(py, "callFromThread", (deferred_callback, obj), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + Err(err) => { + TWISTED_REACTOR + .call_method(py, "callFromThread", (deferred_errback, err), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + } + }); + }); + + Ok(deferred) +} + +/// Try and get the panic message out of the panic +fn get_panic_message<'a>(panic_err: &'a (dyn std::any::Any + Send + 'static)) -> &'a str { + // Apparently this is how you extract the panic message from a panic + if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index d751889874..e33a8cc44c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -8,6 +8,7 @@ pub mod acl; pub mod errors; pub mod events; pub mod http; +pub mod http_client; pub mod identifier; pub mod matrix_const; pub mod push; @@ -50,6 +51,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; + http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; Ok(()) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index ad5d59eef1..0cfdf15d60 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -30,9 +30,6 @@ from authlib.oauth2.rfc7662 import IntrospectionToken from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url from prometheus_client import Histogram -from twisted.web.client import readBody -from twisted.web.http_headers import Headers - from synapse.api.auth.base import BaseAuth from synapse.api.errors import ( AuthError, @@ -43,8 +40,14 @@ from synapse.api.errors import ( UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import make_deferred_yieldable -from synapse.logging.opentracing import active_span, force_tracing, start_active_span +from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import ( + active_span, + force_tracing, + inject_request_headers, + start_active_span, +) +from synapse.synapse_rust.http_client import HttpClient from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall @@ -179,6 +182,10 @@ class MSC3861DelegatedAuth(BaseAuth): self._admin_token: Callable[[], Optional[str]] = self._config.admin_token self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users + self._rust_http_client = HttpClient( + user_agent=self._http_client.user_agent.decode("utf8") + ) + # # Token Introspection Cache # This remembers what users/devices are represented by which access tokens, # in order to reduce overall system load: @@ -301,7 +308,6 @@ class MSC3861DelegatedAuth(BaseAuth): introspection_endpoint = await self._introspection_endpoint() raw_headers: Dict[str, str] = { "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": str(self._http_client.user_agent, "utf-8"), "Accept": "application/json", # Tell MAS that we support reading the device ID as an explicit # value, not encoded in the scope. This is supported by MAS 0.15+ @@ -315,38 +321,34 @@ class MSC3861DelegatedAuth(BaseAuth): uri, raw_headers, body = self._client_auth.prepare( method="POST", uri=introspection_endpoint, headers=raw_headers, body=body ) - headers = Headers({k: [v] for (k, v) in raw_headers.items()}) # Do the actual request - # We're not using the SimpleHttpClient util methods as we don't want to - # check the HTTP status code, and we do the body encoding ourselves. + logger.debug("Fetching token from MAS") start_time = self._clock.time() try: - response = await self._http_client.request( - method="POST", - uri=uri, - data=body.encode("utf-8"), - headers=headers, - ) - - resp_body = await make_deferred_yieldable(readBody(response)) + with start_active_span("mas-introspect-token"): + inject_request_headers(raw_headers) + with PreserveLoggingContext(): + resp_body = await self._rust_http_client.post( + url=uri, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) + except HttpResponseException as e: + end_time = self._clock.time() + introspection_response_timer.labels(e.code).observe(end_time - start_time) + raise except Exception: end_time = self._clock.time() introspection_response_timer.labels("ERR").observe(end_time - start_time) raise - end_time = self._clock.time() - introspection_response_timer.labels(response.code).observe( - end_time - start_time - ) + logger.debug("Fetched token from MAS") - if response.code < 200 or response.code >= 300: - raise HttpResponseException( - response.code, - response.phrase.decode("ascii", errors="replace"), - resp_body, - ) + end_time = self._clock.time() + introspection_response_timer.labels(200).observe(end_time - start_time) resp = json_decoder.decode(resp_body.decode("utf-8")) diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index d976e58e49..46fdd8ab62 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -796,6 +796,13 @@ def inject_response_headers(response_headers: Headers) -> None: response_headers.addRawHeader("Synapse-Trace-Id", f"{trace_id:x}") +@ensure_active_span("inject the span into a header dict") +def inject_request_headers(headers: Dict[str, str]) -> None: + span = opentracing.tracer.active_span + assert span is not None + opentracing.tracer.inject(span.context, opentracing.Format.HTTP_HEADERS, headers) + + @ensure_active_span( "get the active span context as a dict", ret=cast(Dict[str, str], {}) ) diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi new file mode 100644 index 0000000000..5fa6226fd5 --- /dev/null +++ b/synapse/synapse_rust/http_client.pyi @@ -0,0 +1,24 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Awaitable, Mapping + +class HttpClient: + def __init__(self, user_agent: str) -> None: ... + def get(self, url: str, response_limit: int) -> Awaitable[bytes]: ... + def post( + self, + url: str, + response_limit: int, + headers: Mapping[str, str], + request_body: str, + ) -> Awaitable[bytes]: ... diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 934bfee0bc..fefa2f1135 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -19,9 +19,10 @@ # # +import json from http import HTTPStatus from io import BytesIO -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Union from unittest.mock import ANY, AsyncMock, Mock from urllib.parse import parse_qs @@ -33,12 +34,11 @@ from signedjson.key import ( from signedjson.sign import sign_json from twisted.test.proto_helpers import MemoryReactor -from twisted.web.http_headers import Headers -from twisted.web.iweb import IResponse from synapse.api.errors import ( AuthError, Codes, + HttpResponseException, InvalidClientTokenError, OAuthInsufficientScopeError, SynapseError, @@ -52,7 +52,7 @@ from synapse.types import JsonDict, UserID from synapse.util import Clock from tests.server import FakeChannel -from tests.test_utils import FakeResponse, get_awaitable_result +from tests.test_utils import get_awaitable_result from tests.unittest import HomeserverTestCase, override_config, skip_unless from tests.utils import HAS_AUTHLIB, checked_cast, mock_getRawHeaders @@ -145,6 +145,9 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.auth = checked_cast(MSC3861DelegatedAuth, hs.get_auth()) + self._rust_client = Mock(spec=["post"]) + self.auth._rust_http_client = self._rust_client + return hs def prepare( @@ -157,9 +160,15 @@ class MSC3861OAuthDelegation(HomeserverTestCase): store.store_device(USER_ID, DEVICE, initial_device_display_name=None) ) + def _set_introspection_returnvalue(self, response_value: Any) -> AsyncMock: + self._rust_client.post = mock = AsyncMock( + return_value=json.dumps(response_value).encode("utf-8") + ) + return mock + def _assertParams(self) -> None: """Assert that the request parameters are correct.""" - params = parse_qs(self.http_client.request.call_args[1]["data"].decode("utf-8")) + params = parse_qs(self._rust_client.post.call_args[1]["request_body"]) self.assertEqual(params["token"], ["mockAccessToken"]) self.assertEqual(params["client_id"], [CLIENT_ID]) self.assertEqual(params["client_secret"], [CLIENT_SECRET]) @@ -167,128 +176,125 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_inactive_token(self) -> None: """The handler should return a 403 where the token is inactive.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": False}, - ) - ) + self._set_introspection_returnvalue({"active": False}) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_no_scope(self) -> None: """The handler should return a 403 where no scope is given.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": True}, - ) - ) + self._set_introspection_returnvalue({"active": True}) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_user_no_subject(self) -> None: """The handler should return a 500 when no subject is present.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={"active": True, "scope": " ".join([MATRIX_USER_SCOPE])}, - ) + self._set_introspection_returnvalue( + {"active": True, "scope": " ".join([MATRIX_USER_SCOPE])} ) + request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_no_user_scope(self) -> None: """The handler should return a 500 when no subject is present.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_DEVICE_SCOPE]), - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_DEVICE_SCOPE]), + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_admin_not_user(self) -> None: """The handler should raise when the scope has admin right but not user.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() def test_active_admin(self) -> None: """The handler should return a requester with admin rights.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -301,26 +307,26 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_admin_highest_privilege(self) -> None: """The handler should resolve to the most permissive scope.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE, MATRIX_GUEST_SCOPE] - ), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [SYNAPSE_ADMIN_SCOPE, MATRIX_USER_SCOPE, MATRIX_GUEST_SCOPE] + ), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -333,24 +339,24 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_user(self) -> None: """The handler should return a requester with normal user rights.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -363,24 +369,24 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_user_with_device(self) -> None: """The handler should return a requester with normal user rights and a device ID.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -393,32 +399,32 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_user_with_device_explicit_device_id(self) -> None: """The handler should return a requester with normal user rights and a device ID, given explicitly, as supported by MAS 0.15+""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE]), - "device_id": DEVICE, - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE]), + "device_id": DEVICE, + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) # It should have called with the 'X-MAS-Supports-Device-Id: 1' header self.assertEqual( - self.http_client.request.call_args[1]["headers"].getRawHeaders( - b"X-MAS-Supports-Device-Id", + self._rust_client.post.call_args[1]["headers"].get( + "X-MAS-Supports-Device-Id", ), - [b"1"], + "1", ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -431,22 +437,19 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_multiple_devices(self) -> None: """The handler should raise an error if multiple devices are found in the scope.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - MATRIX_USER_SCOPE, - f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", - f"{MATRIX_DEVICE_SCOPE_PREFIX}DDEEFF", - ] - ), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [ + MATRIX_USER_SCOPE, + f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", + f"{MATRIX_DEVICE_SCOPE_PREFIX}DDEEFF", + ] + ), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] @@ -456,16 +459,13 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_guest_not_allowed(self) -> None: """The handler should return an insufficient scope error.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] @@ -474,8 +474,11 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.auth.get_user_by_req(request), OAuthInsufficientScopeError ) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual( @@ -486,16 +489,13 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_active_guest_allowed(self) -> None: """The handler should return a requester with guest user rights and a device ID.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_GUEST_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + } ) request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] @@ -504,8 +504,11 @@ class MSC3861OAuthDelegation(HomeserverTestCase): self.auth.get_user_by_req(request, allow_guest=True) ) self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self.http_client.request.assert_called_once_with( - method="POST", uri=INTROSPECTION_ENDPOINT, data=ANY, headers=ANY + self._rust_client.post.assert_called_once_with( + url=INTROSPECTION_ENDPOINT, + response_limit=ANY, + request_body=ANY, + headers=ANY, ) self._assertParams() self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) @@ -522,30 +525,28 @@ class MSC3861OAuthDelegation(HomeserverTestCase): request.requestHeaders.getRawHeaders = mock_getRawHeaders() # The introspection endpoint is returning an error. - self.http_client.request = AsyncMock( - return_value=FakeResponse(code=500, body=b"Internal Server Error") - ) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint request fails. - self.http_client.request = AsyncMock(side_effect=Exception()) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return a JSON object. - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, payload=["this is an array", "not an object"] + self._rust_client.post = AsyncMock( + side_effect=HttpResponseException( + code=500, msg="Internal Server Error", response=b"{}" ) ) error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) self.assertEqual(error.value.code, 503) + # The introspection endpoint request fails. + self._rust_client.post = AsyncMock(side_effect=Exception()) + error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) + self.assertEqual(error.value.code, 503) + + # The introspection endpoint does not return a JSON object. + self._set_introspection_returnvalue(["this is an array", "not an object"]) + + error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) + self.assertEqual(error.value.code, 503) + # The introspection endpoint does not return valid JSON. - self.http_client.request = AsyncMock( - return_value=FakeResponse(code=200, body=b"this is not valid JSON") - ) + self._set_introspection_returnvalue("this is not valid JSON") + error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) self.assertEqual(error.value.code, 503) @@ -554,23 +555,21 @@ class MSC3861OAuthDelegation(HomeserverTestCase): an expiry time, the introspection response is cached and then the entry is re-requested after it has expired.""" - self.http_client.request = introspection_mock = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - MATRIX_USER_SCOPE, - f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", - ] - ), - "username": USERNAME, - "expires_in": 60, - }, - ) + introspection_mock = self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join( + [ + MATRIX_USER_SCOPE, + f"{MATRIX_DEVICE_SCOPE_PREFIX}AABBCC", + ] + ), + "username": USERNAME, + "expires_in": 60, + } ) + request = Mock(args={}) request.args[b"access_token"] = [b"mockAccessToken"] request.requestHeaders.getRawHeaders = mock_getRawHeaders() @@ -607,16 +606,13 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_cross_signing(self) -> None: """Try uploading device keys with OAuth delegation enabled.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + } ) keys_upload_body = self.make_device_keys(USER_ID, DEVICE) channel = self.make_request( @@ -778,16 +774,13 @@ class MSC3861OAuthDelegation(HomeserverTestCase): # Because we still support those endpoints with ASes, it checks the # access token before returning 404 - self.http_client.request = AsyncMock( - return_value=FakeResponse.json( - code=200, - payload={ - "active": True, - "sub": SUBJECT, - "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), - "username": USERNAME, - }, - ) + self._set_introspection_returnvalue( + { + "active": True, + "sub": SUBJECT, + "scope": " ".join([MATRIX_USER_SCOPE, MATRIX_DEVICE_SCOPE]), + "username": USERNAME, + }, ) self.expect_unrecognized("POST", "/_matrix/client/v3/delete_devices", auth=True) @@ -820,9 +813,7 @@ class MSC3861OAuthDelegation(HomeserverTestCase): def test_admin_token(self) -> None: """The handler should return a requester with admin rights when admin_token is used.""" - self.http_client.request = AsyncMock( - return_value=FakeResponse.json(code=200, payload={"active": False}), - ) + self._set_introspection_returnvalue({"active": False}) request = Mock(args={}) request.args[b"access_token"] = [b"admin_token_value"] @@ -839,7 +830,7 @@ class MSC3861OAuthDelegation(HomeserverTestCase): ) # There should be no call to the introspection endpoint - self.http_client.request.assert_not_called() + self._rust_client.post.assert_not_called() @override_config({"mau_stats_only": True}) def test_request_tracking(self) -> None: @@ -852,28 +843,23 @@ class MSC3861OAuthDelegation(HomeserverTestCase): known_token = "token-token-GOOD-:)" async def mock_http_client_request( - method: str, - uri: str, - data: Optional[bytes] = None, - headers: Optional[Headers] = None, - ) -> IResponse: + url: str, request_body: str, **kwargs: Any + ) -> bytes: """Mocked auth provider response.""" - assert method == "POST" - token = parse_qs(data)[b"token"][0].decode("utf-8") + token = parse_qs(request_body)["token"][0] if token == known_token: - return FakeResponse.json( - code=200, - payload={ + return json.dumps( + { "active": True, "scope": MATRIX_USER_SCOPE, "sub": SUBJECT, "username": USERNAME, }, - ) + ).encode("utf-8") - return FakeResponse.json(code=200, payload={"active": False}) + return json.dumps({"active": False}).encode("utf-8") - self.http_client.request = mock_http_client_request + self._rust_client.post = mock_http_client_request EXAMPLE_IPV4_ADDR = "123.123.123.123" EXAMPLE_USER_AGENT = "httprettygood" From a3b80071cd1d404d918fac006e1acf29062f6bad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Jun 2025 04:57:34 -0500 Subject: [PATCH 380/691] Always run schema workflow on develop (#18551) ... and release branches, so that we catch any problems that slip trough PR review. --- .github/workflows/schema.yaml | 3 +++ changelog.d/18551.misc | 1 + 2 files changed, 4 insertions(+) create mode 100644 changelog.d/18551.misc diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 1e13c1fef0..e815da1560 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -5,6 +5,9 @@ on: paths: - schema/** - docs/usage/configuration/config_documentation.md + push: + branches: ["develop", "release-*"] + workflow_dispatch: jobs: validate-schema: diff --git a/changelog.d/18551.misc b/changelog.d/18551.misc new file mode 100644 index 0000000000..9d60a59adc --- /dev/null +++ b/changelog.d/18551.misc @@ -0,0 +1 @@ +Run config schema CI on develop and release branches. From 3e571561c981f71373afa2fd70e7f62d8020e2eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Jun 2025 05:01:32 -0500 Subject: [PATCH 381/691] Fix Cargo.lock after bad merge (#18561) Broke in #18357 --- Cargo.lock | 6 +++--- changelog.d/18561.misc | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18561.misc diff --git a/Cargo.lock b/Cargo.lock index 71e67bb52a..ec3a5ca24e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,9 +67,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.22.1" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -371,7 +371,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "headers-core", "http", diff --git a/changelog.d/18561.misc b/changelog.d/18561.misc new file mode 100644 index 0000000000..8e4ee71356 --- /dev/null +++ b/changelog.d/18561.misc @@ -0,0 +1 @@ +Increase performance of introspecting access tokens when using delegated auth. From d1139ebfc11b13417aaf9acb289d7a0d0d93482b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Jun 2025 13:16:57 +0100 Subject: [PATCH 382/691] 1.132.0 --- CHANGES.md | 9 +++++++++ changelog.d/18522.doc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18522.doc diff --git a/CHANGES.md b/CHANGES.md index c7d6c1f129..0c736bfd95 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.132.0 (2025-06-17) + +### Improved Documentation + +- Improvements to generate config documentation from JSON Schema file. ([\#18522](https://github.com/element-hq/synapse/issues/18522)) + + + + # Synapse 1.132.0rc1 (2025-06-10) ### Features diff --git a/changelog.d/18522.doc b/changelog.d/18522.doc deleted file mode 100644 index 83c03190dd..0000000000 --- a/changelog.d/18522.doc +++ /dev/null @@ -1 +0,0 @@ -Generate config documentation from JSON Schema file. diff --git a/debian/changelog b/debian/changelog index e0d7ea20fd..6afa33ad74 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.132.0) stable; urgency=medium + + * New Synapse release 1.132.0. + + -- Synapse Packaging team Tue, 17 Jun 2025 13:16:20 +0100 + matrix-synapse-py3 (1.132.0~rc1) stable; urgency=medium * New Synapse release 1.132.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 2359c74032..103cb3c758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.132.0rc1" +version = "1.132.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 6363d6382252cc4e39baf69e461b84edbd344900 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:42:28 +0100 Subject: [PATCH 383/691] Bump actions/setup-python from 5.5.0 to 5.6.0 (#18555) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index e815da1560..698fb6d982 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - name: Install PyYAML From 51dbbbb40fdc7db5b9212530342741e866d55d1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:43:01 +0100 Subject: [PATCH 384/691] Bump types-requests from 2.32.0.20250328 to 2.32.4.20250611 (#18558) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8e15c7eade..a9ec91acfa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3058,14 +3058,14 @@ files = [ [[package]] name = "types-requests" -version = "2.32.0.20250328" +version = "2.32.4.20250611" description = "Typing stubs for requests" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_requests-2.32.0.20250328-py3-none-any.whl", hash = "sha256:72ff80f84b15eb3aa7a8e2625fffb6a93f2ad5a0c20215fc1dcfa61117bcb2a2"}, - {file = "types_requests-2.32.0.20250328.tar.gz", hash = "sha256:c9e67228ea103bd811c96984fac36ed2ae8da87a36a633964a21f199d60baf32"}, + {file = "types_requests-2.32.4.20250611-py3-none-any.whl", hash = "sha256:ad2fe5d3b0cb3c2c902c8815a70e7fb2302c4b8c1f77bdcd738192cdb3878072"}, + {file = "types_requests-2.32.4.20250611.tar.gz", hash = "sha256:741c8777ed6425830bf51e54d6abe245f79b4dcb9019f1622b773463946bf826"}, ] [package.dependencies] From 4a803e825746b03f55ab8a2d3d2922b80631f1c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:47:42 +0100 Subject: [PATCH 385/691] Bump dawidd6/action-download-artifact from 9 to 11 (#18556) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr-netlify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 8a06ad6362..53a2d6b597 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@07ab29fd4a977ae4d2b275087cf67563dfdf0295 # v9 + uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11 with: workflow: docs-pr.yaml run_id: ${{ github.event.workflow_run.id }} From 33e0c25279c68b0671f40a42a2512aab7fd85922 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jun 2025 12:58:31 +0100 Subject: [PATCH 386/691] Clean up old `device_federation_inbox` rows (#18546) Fixes https://github.com/element-hq/synapse/issues/17370 --- changelog.d/18546.misc | 1 + synapse/storage/databases/main/deviceinbox.py | 76 ++++++++- .../92/06_device_federation_inbox_index.sql | 16 ++ synapse/util/__init__.py | 9 ++ .../databases/main/test_deviceinbox.py | 149 ++++++++++++++++++ 5 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18546.misc create mode 100644 synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql diff --git a/changelog.d/18546.misc b/changelog.d/18546.misc new file mode 100644 index 0000000000..046c61e16f --- /dev/null +++ b/changelog.d/18546.misc @@ -0,0 +1 @@ +Clean up old, unused rows from the `device_federation_inbox` table. diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index d47833655d..579d29ac15 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -42,6 +42,7 @@ from synapse.logging.opentracing import ( start_active_span, trace, ) +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.streams import ToDeviceStream from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( @@ -52,7 +53,7 @@ from synapse.storage.database import ( ) from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict -from synapse.util import json_encoder +from synapse.util import Duration, json_encoder from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.stringutils import parse_and_validate_server_name @@ -63,6 +64,18 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# How long to keep messages in the device federation inbox before deleting them. +DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS = 7 * Duration.DAY_MS + +# How often to run the task to clean up old device_federation_inbox rows. +DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS = 5 * Duration.MINUTE_MS + +# Update name for the device federation inbox received timestamp index. +DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE = ( + "device_federation_inbox_received_ts_index" +) + + class DeviceInboxWorkerStore(SQLBaseStore): def __init__( self, @@ -134,6 +147,14 @@ class DeviceInboxWorkerStore(SQLBaseStore): prefilled_cache=device_outbox_prefill, ) + if hs.config.worker.run_background_tasks: + self._clock.looping_call( + run_as_background_process, + DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, + "_delete_old_federation_inbox_rows", + self._delete_old_federation_inbox_rows, + ) + def process_replication_rows( self, stream_name: str, @@ -960,6 +981,52 @@ class DeviceInboxWorkerStore(SQLBaseStore): ], ) + async def _delete_old_federation_inbox_rows(self, batch_size: int = 1000) -> None: + """Delete old rows from the device_federation_inbox table.""" + + # We wait until we have the index on `received_ts`, otherwise the query + # will take a very long time. + if not await self.db_pool.updates.has_completed_background_update( + DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE + ): + return + + def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: + # We delete at most 100 rows that are older than + # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + delete_before_ts = ( + self._clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + ) + sql = """ + WITH to_delete AS ( + SELECT origin, message_id + FROM device_federation_inbox + WHERE received_ts < ? + ORDER BY received_ts ASC + LIMIT ? + ) + DELETE FROM device_federation_inbox + WHERE + (origin, message_id) IN ( + SELECT origin, message_id FROM to_delete + ) + """ + txn.execute(sql, (delete_before_ts, batch_size)) + return txn.rowcount < batch_size + + while True: + finished = await self.db_pool.runInteraction( + "_delete_old_federation_inbox_rows", + _delete_old_federation_inbox_rows_txn, + db_autocommit=True, # We don't need to run in a transaction + ) + if finished: + return + + # We sleep a bit so that we don't hammer the database in a tight + # loop first time we run this. + self._clock.sleep(1) + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" @@ -995,6 +1062,13 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore): self._cleanup_device_federation_outbox, ) + self.db_pool.updates.register_background_index_update( + update_name=DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE, + index_name="device_federation_inbox_received_ts_index", + table="device_federation_inbox", + columns=["received_ts"], + ) + async def _background_drop_index_device_inbox( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql b/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql new file mode 100644 index 0000000000..0ba5b56148 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql @@ -0,0 +1,16 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Background update that adds an index to `device_federation_inbox.received_ts` +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9206, 'device_federation_inbox_received_ts_index', '{}'); diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 17836d21b3..bd4d20accb 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -55,6 +55,15 @@ if typing.TYPE_CHECKING: logger = logging.getLogger(__name__) +class Duration: + """Helper class that holds constants for common time durations in + milliseconds.""" + + MINUTE_MS = 60 * 1000 + HOUR_MS = 60 * MINUTE_MS + DAY_MS = 24 * HOUR_MS + + def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) diff --git a/tests/storage/databases/main/test_deviceinbox.py b/tests/storage/databases/main/test_deviceinbox.py index f89f96fdcb..7556111f16 100644 --- a/tests/storage/databases/main/test_deviceinbox.py +++ b/tests/storage/databases/main/test_deviceinbox.py @@ -19,11 +19,17 @@ # # +from unittest.mock import patch + +from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor from synapse.rest import admin from synapse.rest.client import devices from synapse.server import HomeServer +from synapse.storage.databases.main.deviceinbox import ( + DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS, +) from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -172,3 +178,146 @@ class DeviceInboxBackgroundUpdateStoreTestCase(HomeserverTestCase): ) self.assertEqual(1, len(res)) self.assertEqual(res[0], "cur_device") + + +class DeviceInboxFederationInboxCleanupTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + devices.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.db_pool = self.store.db_pool + + # Advance time to ensure we are past the cleanup delay + self.reactor.advance(DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS * 2 / 1000) + + def test_delete_old_federation_inbox_rows_skips_if_no_index(self) -> None: + """Test that we don't delete rows if the index hasn't been created yet.""" + + # Insert some test data into device_federation_inbox + for i in range(5): + self.get_success( + self.db_pool.simple_insert( + "device_federation_inbox", + { + "origin": "example.com", + "message_id": f"msg_{i}", + "received_ts": 0, + }, + ) + ) + + # Mock to report the update as not completed + with patch( + "synapse.storage.background_updates.BackgroundUpdater.has_completed_background_update" + ) as mock: + mock.return_value = False + + self.get_success(self.store._delete_old_federation_inbox_rows()) + + # Check that no rows were deleted + rows = self.get_success( + self.db_pool.simple_select_list( + "device_federation_inbox", + keyvalues={}, + retcols=["origin", "message_id", "received_ts"], + ) + ) + self.assertEqual( + len(rows), 5, "Expected no rows to be deleted when index is missing" + ) + + def test_delete_old_federation_inbox_rows(self) -> None: + """Test that old rows are deleted from device_federation_inbox.""" + + # Insert old messages + for i in range(5): + self.get_success( + self.db_pool.simple_insert( + "device_federation_inbox", + { + "origin": "old.example.com", + "message_id": f"old_msg_{i}", + "received_ts": self.clock.time_msec(), + }, + ) + ) + + self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS / 1000) + + # Insert new messages + for i in range(5): + self.get_success( + self.db_pool.simple_insert( + "device_federation_inbox", + { + "origin": "new.example.com", + "message_id": f"new_msg_{i}", + "received_ts": self.clock.time_msec(), + }, + ) + ) + + # Run the cleanup + self.get_success(self.store._delete_old_federation_inbox_rows()) + + # Check that only old messages were deleted + rows = self.get_success( + self.db_pool.simple_select_onecol( + "device_federation_inbox", + keyvalues={}, + retcol="origin", + ) + ) + + self.assertEqual(len(rows), 5, "Expected only 5 new messages to remain") + for origin in rows: + self.assertEqual(origin, "new.example.com") + + def test_delete_old_federation_inbox_rows_batch_limit(self) -> None: + """Test that the deletion happens in batches.""" + + # Insert 10 old messages (more than the 5 batch limit) + for i in range(10): + self.get_success( + self.db_pool.simple_insert( + "device_federation_inbox", + { + "origin": "old.example.com", + "message_id": f"old_msg_{i}", + "received_ts": self.clock.time_msec(), + }, + ) + ) + + # Advance time to ensure we are past the cleanup delay + self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS / 1000) + + # Run the cleanup - it should delete in batches and sleep between them + deferred = defer.ensureDeferred( + self.store._delete_old_federation_inbox_rows(batch_size=5) + ) + + # Check that the deferred doesn't resolve immediately + self.assertFalse(deferred.called) + + # Advance the reactor to allow the cleanup to continue and complete + self.reactor.advance(2) + self.get_success(deferred) + + # Check that all messages were deleted after multiple batches + rows = self.get_success( + self.db_pool.simple_select_list( + "device_federation_inbox", + keyvalues={}, + retcols=["origin", "message_id"], + ) + ) + + self.assertEqual( + len(rows), + 0, + "Expected all messages to be deleted after multiple batches", + ) From 51025653697dffbb3e346af8a5479f2b18e1e4c1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jun 2025 16:40:52 +0100 Subject: [PATCH 387/691] Fixup generated config documentation (#18568) Somehow its got out of sync, picked up by CI on develop. --- changelog.d/18568.doc | 1 + docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18568.doc diff --git a/changelog.d/18568.doc b/changelog.d/18568.doc new file mode 100644 index 0000000000..a692410f4c --- /dev/null +++ b/changelog.d/18568.doc @@ -0,0 +1 @@ +Fix typo in user type documentation. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1e8953ae37..860837cae5 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -770,7 +770,7 @@ This setting has the following sub-options: * `default_user_type` (string|null): The default user type to use for registering new users when no value has been specified. Defaults to none. Defaults to `null`. -* `extra_user_types` (list): Array of additional user types to allow. These are treated as real users. Defaults to `[]`. +* `extra_user_types` (array): Array of additional user types to allow. These are treated as real users. Defaults to `[]`. Example configuration: ```yaml From 74ca7ae720c9fdb09a2271ade28d909132bd9b7b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 20 Jun 2025 06:02:14 -0600 Subject: [PATCH 388/691] Add report user API from MSC4260 (#18120) Co-authored-by: turt2live <1190097+turt2live@users.noreply.github.com> Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18120.feature | 1 + .../configuration/config_documentation.md | 27 +++++ schema/synapse-config.schema.yaml | 17 ++++ synapse/config/ratelimiting.py | 6 ++ synapse/handlers/reports.py | 98 +++++++++++++++++++ synapse/rest/client/reporting.py | 38 +++++++ synapse/server.py | 5 + synapse/storage/databases/main/room.py | 32 ++++++ .../main/delta/92/07_add_user_reports.sql | 22 +++++ tests/rest/client/test_reporting.py | 89 +++++++++++++++++ 10 files changed, 335 insertions(+) create mode 100644 changelog.d/18120.feature create mode 100644 synapse/handlers/reports.py create mode 100644 synapse/storage/schema/main/delta/92/07_add_user_reports.sql diff --git a/changelog.d/18120.feature b/changelog.d/18120.feature new file mode 100644 index 0000000000..15cfabba42 --- /dev/null +++ b/changelog.d/18120.feature @@ -0,0 +1 @@ +Add support for the [MSC4260 user report API](https://github.com/matrix-org/matrix-spec-proposals/pull/4260). \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 860837cae5..257ea4a1a2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1937,6 +1937,33 @@ rc_delayed_event_mgmt: burst_count: 20.0 ``` --- +### `rc_reports` + +*(object)* Ratelimiting settings for reporting content. +This is a ratelimiting option that ratelimits reports made by users about content they see. +Setting this to a high value allows users to report content quickly, possibly in duplicate. This can result in higher database usage. + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_reports: + per_user: + per_second: 1.0 + burst_count: 5.0 +``` + +Example configuration: +```yaml +rc_reports: + per_second: 2.0 + burst_count: 20.0 +``` +--- ### `federation_rr_transactions_per_room_per_second` *(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 52a6d5cf71..8a5c685049 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2185,6 +2185,23 @@ properties: examples: - per_second: 2.0 burst_count: 20.0 + rc_reports: + $ref: "#/$defs/rc" + description: >- + Ratelimiting settings for reporting content. + + This is a ratelimiting option that ratelimits reports made by users + about content they see. + + Setting this to a high value allows users to report content quickly, possibly in + duplicate. This can result in higher database usage. + default: + per_user: + per_second: 1.0 + burst_count: 5.0 + examples: + - per_second: 2.0 + burst_count: 20.0 federation_rr_transactions_per_room_per_second: type: integer description: >- diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index eb1dc2dacb..290701615f 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -240,3 +240,9 @@ class RatelimitConfig(Config): "rc_delayed_event_mgmt", defaults={"per_second": 1, "burst_count": 5}, ) + + self.rc_reports = RatelimitSettings.parse( + config, + "rc_reports", + defaults={"per_second": 1, "burst_count": 5}, + ) diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py new file mode 100644 index 0000000000..a7b8a4bed7 --- /dev/null +++ b/synapse/handlers/reports.py @@ -0,0 +1,98 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2015, 2016 OpenMarket Ltd +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.errors import Codes, SynapseError +from synapse.api.ratelimiting import Ratelimiter +from synapse.types import ( + Requester, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReportsHandler: + def __init__(self, hs: "HomeServer"): + self._hs = hs + self._store = hs.get_datastores().main + self._clock = hs.get_clock() + + # Ratelimiter for management of existing delayed events, + # keyed by the requesting user ID. + self._reports_ratelimiter = Ratelimiter( + store=self._store, + clock=self._clock, + cfg=hs.config.ratelimiting.rc_reports, + ) + + async def report_user( + self, requester: Requester, target_user_id: str, reason: str + ) -> None: + """Files a report against a user from a user. + + Rate and size limits are applied to the report. If the user being reported + does not belong to this server, the report is ignored. This check is done + after the limits to reduce DoS potential. + + If the user being reported belongs to this server, but doesn't exist, we + similarly ignore the report. The spec allows us to return an error if we + want to, but we choose to hide that user's existence instead. + + If the report is otherwise valid (for a user which exists on our server), + we append it to the database for later processing. + + Args: + requester - The user filing the report. + target_user_id - The user being reported. + reason - The user-supplied reason the user is being reported. + + Raises: + SynapseError for BAD_REQUEST/BAD_JSON if the reason is too long. + """ + + await self._check_limits(requester) + + if len(reason) > 1000: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Reason must be less than 1000 characters", + Codes.BAD_JSON, + ) + + if not self._hs.is_mine_id(target_user_id): + return # hide that they're not ours/that we can't do anything about them + + user = await self._store.get_user_by_id(target_user_id) + if user is None: + return # hide that they don't exist + + await self._store.add_user_report( + target_user_id=target_user_id, + user_id=requester.user.to_string(), + reason=reason, + received_ts=self._clock.time_msec(), + ) + + async def _check_limits(self, requester: Requester) -> None: + await self._reports_ratelimiter.ratelimit( + requester, + requester.user.to_string(), + ) diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index c5037be8b7..484827d9f2 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -150,6 +150,44 @@ class ReportRoomRestServlet(RestServlet): return 200, {} +class ReportUserRestServlet(RestServlet): + """This endpoint lets clients report a user for abuse. + + Introduced by MSC4260: https://github.com/matrix-org/matrix-spec-proposals/pull/4260 + """ + + PATTERNS = list( + client_patterns( + "/users/(?P[^/]*)/report$", + releases=("v3",), + unstable=False, + v1=False, + ) + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.hs = hs + self.auth = hs.get_auth() + self.clock = hs.get_clock() + self.store = hs.get_datastores().main + self.handler = hs.get_reports_handler() + + class PostBody(RequestBodyModel): + reason: StrictStr + + async def on_POST( + self, request: SynapseRequest, target_user_id: str + ) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + body = parse_and_validate_json_object_from_request(request, self.PostBody) + + await self.handler.report_user(requester, target_user_id, body.reason) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReportEventRestServlet(hs).register(http_server) ReportRoomRestServlet(hs).register(http_server) + ReportUserRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index 2add4d4e6e..fd16abb9ea 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -94,6 +94,7 @@ from synapse.handlers.read_marker import ReadMarkerHandler from synapse.handlers.receipts import ReceiptsHandler from synapse.handlers.register import RegistrationHandler from synapse.handlers.relations import RelationsHandler +from synapse.handlers.reports import ReportsHandler from synapse.handlers.room import ( RoomContextHandler, RoomCreationHandler, @@ -718,6 +719,10 @@ class HomeServer(metaclass=abc.ABCMeta): def get_receipts_handler(self) -> ReceiptsHandler: return ReceiptsHandler(self) + @cache_in_self + def get_reports_handler(self) -> ReportsHandler: + return ReportsHandler(self) + @cache_in_self def get_read_marker_handler(self) -> ReadMarkerHandler: return ReadMarkerHandler(self) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 1df06a5171..58451d3ff1 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -2421,6 +2421,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id") self._room_reports_id_gen = IdGenerator(db_conn, "room_reports", "id") + self._user_reports_id_gen = IdGenerator(db_conn, "user_reports", "id") self._instance_name = hs.get_instance_name() @@ -2662,6 +2663,37 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): ) return next_id + async def add_user_report( + self, + target_user_id: str, + user_id: str, + reason: str, + received_ts: int, + ) -> int: + """Add a user report + + Args: + target_user_id: The user ID being reported. + user_id: User who reported the user. + reason: Description that the user specifies. + received_ts: Time when the user submitted the report (milliseconds). + Returns: + ID of the room report. + """ + next_id = self._user_reports_id_gen.get_next() + await self.db_pool.simple_insert( + table="user_reports", + values={ + "id": next_id, + "received_ts": received_ts, + "target_user_id": target_user_id, + "user_id": user_id, + "reason": reason, + }, + desc="add_user_report", + ) + return next_id + async def clear_partial_state_room(self, room_id: str) -> Optional[int]: """Clears the partial state flag for a room. diff --git a/synapse/storage/schema/main/delta/92/07_add_user_reports.sql b/synapse/storage/schema/main/delta/92/07_add_user_reports.sql new file mode 100644 index 0000000000..7439dad6d6 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/07_add_user_reports.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE TABLE user_reports ( + id BIGINT NOT NULL PRIMARY KEY, + received_ts BIGINT NOT NULL, + target_user_id TEXT NOT NULL, + user_id TEXT NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX user_reports_target_user_id ON user_reports(target_user_id); -- for lookups +CREATE INDEX user_reports_user_id ON user_reports(user_id); -- for lookups diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 723553979f..9335d46eb5 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +from typing import Optional from twisted.test.proto_helpers import MemoryReactor @@ -201,3 +202,91 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(response_status, channel.code, msg=channel.result["body"]) + + +class ReportUserTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.other_user = self.register_user("user", "pass") + self.other_user_tok = self.login("user", "pass") + + self.target_user_id = self.register_user("target_user", "pass") + + def test_reason_str(self) -> None: + data = {"reason": "this makes me sad"} + self._assert_status(200, data) + + rows = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_onecol( + table="user_reports", + keyvalues={"target_user_id": self.target_user_id}, + retcol="id", + desc="get_user_report_ids", + ) + ) + self.assertEqual(len(rows), 1) + + def test_no_reason(self) -> None: + data = {"not_reason": "for typechecking"} + self._assert_status(400, data) + + def test_reason_nonstring(self) -> None: + data = {"reason": 42} + self._assert_status(400, data) + + def test_reason_null(self) -> None: + data = {"reason": None} + self._assert_status(400, data) + + def test_reason_long(self) -> None: + data = {"reason": "x" * 1001} + self._assert_status(400, data) + + def test_cannot_report_nonlocal_user(self) -> None: + """ + Tests that we ignore reports for nonlocal users. + """ + target_user_id = "@bloop:example.org" + data = {"reason": "i am very sad"} + self._assert_status(200, data, target_user_id) + self._assert_no_reports_for_user(target_user_id) + + def test_can_report_nonexistent_user(self) -> None: + """ + Tests that we ignore reports for nonexistent users. + """ + target_user_id = f"@bloop:{self.hs.hostname}" + data = {"reason": "i am very sad"} + self._assert_status(200, data, target_user_id) + self._assert_no_reports_for_user(target_user_id) + + def _assert_no_reports_for_user(self, target_user_id: str) -> None: + rows = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_onecol( + table="user_reports", + keyvalues={"target_user_id": target_user_id}, + retcol="id", + desc="get_user_report_ids", + ) + ) + self.assertEqual(len(rows), 0) + + def _assert_status( + self, response_status: int, data: JsonDict, user_id: Optional[str] = None + ) -> None: + if user_id is None: + user_id = self.target_user_id + channel = self.make_request( + "POST", + f"/_matrix/client/v3/users/{user_id}/report", + data, + access_token=self.other_user_tok, + shorthand=False, + ) + self.assertEqual(response_status, channel.code, msg=channel.result["body"]) From 3cabaa84ca7d93d05e32a4f45601567751bb15ed Mon Sep 17 00:00:00 2001 From: V02460 Date: Mon, 23 Jun 2025 14:48:07 +0200 Subject: [PATCH 389/691] Update PyO3 to version 0.25 (#18578) Updates `pyo3` to version 0.25.1 and, accordingly, `pyo3-log` to v0.12.4 and `pythonize` to v0.25.0. PyO3 v0.25 enables Python 3.14 support. --- Cargo.lock | 25 ++++++++++++------------- changelog.d/18578.misc | 1 + rust/Cargo.toml | 6 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 changelog.d/18578.misc diff --git a/Cargo.lock b/Cargo.lock index ec3a5ca24e..39d9a9f20f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,12 +809,11 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ "anyhow", - "cfg-if", "indoc", "libc", "memoffset", @@ -828,9 +827,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ "once_cell", "target-lexicon", @@ -838,9 +837,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -859,9 +858,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -871,9 +870,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck", "proc-macro2", @@ -884,9 +883,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5bcac0d0b71821f0d69e42654f1e15e5c94b85196446c4de9588951a2117e7b" +checksum = "597907139a488b22573158793aa7539df36ae863eba300c75f3a0d65fc475e27" dependencies = [ "pyo3", "serde", diff --git a/changelog.d/18578.misc b/changelog.d/18578.misc new file mode 100644 index 0000000000..cafba77722 --- /dev/null +++ b/changelog.d/18578.misc @@ -0,0 +1 @@ +Update PyO3 to version 0.25. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 446db1bcf1..3a47f7ddf7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,14 +30,14 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.24.2", features = [ +pyo3 = { version = "0.25.1", features = [ "macros", "anyhow", "abi3", "abi3-py39", ] } -pyo3-log = "0.12.3" -pythonize = "0.24.0" +pyo3-log = "0.12.4" +pythonize = "0.25.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } From 6791e6e250eac1116e077aa8bbec1b7c432f898f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 24 Jun 2025 11:52:06 +0100 Subject: [PATCH 390/691] Unbreak unit tests with Twisted `25.5.0` by add `parsePOSTFormSubmission` arg to `FakeSite` (#18577) Co-authored-by: anoa's Codex Agent --- changelog.d/18577.misc | 1 + poetry.lock | 91 +++++++++---------- synapse/http/proxyagent.py | 17 +++- synapse/http/replicationagent.py | 13 ++- synapse/http/server.py | 2 +- tests/logging/test_terse_json.py | 9 +- tests/replication/_base.py | 2 +- tests/server.py | 9 +- .../databases/main/test_events_worker.py | 35 ++++--- tests/test_server.py | 3 +- 10 files changed, 107 insertions(+), 75 deletions(-) create mode 100644 changelog.d/18577.misc diff --git a/changelog.d/18577.misc b/changelog.d/18577.misc new file mode 100644 index 0000000000..339591b320 --- /dev/null +++ b/changelog.d/18577.misc @@ -0,0 +1 @@ +Add support for Twisted `25.5.0`+ releases. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index a9ec91acfa..8211b096c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.5.2-py2.py3-none-any.whl", hash = "sha256:8804dd4402ac5e4a0435ac49e0b6e19e395357cfa632a3f624dcb4f6df13b4b1"}, {file = "authlib-1.5.2.tar.gz", hash = "sha256:fe85ec7e50c5f86f1e2603518bb3b4f632985eb4a355e52256530790e326c512"}, @@ -50,19 +50,18 @@ cryptography = "*" [[package]] name = "automat" -version = "22.10.0" +version = "25.4.16" description = "Self-service finite-state machines for the programmer on the go." optional = false -python-versions = "*" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "Automat-22.10.0-py2.py3-none-any.whl", hash = "sha256:c3164f8742b9dc440f3682482d32aaff7bb53f71740dd018533f9de286b64180"}, - {file = "Automat-22.10.0.tar.gz", hash = "sha256:e56beb84edad19dcc11d30e8d9b895f75deeb5ef5e96b84a467066b3b84bb04e"}, + {file = "automat-25.4.16-py3-none-any.whl", hash = "sha256:04e9bce696a8d5671ee698005af6e5a9fa15354140a87f4870744604dcdd3ba1"}, + {file = "automat-25.4.16.tar.gz", hash = "sha256:0017591a5477066e90d26b0e696ddc143baafd87b588cfac8100bc6be9634de0"}, ] [package.dependencies] -attrs = ">=19.2.0" -six = "*" +typing_extensions = {version = "*", markers = "python_version < \"3.10\""} [package.extras] visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] @@ -451,7 +450,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -494,7 +493,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -544,7 +543,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -890,7 +889,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1028,7 +1027,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1044,7 +1043,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, @@ -1324,7 +1323,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1545,7 +1544,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1714,7 +1713,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1722,7 +1721,6 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1735,7 +1733,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1751,7 +1749,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1974,7 +1972,7 @@ description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"user-search\"" +markers = "extra == \"user-search\" or extra == \"all\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -2023,7 +2021,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2083,7 +2081,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2108,7 +2106,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2136,7 +2134,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2500,7 +2498,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2688,7 +2686,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2704,7 +2702,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2766,7 +2764,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2857,19 +2855,19 @@ keyring = ["keyring (>=15.1)"] [[package]] name = "twisted" -version = "24.7.0" +version = "25.5.0" description = "An asynchronous networking framework written in Python" optional = false python-versions = ">=3.8.0" groups = ["main"] files = [ - {file = "twisted-24.7.0-py3-none-any.whl", hash = "sha256:734832ef98108136e222b5230075b1079dad8a3fc5637319615619a7725b0c81"}, - {file = "twisted-24.7.0.tar.gz", hash = "sha256:5a60147f044187a127ec7da96d170d49bcce50c6fd36f594e60f4587eff4d394"}, + {file = "twisted-25.5.0-py3-none-any.whl", hash = "sha256:8559f654d01a54a8c3efe66d533d43f383531ebf8d81d9f9ab4769d91ca15df7"}, + {file = "twisted-25.5.0.tar.gz", hash = "sha256:1deb272358cb6be1e3e8fc6f9c8b36f78eb0fa7c2233d2dbe11ec6fee04ea316"}, ] [package.dependencies] -attrs = ">=21.3.0" -automat = ">=0.8.0" +attrs = ">=22.2.0" +automat = ">=24.8.0" constantly = ">=15.1" hyperlink = ">=17.1.1" idna = {version = ">=2.4", optional = true, markers = "extra == \"tls\""} @@ -2880,19 +2878,20 @@ typing-extensions = ">=4.2.0" zope-interface = ">=5" [package.extras] -all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] -dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] -dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] -gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] -osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] +dev-release = ["pydoctor (>=24.11.1,<24.12.0)", "pydoctor (>=24.11.1,<24.12.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] +gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +http2 = ["h2 (>=3.2,<5.0)", "priority (>=1.1.0,<2.0)"] +macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] +mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (==1.10.1)", "mypy-zope (==1.0.6)", "priority (>=1.1.0,<2.0)", "pydoctor (>=24.11.1,<24.12.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools", "wsproto"] +osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core (<11) ; python_version < \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-core ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork (<11) ; python_version < \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cfnetwork ; python_version >= \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa (<11) ; python_version < \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyobjc-framework-cocoa ; python_version >= \"3.9\"", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "wsproto", "wsproto"] serial = ["pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\""] -test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] +test = ["cython-test-exception-raiser (>=1.0.2,<2)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] -windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] +websocket = ["wsproto"] +windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.2,<5.0)", "h2 (>=3.2,<5.0)", "httpx[http2] (>=0.27)", "httpx[http2] (>=0.27)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)", "wsproto", "wsproto"] [[package]] name = "txredisapi" @@ -2901,7 +2900,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3244,7 +3243,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 6817199035..6217f9b0b2 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -21,7 +21,7 @@ import logging import random import re -from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple, Union, cast from urllib.parse import urlparse from urllib.request import ( # type: ignore[attr-defined] getproxies_environment, @@ -40,6 +40,7 @@ from twisted.internet.interfaces import ( IProtocol, IProtocolFactory, IReactorCore, + IReactorTime, IStreamClientEndpoint, ) from twisted.python.failure import Failure @@ -129,7 +130,9 @@ class ProxyAgent(_AgentBase): ): contextFactory = contextFactory or BrowserLikePolicyForHTTPS() - _AgentBase.__init__(self, reactor, pool) + # `_AgentBase` expects an `IReactorTime` provider. `IReactorCore` + # extends `IReactorTime`, so this cast is safe. + _AgentBase.__init__(self, cast(IReactorTime, reactor), pool) if proxy_reactor is None: self.proxy_reactor = reactor @@ -168,7 +171,7 @@ class ProxyAgent(_AgentBase): self.no_proxy = no_proxy self._policy_for_https = contextFactory - self._reactor = reactor + self._reactor = cast(IReactorTime, reactor) self._federation_proxy_endpoint: Optional[IStreamClientEndpoint] = None self._federation_proxy_credentials: Optional[ProxyCredentials] = None @@ -257,7 +260,11 @@ class ProxyAgent(_AgentBase): raise ValueError(f"Invalid URI {uri!r}") parsed_uri = URI.fromBytes(uri) - pool_key = f"{parsed_uri.scheme!r}{parsed_uri.host!r}{parsed_uri.port}" + pool_key: tuple[bytes, bytes, int] = ( + parsed_uri.scheme, + parsed_uri.host, + parsed_uri.port, + ) request_path = parsed_uri.originForm should_skip_proxy = False @@ -283,7 +290,7 @@ class ProxyAgent(_AgentBase): ) # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - pool_key = "http-proxy" + pool_key = (b"http-proxy", b"", 0) endpoint = self.http_proxy_endpoint request_path = uri elif ( diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index 4eabbc8af9..d70575dbd5 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -180,9 +180,16 @@ class ReplicationAgent(_AgentBase): worker_name = parsedURI.netloc.decode("utf-8") key_scheme = self._endpointFactory.instance_map[worker_name].scheme() key_netloc = self._endpointFactory.instance_map[worker_name].netloc() - # This sets the Pool key to be: - # (http(s), ) or (unix, ) - key = (key_scheme, key_netloc) + # Build a connection pool key. + # + # `_AgentBase` expects this to be a three-tuple of `(scheme, host, + # port)` of type `bytes`. We don't have a real port when connecting via + # a Unix socket, so use `0`. + key = ( + key_scheme.encode("ascii"), + key_netloc.encode("utf-8"), + 0, + ) # _requestWithEndpoint comes from _AgentBase class return self._requestWithEndpoint( diff --git a/synapse/http/server.py b/synapse/http/server.py index 8bebafc18e..311e21774f 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -497,7 +497,7 @@ class JsonResource(DirectServeJsonResource): key word arguments to pass to the callback """ # At this point the path must be bytes. - request_path_bytes: bytes = request.path # type: ignore + request_path_bytes: bytes = request.path request_path = request_path_bytes.decode("ascii") # Treat HEAD requests as GET requests. request_method = request.method diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index 33b94cf9fa..d9cbbbd51e 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -160,7 +160,14 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): logger = self.get_logger(handler) # A full request isn't needed here. - site = Mock(spec=["site_tag", "server_version_string", "getResourceFor"]) + site = Mock( + spec=[ + "site_tag", + "server_version_string", + "getResourceFor", + "_parsePOSTFormSubmission", + ] + ) site.site_tag = "test-site" site.server_version_string = "Server v1" site.reactor = Mock() diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 8437da1cdd..75a36303c9 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -220,7 +220,7 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): fetching updates for given stream. """ - path: bytes = request.path # type: ignore + path: bytes = request.path self.assertRegex( path, rb"^/_synapse/replication/get_repl_stream_updates/%s/[^/]+$" diff --git a/tests/server.py b/tests/server.py index f01708b77f..0c519bc4c9 100644 --- a/tests/server.py +++ b/tests/server.py @@ -343,6 +343,8 @@ class FakeSite: self, resource: IResource, reactor: IReactorTime, + *, + parsePOSTFormSubmission: bool = True, ): """ @@ -351,6 +353,7 @@ class FakeSite: """ self._resource = resource self.reactor = reactor + self._parsePOSTFormSubmission = parsePOSTFormSubmission def getResourceFor(self, request: Request) -> IResource: return self._resource @@ -514,9 +517,13 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): tls._get_default_clock = lambda: self - self.nameResolver = SimpleResolverComplexifier(FakeResolver()) super().__init__() + # Override the default name resolver with our fake resolver. This must + # happen after `super().__init__()` so that the base class doesn't + # overwrite it again. + self.nameResolver = SimpleResolverComplexifier(FakeResolver()) + def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver: raise NotImplementedError() diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 104d141a72..18039a07e2 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -449,24 +449,29 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase): def test_recovery(self) -> None: """Test that event fetchers recover after a database outage.""" - with self._outage(): - # Kick off a bunch of event fetches but do not pump the reactor - event_deferreds = [] - for event_id in self.event_ids: - event_deferreds.append(ensureDeferred(self.store.get_event(event_id))) + with self.assertLogs( + "synapse.metrics.background_process_metrics", level="ERROR" + ): + with self._outage(): + # Kick off a bunch of event fetches but do not pump the reactor + event_deferreds = [] + for event_id in self.event_ids: + event_deferreds.append( + ensureDeferred(self.store.get_event(event_id)) + ) - # We should have maxed out on event fetcher threads - self.assertEqual(self.store._event_fetch_ongoing, EVENT_QUEUE_THREADS) + # We should have maxed out on event fetcher threads + self.assertEqual(self.store._event_fetch_ongoing, EVENT_QUEUE_THREADS) - # All the event fetchers will fail - self.pump() - self.assertEqual(self.store._event_fetch_ongoing, 0) + # All the event fetchers will fail + self.pump() + self.assertEqual(self.store._event_fetch_ongoing, 0) - for event_deferred in event_deferreds: - failure = self.get_failure(event_deferred, Exception) - self.assertEqual( - str(failure.value), "Could not connect to the database." - ) + for event_deferred in event_deferreds: + failure = self.get_failure(event_deferred, Exception) + self.assertEqual( + str(failure.value), "Could not connect to the database." + ) # This next event fetch should succeed self.get_success(self.store.get_event(self.event_ids[0])) diff --git a/tests/test_server.py b/tests/test_server.py index 9cb6766b5f..9ebfbf96e7 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -226,8 +226,7 @@ class OptionsResourceTests(unittest.TestCase): isLeaf = True def render(self, request: SynapseRequest) -> bytes: - # Type-ignore: mypy thinks request.path is Optional[Any], not bytes. - return request.path # type: ignore[return-value] + return request.path # Setup a resource with some children. self.resource = OptionsResource() From cb259eb206c42656c962b41d187aee1467301543 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Jun 2025 11:59:23 +0100 Subject: [PATCH 391/691] 1.133.0rc1 --- CHANGES.md | 39 +++++++++++++++++++++++++++++++ changelog.d/18120.feature | 1 - changelog.d/18357.misc | 1 - changelog.d/18528.doc | 1 - changelog.d/18534.bugfix | 1 - changelog.d/18535.bugfix | 1 - changelog.d/18541.misc | 1 - changelog.d/18542.misc | 1 - changelog.d/18543.bugfix | 1 - changelog.d/18545.bugfix | 1 - changelog.d/18546.misc | 1 - changelog.d/18547.bugfix | 1 - changelog.d/18551.misc | 1 - changelog.d/18561.misc | 1 - changelog.d/18568.doc | 1 - changelog.d/18577.misc | 1 - changelog.d/18578.misc | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 20 files changed, 47 insertions(+), 18 deletions(-) delete mode 100644 changelog.d/18120.feature delete mode 100644 changelog.d/18357.misc delete mode 100644 changelog.d/18528.doc delete mode 100644 changelog.d/18534.bugfix delete mode 100644 changelog.d/18535.bugfix delete mode 100644 changelog.d/18541.misc delete mode 100644 changelog.d/18542.misc delete mode 100644 changelog.d/18543.bugfix delete mode 100644 changelog.d/18545.bugfix delete mode 100644 changelog.d/18546.misc delete mode 100644 changelog.d/18547.bugfix delete mode 100644 changelog.d/18551.misc delete mode 100644 changelog.d/18561.misc delete mode 100644 changelog.d/18568.doc delete mode 100644 changelog.d/18577.misc delete mode 100644 changelog.d/18578.misc diff --git a/CHANGES.md b/CHANGES.md index 0c736bfd95..23debccd63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,42 @@ +# Synapse 1.133.0rc1 (2025-06-24) + +### Features + +- Add support for the [MSC4260 user report API](https://github.com/matrix-org/matrix-spec-proposals/pull/4260). ([\#18120](https://github.com/element-hq/synapse/issues/18120)) + +### Bugfixes + +- Fix an issue where, during state resolution for v11 rooms, Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. ([\#18534](https://github.com/element-hq/synapse/issues/18534), [\#18547](https://github.com/element-hq/synapse/issues/18547)) +- Fix long-standing bug where sliding sync did not honour the `room_id_to_include` config option. ([\#18535](https://github.com/element-hq/synapse/issues/18535)) +- Fix an issue where "Lock timeout is getting excessive" warnings would be logged even when the lock timeout was <10 minutes. ([\#18543](https://github.com/element-hq/synapse/issues/18543)) +- Fix an issue where Synapse could calculate the wrong power level for the creator of the room if there was no power levels event. ([\#18545](https://github.com/element-hq/synapse/issues/18545)) + +### Improved Documentation + +- Generate config documentation from JSON Schema file. ([\#18528](https://github.com/element-hq/synapse/issues/18528)) +- Fix typo in user type documentation. ([\#18568](https://github.com/element-hq/synapse/issues/18568)) + +### Internal Changes + +- Increase performance of introspecting access tokens when using delegated auth. ([\#18357](https://github.com/element-hq/synapse/issues/18357), [\#18561](https://github.com/element-hq/synapse/issues/18561)) +- Log user deactivations. ([\#18541](https://github.com/element-hq/synapse/issues/18541)) +- Enable [`flake8-logging`](https://docs.astral.sh/ruff/rules/#flake8-logging-log) and [`flake8-logging-format`](https://docs.astral.sh/ruff/rules/#flake8-logging-format-g) rules in Ruff and fix related issues throughout the codebase. ([\#18542](https://github.com/element-hq/synapse/issues/18542)) +- Clean up old, unused rows from the `device_federation_inbox` table. ([\#18546](https://github.com/element-hq/synapse/issues/18546)) +- Run config schema CI on develop and release branches. ([\#18551](https://github.com/element-hq/synapse/issues/18551)) +- Add support for Twisted `25.5.0`+ releases. ([\#18577](https://github.com/element-hq/synapse/issues/18577)) +- Update PyO3 to version 0.25. ([\#18578](https://github.com/element-hq/synapse/issues/18578)) + + + +### Updates to locked dependencies + +* Bump actions/setup-python from 5.5.0 to 5.6.0. ([\#18555](https://github.com/element-hq/synapse/issues/18555)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18559](https://github.com/element-hq/synapse/issues/18559)) +* Bump dawidd6/action-download-artifact from 9 to 11. ([\#18556](https://github.com/element-hq/synapse/issues/18556)) +* Bump headers from 0.4.0 to 0.4.1. ([\#18529](https://github.com/element-hq/synapse/issues/18529)) +* Bump requests from 2.32.2 to 2.32.4. ([\#18533](https://github.com/element-hq/synapse/issues/18533)) +* Bump types-requests from 2.32.0.20250328 to 2.32.4.20250611. ([\#18558](https://github.com/element-hq/synapse/issues/18558)) + # Synapse 1.132.0 (2025-06-17) ### Improved Documentation diff --git a/changelog.d/18120.feature b/changelog.d/18120.feature deleted file mode 100644 index 15cfabba42..0000000000 --- a/changelog.d/18120.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for the [MSC4260 user report API](https://github.com/matrix-org/matrix-spec-proposals/pull/4260). \ No newline at end of file diff --git a/changelog.d/18357.misc b/changelog.d/18357.misc deleted file mode 100644 index 8e4ee71356..0000000000 --- a/changelog.d/18357.misc +++ /dev/null @@ -1 +0,0 @@ -Increase performance of introspecting access tokens when using delegated auth. diff --git a/changelog.d/18528.doc b/changelog.d/18528.doc deleted file mode 100644 index 83c03190dd..0000000000 --- a/changelog.d/18528.doc +++ /dev/null @@ -1 +0,0 @@ -Generate config documentation from JSON Schema file. diff --git a/changelog.d/18534.bugfix b/changelog.d/18534.bugfix deleted file mode 100644 index ee1bfc8f27..0000000000 --- a/changelog.d/18534.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix an issue where during state resolution for v11 rooms Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. diff --git a/changelog.d/18535.bugfix b/changelog.d/18535.bugfix deleted file mode 100644 index 246b1bc01f..0000000000 --- a/changelog.d/18535.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix long-standing bug where sliding sync did not honour the `room_id_to_include` config option. diff --git a/changelog.d/18541.misc b/changelog.d/18541.misc deleted file mode 100644 index 2c733e3e50..0000000000 --- a/changelog.d/18541.misc +++ /dev/null @@ -1 +0,0 @@ -Log user deactivations. diff --git a/changelog.d/18542.misc b/changelog.d/18542.misc deleted file mode 100644 index 8a2c2d5bf6..0000000000 --- a/changelog.d/18542.misc +++ /dev/null @@ -1 +0,0 @@ -Enable [`flake8-logging`](https://docs.astral.sh/ruff/rules/#flake8-logging-log) and [`flake8-logging-format`](https://docs.astral.sh/ruff/rules/#flake8-logging-format-g) rules in Ruff and fix related issues throughout the codebase. diff --git a/changelog.d/18543.bugfix b/changelog.d/18543.bugfix deleted file mode 100644 index 3db7b04a56..0000000000 --- a/changelog.d/18543.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix an issue where "Lock timeout is getting excessive" warnings would be logged even when the lock timeout was <10 minutes. \ No newline at end of file diff --git a/changelog.d/18545.bugfix b/changelog.d/18545.bugfix deleted file mode 100644 index 3ca6b4d624..0000000000 --- a/changelog.d/18545.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix an issue where Synapse could calculate the wrong power level for the creator of the room if there was no power levels event. diff --git a/changelog.d/18546.misc b/changelog.d/18546.misc deleted file mode 100644 index 046c61e16f..0000000000 --- a/changelog.d/18546.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up old, unused rows from the `device_federation_inbox` table. diff --git a/changelog.d/18547.bugfix b/changelog.d/18547.bugfix deleted file mode 100644 index ee1bfc8f27..0000000000 --- a/changelog.d/18547.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix an issue where during state resolution for v11 rooms Synapse would incorrectly calculate the power level of the creator when there was no power levels event in the room. diff --git a/changelog.d/18551.misc b/changelog.d/18551.misc deleted file mode 100644 index 9d60a59adc..0000000000 --- a/changelog.d/18551.misc +++ /dev/null @@ -1 +0,0 @@ -Run config schema CI on develop and release branches. diff --git a/changelog.d/18561.misc b/changelog.d/18561.misc deleted file mode 100644 index 8e4ee71356..0000000000 --- a/changelog.d/18561.misc +++ /dev/null @@ -1 +0,0 @@ -Increase performance of introspecting access tokens when using delegated auth. diff --git a/changelog.d/18568.doc b/changelog.d/18568.doc deleted file mode 100644 index a692410f4c..0000000000 --- a/changelog.d/18568.doc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in user type documentation. diff --git a/changelog.d/18577.misc b/changelog.d/18577.misc deleted file mode 100644 index 339591b320..0000000000 --- a/changelog.d/18577.misc +++ /dev/null @@ -1 +0,0 @@ -Add support for Twisted `25.5.0`+ releases. \ No newline at end of file diff --git a/changelog.d/18578.misc b/changelog.d/18578.misc deleted file mode 100644 index cafba77722..0000000000 --- a/changelog.d/18578.misc +++ /dev/null @@ -1 +0,0 @@ -Update PyO3 to version 0.25. diff --git a/debian/changelog b/debian/changelog index 6afa33ad74..614901e3fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.133.0~rc1) stable; urgency=medium + + * New Synapse release 1.133.0rc1. + + -- Synapse Packaging team Tue, 24 Jun 2025 11:57:47 +0100 + matrix-synapse-py3 (1.132.0) stable; urgency=medium * New Synapse release 1.132.0. diff --git a/pyproject.toml b/pyproject.toml index a0b8981454..a354e26d39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.132.0" +version = "1.133.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 8a5c685049..e4dcd36c30 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.132/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.133/synapse-config.schema.json type: object properties: modules: From 6fabf82f4f305f42a01b2f4de9aac282fa1fd4f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 17:30:36 +0100 Subject: [PATCH 392/691] Bump types-opentracing from 2.4.10.6 to 2.4.10.20250622 (#18586) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8211b096c8..f3d807f6ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2993,14 +2993,14 @@ files = [ [[package]] name = "types-opentracing" -version = "2.4.10.6" +version = "2.4.10.20250622" description = "Typing stubs for opentracing" optional = false -python-versions = "*" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types-opentracing-2.4.10.6.tar.gz", hash = "sha256:87a1bdfce9de5e555e30497663583b9b9c3bb494d029ef9806aa1f137c19e744"}, - {file = "types_opentracing-2.4.10.6-py3-none-any.whl", hash = "sha256:25914c834db033a4a38fc322df0b5e5e14503b0ac97f78304ae180d721555e97"}, + {file = "types_opentracing-2.4.10.20250622-py3-none-any.whl", hash = "sha256:26bc21f9e385d54898b47e9bd1fa13f200c2dada50394f6eafd063ed53813062"}, + {file = "types_opentracing-2.4.10.20250622.tar.gz", hash = "sha256:00db48b7f57136c45ac3250218bd0f18b9792566dfcbd5ad1de9f7e180347e74"}, ] [[package]] From 0c7d9919fac98a1ec833ee8ba043f072e344499e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 25 Jun 2025 08:59:18 -0400 Subject: [PATCH 393/691] Fix registering of background updates for split main/state db (#18509) The background updates are being registered on an object that is for the _state_ database, but the actual tables are on the _main_ database. This just moves them to a different store that can access the right stuff. I noticed this when trying to do a full schema dump cause I was curious what has changed since the last one. Fixes #16054 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18509.bugfix | 1 + rust/src/identifier.rs | 2 +- .../databases/main/events_bg_updates.py | 21 +++++++++++++ synapse/storage/databases/state/bg_updates.py | 31 ------------------- synapse/types/storage/__init__.py | 10 ++++++ 5 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 changelog.d/18509.bugfix diff --git a/changelog.d/18509.bugfix b/changelog.d/18509.bugfix new file mode 100644 index 0000000000..2c821d0395 --- /dev/null +++ b/changelog.d/18509.bugfix @@ -0,0 +1 @@ +Fix `KeyError` on background updates when using split main/state databases. diff --git a/rust/src/identifier.rs b/rust/src/identifier.rs index b70f6a30c7..03d1ebdc8a 100644 --- a/rust/src/identifier.rs +++ b/rust/src/identifier.rs @@ -27,7 +27,7 @@ pub enum IdentifierError { impl fmt::Display for IdentifierError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 5c83a9f779..8a59091da0 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -294,6 +294,27 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS where_clause="NOT outlier", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + # Handle background updates for Sliding Sync tables # self.db_pool.updates.register_background_update_handler( diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 5b594fe8dd..ac38b2ab19 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -290,16 +290,6 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" - CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "current_state_events_stream_ordering_idx" - ) - ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "room_memberships_stream_ordering_idx" - ) - LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( - "local_current_membership_stream_ordering_idx" - ) - def __init__( self, database: DatabasePool, @@ -336,27 +326,6 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): replaces_index="state_group_edges_idx", ) - # These indices are needed to validate the foreign key constraint - # when events are deleted. - self.db_pool.updates.register_background_index_update( - self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="current_state_events_stream_ordering_idx", - table="current_state_events", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="room_memberships_stream_ordering_idx", - table="room_memberships", - columns=["event_stream_ordering"], - ) - self.db_pool.updates.register_background_index_update( - self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, - index_name="local_current_membership_stream_ordering_idx", - table="local_current_membership", - columns=["event_stream_ordering"], - ) - async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index 378a15e038..b01653246a 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -38,6 +38,16 @@ class _BackgroundUpdates: EVENTS_JUMP_TO_DATE_INDEX = "events_jump_to_date_index" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + SLIDING_SYNC_PREFILL_JOINED_ROOMS_TO_RECALCULATE_TABLE_BG_UPDATE = ( "sliding_sync_prefill_joined_rooms_to_recalculate_table_bg_update" ) From 0779587f9f0f924ec4d83498737ab4b06874d148 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Jun 2025 15:32:55 +0100 Subject: [PATCH 394/691] Lift pausing on ratelimited requests to http layer (#18595) When a request gets ratelimited we (optionally) wait ~500ms before returning to mitigate clients that like to tightloop on request failures. However, this is currently implemented by pausing request processing when we check for ratelimits, which might be deep within request processing, and e.g. while locks are held. Instead, let's hoist the pause to the very top of the HTTP handler. Hopefully, this mitigates the issue where a user sending lots of events to a single room can see their requests time out due to the combination of the linearizer and the pausing of the request. Instead, they should see the requests 429 after ~500ms. The first commit is a refactor to pass the `Clock` to `AsyncResource`, the second commit is the behavioural change. --- changelog.d/18595.misc | 1 + synapse/api/errors.py | 8 ++++++- synapse/api/ratelimiting.py | 4 +--- synapse/http/additional_resource.py | 2 +- synapse/http/proxy.py | 2 +- synapse/http/server.py | 21 +++++++++++++------ synapse/rest/consent/consent_resource.py | 2 +- .../synapse/client/federation_whitelist.py | 2 +- synapse/rest/synapse/client/jwks.py | 2 +- .../rest/synapse/client/new_user_consent.py | 2 +- .../oidc/backchannel_logout_resource.py | 2 +- .../synapse/client/oidc/callback_resource.py | 2 +- synapse/rest/synapse/client/password_reset.py | 2 +- synapse/rest/synapse/client/pick_idp.py | 2 +- synapse/rest/synapse/client/pick_username.py | 4 ++-- synapse/rest/synapse/client/rendezvous.py | 2 +- .../synapse/client/saml2/response_resource.py | 2 +- synapse/rest/synapse/client/sso_register.py | 2 +- synapse/rest/synapse/client/unsubscribe.py | 2 +- synapse/rest/well_known.py | 2 +- tests/test_server.py | 15 ++++++------- 21 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 changelog.d/18595.misc diff --git a/changelog.d/18595.misc b/changelog.d/18595.misc new file mode 100644 index 0000000000..b1a39b43f7 --- /dev/null +++ b/changelog.d/18595.misc @@ -0,0 +1 @@ +Better handling of ratelimited requests. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 3eb533f7d5..601a09efe1 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -527,7 +527,11 @@ class InvalidCaptchaError(SynapseError): class LimitExceededError(SynapseError): - """A client has sent too many requests and is being throttled.""" + """A client has sent too many requests and is being throttled. + + Args: + pause: Optional time in seconds to pause before responding to the client. + """ def __init__( self, @@ -535,6 +539,7 @@ class LimitExceededError(SynapseError): code: int = 429, retry_after_ms: Optional[int] = None, errcode: str = Codes.LIMIT_EXCEEDED, + pause: Optional[float] = None, ): # Use HTTP header Retry-After to enable library-assisted retry handling. headers = ( @@ -545,6 +550,7 @@ class LimitExceededError(SynapseError): super().__init__(code, "Too Many Requests", errcode, headers=headers) self.retry_after_ms = retry_after_ms self.limiter_name = limiter_name + self.pause = pause def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict": return cs_error(self.msg, self.errcode, retry_after_ms=self.retry_after_ms) diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 4f3bf8f770..509ef6b2c1 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -338,12 +338,10 @@ class Ratelimiter: ) if not allowed: - if pause: - await self.clock.sleep(pause) - raise LimitExceededError( limiter_name=self._limiter_name, retry_after_ms=int(1000 * (time_allowed - time_now_s)), + pause=pause, ) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index 2b9830b540..e65af85570 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -53,7 +53,7 @@ class AdditionalResource(DirectServeJsonResource): hs: homeserver handler: function to be called to handle the request. """ - super().__init__() + super().__init__(hs.get_clock()) self._handler = handler async def _async_render(self, request: Request) -> Optional[Tuple[int, Any]]: diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 5cd990b0d0..9b044f3b0a 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -106,7 +106,7 @@ class ProxyResource(_AsyncResource): isLeaf = True def __init__(self, reactor: ISynapseReactor, hs: "HomeServer"): - super().__init__(True) + super().__init__(hs.get_clock(), True) self.reactor = reactor self.agent = hs.get_federation_http_client().agent diff --git a/synapse/http/server.py b/synapse/http/server.py index 311e21774f..ec99aa39f9 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -67,6 +67,7 @@ from twisted.web.util import redirectTo from synapse.api.errors import ( CodeMessageException, Codes, + LimitExceededError, RedirectException, SynapseError, UnrecognizedRequestError, @@ -74,7 +75,7 @@ from synapse.api.errors import ( from synapse.config.homeserver import HomeServerConfig from synapse.logging.context import defer_to_thread, preserve_fn, run_in_background from synapse.logging.opentracing import active_span, start_active_span, trace_servlet -from synapse.util import json_encoder +from synapse.util import Clock, json_encoder from synapse.util.caches import intern_dict from synapse.util.cancellation import is_function_cancellable from synapse.util.iterutils import chunk_seq @@ -308,9 +309,10 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): context from the request the servlet is handling. """ - def __init__(self, extract_context: bool = False): + def __init__(self, clock: Clock, extract_context: bool = False): super().__init__() + self._clock = clock self._extract_context = extract_context def render(self, request: "SynapseRequest") -> int: @@ -329,7 +331,12 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): request.request_metrics.name = self.__class__.__name__ with trace_servlet(request, self._extract_context): - callback_return = await self._async_render(request) + try: + callback_return = await self._async_render(request) + except LimitExceededError as e: + if e.pause: + self._clock.sleep(e.pause) + raise if callback_return is not None: code, response = callback_return @@ -393,8 +400,10 @@ class DirectServeJsonResource(_AsyncResource): formatting responses and errors as JSON. """ - def __init__(self, canonical_json: bool = False, extract_context: bool = False): - super().__init__(extract_context) + def __init__( + self, clock: Clock, canonical_json: bool = False, extract_context: bool = False + ): + super().__init__(clock, extract_context) self.canonical_json = canonical_json def _send_response( @@ -450,8 +459,8 @@ class JsonResource(DirectServeJsonResource): canonical_json: bool = True, extract_context: bool = False, ): - super().__init__(canonical_json, extract_context) self.clock = hs.get_clock() + super().__init__(self.clock, canonical_json, extract_context) # Map of path regex -> method -> callback. self._routes: Dict[Pattern[str], Dict[bytes, _PathEntry]] = {} self.hs = hs diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index ff5dc51a01..a20eaf70e7 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -81,7 +81,7 @@ class ConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self.hs = hs self.store = hs.get_datastores().main diff --git a/synapse/rest/synapse/client/federation_whitelist.py b/synapse/rest/synapse/client/federation_whitelist.py index 2b8f0320e0..bccc3c3f37 100644 --- a/synapse/rest/synapse/client/federation_whitelist.py +++ b/synapse/rest/synapse/client/federation_whitelist.py @@ -44,7 +44,7 @@ class FederationWhitelistResource(DirectServeJsonResource): PATH = "/_synapse/client/v1/config/federation_whitelist" def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._federation_whitelist = hs.config.federation.federation_domain_whitelist diff --git a/synapse/rest/synapse/client/jwks.py b/synapse/rest/synapse/client/jwks.py index 5f581d3445..dd70a4015a 100644 --- a/synapse/rest/synapse/client/jwks.py +++ b/synapse/rest/synapse/client/jwks.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) class JwksResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__(extract_context=True) + super().__init__(hs.get_clock(), extract_context=True) # Parameters that are allowed to be exposed in the public key. # This is done manually, because authlib's private to public key conversion diff --git a/synapse/rest/synapse/client/new_user_consent.py b/synapse/rest/synapse/client/new_user_consent.py index 8b00b8c012..dfe5dadf60 100644 --- a/synapse/rest/synapse/client/new_user_consent.py +++ b/synapse/rest/synapse/client/new_user_consent.py @@ -48,7 +48,7 @@ class NewUserConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._server_name = hs.hostname self._consent_version = hs.config.consent.user_consent_version diff --git a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py index 3f4cf16934..abc39234a7 100644 --- a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py +++ b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py @@ -35,7 +35,7 @@ class OIDCBackchannelLogoutResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_POST(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/oidc/callback_resource.py b/synapse/rest/synapse/client/oidc/callback_resource.py index 84077684d4..6ce259d0ae 100644 --- a/synapse/rest/synapse/client/oidc/callback_resource.py +++ b/synapse/rest/synapse/client/oidc/callback_resource.py @@ -35,7 +35,7 @@ class OIDCCallbackResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/password_reset.py b/synapse/rest/synapse/client/password_reset.py index 29e4b2d07a..28f924f4c7 100644 --- a/synapse/rest/synapse/client/password_reset.py +++ b/synapse/rest/synapse/client/password_reset.py @@ -47,7 +47,7 @@ class PasswordResetSubmitTokenResource(DirectServeHtmlResource): Args: hs: server """ - super().__init__() + super().__init__(hs.get_clock()) self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py index 5e599f85b0..e980e1e2ab 100644 --- a/synapse/rest/synapse/client/pick_idp.py +++ b/synapse/rest/synapse/client/pick_idp.py @@ -44,7 +44,7 @@ class PickIdpResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._sso_login_idp_picker_template = ( hs.config.sso.sso_login_idp_picker_template diff --git a/synapse/rest/synapse/client/pick_username.py b/synapse/rest/synapse/client/pick_username.py index 7d16b796d4..e11c3f4302 100644 --- a/synapse/rest/synapse/client/pick_username.py +++ b/synapse/rest/synapse/client/pick_username.py @@ -62,7 +62,7 @@ def pick_username_resource(hs: "HomeServer") -> Resource: class AvailabilityCheckResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> Tuple[int, JsonDict]: @@ -78,7 +78,7 @@ class AvailabilityCheckResource(DirectServeJsonResource): class AccountDetailsResource(DirectServeHtmlResource): def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._sso_handler = hs.get_sso_handler() def template_search_dirs() -> Generator[str, None, None]: diff --git a/synapse/rest/synapse/client/rendezvous.py b/synapse/rest/synapse/client/rendezvous.py index 5216d30d1f..ceb61ef7a1 100644 --- a/synapse/rest/synapse/client/rendezvous.py +++ b/synapse/rest/synapse/client/rendezvous.py @@ -30,7 +30,7 @@ class MSC4108RendezvousSessionResource(DirectServeJsonResource): isLeaf = True def __init__(self, hs: "HomeServer") -> None: - super().__init__() + super().__init__(hs.get_clock()) self._handler = hs.get_rendezvous_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/saml2/response_resource.py b/synapse/rest/synapse/client/saml2/response_resource.py index 7b8667e04c..74e22f49bc 100644 --- a/synapse/rest/synapse/client/saml2/response_resource.py +++ b/synapse/rest/synapse/client/saml2/response_resource.py @@ -35,7 +35,7 @@ class SAML2ResponseResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._saml_handler = hs.get_saml_handler() self._sso_handler = hs.get_sso_handler() diff --git a/synapse/rest/synapse/client/sso_register.py b/synapse/rest/synapse/client/sso_register.py index be562ad8dc..1a5afddaee 100644 --- a/synapse/rest/synapse/client/sso_register.py +++ b/synapse/rest/synapse/client/sso_register.py @@ -43,7 +43,7 @@ class SsoRegisterResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> None: diff --git a/synapse/rest/synapse/client/unsubscribe.py b/synapse/rest/synapse/client/unsubscribe.py index 6d4bd9f2ed..b5077d5741 100644 --- a/synapse/rest/synapse/client/unsubscribe.py +++ b/synapse/rest/synapse/client/unsubscribe.py @@ -38,7 +38,7 @@ class UnsubscribeResource(DirectServeHtmlResource): SUCCESS_HTML = b"You have been unsubscribed" def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self.notifier = hs.get_notifier() self.auth = hs.get_auth() self.pusher_pool = hs.get_pusherpool() diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index d336d60c93..45d81af842 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -86,7 +86,7 @@ class ClientWellKnownResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__() + super().__init__(hs.get_clock()) self._well_known_builder = WellKnownBuilder(hs) async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: diff --git a/tests/test_server.py b/tests/test_server.py index 9ebfbf96e7..b7e62ba0e6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -316,15 +316,16 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): await self.callback(request) def setUp(self) -> None: - reactor, _ = get_clock() + reactor, clock = get_clock() self.reactor = reactor + self.clock = clock def test_good_response(self) -> None: async def callback(request: SynapseRequest) -> None: request.write(b"response") request.finish() - res = WrapHtmlRequestHandlerTests.TestResource() + res = WrapHtmlRequestHandlerTests.TestResource(self.clock) res.callback = callback channel = make_request( @@ -344,7 +345,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): async def callback(request: SynapseRequest, **kwargs: object) -> None: raise RedirectException(b"/look/an/eagle", 301) - res = WrapHtmlRequestHandlerTests.TestResource() + res = WrapHtmlRequestHandlerTests.TestResource(self.clock) res.callback = callback channel = make_request( @@ -366,7 +367,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): e.cookies.append(b"session=yespls") raise e - res = WrapHtmlRequestHandlerTests.TestResource() + res = WrapHtmlRequestHandlerTests.TestResource(self.clock) res.callback = callback channel = make_request( @@ -387,7 +388,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): request.write(b"response") request.finish() - res = WrapHtmlRequestHandlerTests.TestResource() + res = WrapHtmlRequestHandlerTests.TestResource(self.clock) res.callback = callback channel = make_request( @@ -400,7 +401,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): class CancellableDirectServeJsonResource(DirectServeJsonResource): def __init__(self, clock: Clock): - super().__init__() + super().__init__(clock) self.clock = clock @cancellable @@ -417,7 +418,7 @@ class CancellableDirectServeHtmlResource(DirectServeHtmlResource): ERROR_TEMPLATE = "{code} {msg}" def __init__(self, clock: Clock): - super().__init__() + super().__init__(clock) self.clock = clock @cancellable From 62b5b0b96255c2a1cb66d9cefd104cc638fd0392 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 15:45:28 +0100 Subject: [PATCH 395/691] Bump reqwest from 0.12.15 to 0.12.20 (#18590) --- Cargo.lock | 178 +++++++++++++++-------------------------------------- 1 file changed, 49 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39d9a9f20f..b85f5f8b98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,7 +62,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -331,7 +331,7 @@ dependencies = [ "cfg-if", "libc", "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -487,17 +487,21 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -666,6 +670,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1041,9 +1055,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ "base64 0.22.1", "bytes", @@ -1056,17 +1070,13 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", "rustls", "rustls-native-certs", - "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", @@ -1076,13 +1086,13 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "windows-registry", ] [[package]] @@ -1137,15 +1147,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.11.0" @@ -1479,6 +1480,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -1707,48 +1726,13 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "windows-link" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" - -[[package]] -name = "windows-registry" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" -dependencies = [ - "windows-result", - "windows-strings", - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-result" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1757,7 +1741,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1766,30 +1750,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -1798,96 +1766,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "wit-bindgen-rt" version = "0.33.0" From ec13ed4169d9709d3fe19ac7955ea97b3c2f4f3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 15:46:10 +0100 Subject: [PATCH 396/691] Bump docker/setup-buildx-action from 3.10.0 to 3.11.1 (#18587) --- .github/workflows/docker.yml | 2 +- .github/workflows/release-artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index feeadf170d..357f146e10 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Inspect builder run: docker buildx inspect diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 572d73e6ad..3e2e76d18d 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -61,7 +61,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: install: true From 99474e7fdfc69529242ea0dabde657f0f1317185 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 15:49:25 +0100 Subject: [PATCH 397/691] Bump sigstore/cosign-installer from 3.8.2 to 3.9.0 (#18588) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 357f146e10..3bff17ed7a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 + uses: sigstore/cosign-installer@fb28c2b6339dcd94da6e4cbcbc5e888961f6f8c3 # v3.9.0 - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From 3807fd42e183ba73827fc07be07b73b1b166d717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 15:50:11 +0100 Subject: [PATCH 398/691] Bump urllib3 from 2.2.2 to 2.5.0 (#18572) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index f3d807f6ac..4355d54cb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3123,14 +3123,14 @@ files = [ [[package]] name = "urllib3" -version = "2.2.2" +version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, + {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] [package.extras] From 2f21b27465569ba43168ae3fe81de5843fcc812a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 17:00:19 +0100 Subject: [PATCH 399/691] Bump pyasn1-modules from 0.4.1 to 0.4.2 (#18495) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4355d54cb2..e3463aba32 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1771,18 +1771,18 @@ files = [ [[package]] name = "pyasn1-modules" -version = "0.4.1" +version = "0.4.2" description = "A collection of ASN.1-based protocols modules" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, - {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, ] [package.dependencies] -pyasn1 = ">=0.4.6,<0.7.0" +pyasn1 = ">=0.6.1,<0.7.0" [[package]] name = "pycparser" From b088194f4807af226648a70efeda3909fd275999 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 17:12:24 +0100 Subject: [PATCH 400/691] Bump docker/build-push-action from 6.17.0 to 6.18.0 (#18497) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3bff17ed7a..e31b6aea90 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -72,7 +72,7 @@ jobs: - name: Build and push all platforms id: build-and-push - uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: push: true labels: | From b1396475c489cd6d80ce37076fd9b55d31e20fe4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 17:22:39 +0100 Subject: [PATCH 401/691] Bump base64 from 0.21.7 to 0.22.1 (#18589) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b85f5f8b98..636b4f5a62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,12 +65,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -371,7 +365,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "headers-core", "http", @@ -491,7 +485,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -1059,7 +1053,7 @@ version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-util", @@ -1333,7 +1327,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.7", + "base64", "blake2", "bytes", "futures", From 434e38941a6b169c1905cb10ff9bd6907c765209 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 Jun 2025 13:27:21 +0200 Subject: [PATCH 402/691] Add `federated_user_may_invite` spam checker callback (#18241) Co-authored-by: Sebastian Spaeth Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18241.feature | 1 + docs/modules/spam_checker_callbacks.md | 30 +++++++++ synapse/handlers/federation.py | 4 +- synapse/module_api/__init__.py | 3 + .../callbacks/spamchecker_callbacks.py | 63 +++++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18241.feature diff --git a/changelog.d/18241.feature b/changelog.d/18241.feature new file mode 100644 index 0000000000..01760c9d67 --- /dev/null +++ b/changelog.d/18241.feature @@ -0,0 +1 @@ +Add `federated_user_may_invite` spam checker callback which receives the entire invite event. Contributed by @tulir @ Beeper. diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 39d7cbc000..49b7e06bb3 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -80,6 +80,8 @@ Called when processing an invitation, both when one is created locally or when receiving an invite over federation. Both inviter and invitee are represented by their Matrix user ID (e.g. `@alice:example.com`). +Note that federated invites will call `federated_user_may_invite` before this callback. + The callback must return one of: - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still @@ -97,6 +99,34 @@ be used. If this happens, Synapse will not call any of the subsequent implementa this callback. +### `federated_user_may_invite` + +_First introduced in Synapse v1.133.0_ + +```python +async def federated_user_may_invite(event: "synapse.events.EventBase") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] +``` + +Called when processing an invitation received over federation. Unlike `user_may_invite`, +this callback receives the entire event, including any stripped state in the `unsigned` +section, not just the room and user IDs. + +The callback must return one of: + - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. + - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. +The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback. + +If all of the callbacks return `synapse.module_api.NOT_SPAM`, Synapse will also fall +through to the `user_may_invite` callback before approving the invite. + + ### `user_may_send_3pid_invite` _First introduced in Synapse v1.45.0_ diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 729b676163..015fb3edca 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1062,8 +1062,8 @@ class FederationHandler: if self.hs.config.server.block_non_admin_invites: raise SynapseError(403, "This server does not accept room invites") - spam_check = await self._spam_checker_module_callbacks.user_may_invite( - event.sender, event.state_key, event.room_id + spam_check = ( + await self._spam_checker_module_callbacks.federated_user_may_invite(event) ) if spam_check != NOT_SPAM: raise SynapseError( diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 656fe323f3..b1a2476da8 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -104,6 +104,7 @@ from synapse.module_api.callbacks.spamchecker_callbacks import ( CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK, CHECK_REGISTRATION_FOR_SPAM_CALLBACK, CHECK_USERNAME_FOR_SPAM_CALLBACK, + FEDERATED_USER_MAY_INVITE_CALLBACK, SHOULD_DROP_FEDERATED_EVENT_CALLBACK, USER_MAY_CREATE_ROOM_ALIAS_CALLBACK, USER_MAY_CREATE_ROOM_CALLBACK, @@ -315,6 +316,7 @@ class ModuleApi: ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, + federated_user_may_invite: Optional[FEDERATED_USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None, user_may_create_room_alias: Optional[ @@ -338,6 +340,7 @@ class ModuleApi: should_drop_federated_event=should_drop_federated_event, user_may_join_room=user_may_join_room, user_may_invite=user_may_invite, + federated_user_may_invite=federated_user_may_invite, user_may_send_3pid_invite=user_may_send_3pid_invite, user_may_create_room=user_may_create_room, user_may_create_room_alias=user_may_create_room_alias, diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index bea5c5badf..c43824f213 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -105,6 +105,22 @@ USER_MAY_INVITE_CALLBACK = Callable[ ] ], ] +FEDERATED_USER_MAY_INVITE_CALLBACK = Callable[ + ["synapse.events.EventBase"], + Awaitable[ + Union[ + Literal["NOT_SPAM"], + Codes, + # Highly experimental, not officially part of the spamchecker API, may + # disappear without warning depending on the results of ongoing + # experiments. + # Use this to return additional information as part of an error. + Tuple[Codes, JsonDict], + # Deprecated + bool, + ] + ], +] USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[ [str, str, str, str], Awaitable[ @@ -266,6 +282,7 @@ def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None: spam_checker_methods = { "check_event_for_spam", "user_may_invite", + "federated_user_may_invite", "user_may_create_room", "user_may_create_room_alias", "user_may_publish_room", @@ -347,6 +364,9 @@ class SpamCheckerModuleApiCallbacks: ] = [] self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = [] self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] + self._federated_user_may_invite_callbacks: List[ + FEDERATED_USER_MAY_INVITE_CALLBACK + ] = [] self._user_may_send_3pid_invite_callbacks: List[ USER_MAY_SEND_3PID_INVITE_CALLBACK ] = [] @@ -377,6 +397,7 @@ class SpamCheckerModuleApiCallbacks: ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, + federated_user_may_invite: Optional[FEDERATED_USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None, user_may_create_room_alias: Optional[ @@ -406,6 +427,11 @@ class SpamCheckerModuleApiCallbacks: if user_may_invite is not None: self._user_may_invite_callbacks.append(user_may_invite) + if federated_user_may_invite is not None: + self._federated_user_may_invite_callbacks.append( + federated_user_may_invite, + ) + if user_may_send_3pid_invite is not None: self._user_may_send_3pid_invite_callbacks.append( user_may_send_3pid_invite, @@ -605,6 +631,43 @@ class SpamCheckerModuleApiCallbacks: # No spam-checker has rejected the request, let it pass. return self.NOT_SPAM + async def federated_user_may_invite( + self, event: "synapse.events.EventBase" + ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: + """Checks if a given user may send an invite + + Args: + event: The event to be checked + + Returns: + NOT_SPAM if the operation is permitted, Codes otherwise. + """ + for callback in self._federated_user_may_invite_callbacks: + with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + res = await delay_cancellation(callback(event)) + # Normalize return values to `Codes` or `"NOT_SPAM"`. + if res is True or res is self.NOT_SPAM: + continue + elif res is False: + return synapse.api.errors.Codes.FORBIDDEN, {} + elif isinstance(res, synapse.api.errors.Codes): + return res, {} + elif ( + isinstance(res, tuple) + and len(res) == 2 + and isinstance(res[0], synapse.api.errors.Codes) + and isinstance(res[1], dict) + ): + return res + else: + logger.warning( + "Module returned invalid value, rejecting invite as spam" + ) + return synapse.api.errors.Codes.FORBIDDEN, {} + + # Check the standard user_may_invite callback if no module has rejected the invite yet. + return await self.user_may_invite(event.sender, event.state_key, event.room_id) + async def user_may_send_3pid_invite( self, inviter_userid: str, medium: str, address: str, room_id: str ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]: From de29c13d41e9ae0b469597ae86a755db70df1fcd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Jun 2025 15:05:48 +0100 Subject: [PATCH 403/691] Fix backwards compat for `DirectServeJsonResource` (#18600) As that appears in the module API. Broke in #18595. --- Cargo.lock | 14 +++++++--- changelog.d/18600.misc | 1 + synapse/http/additional_resource.py | 2 +- synapse/http/server.py | 26 ++++++++++++++++--- synapse/rest/consent/consent_resource.py | 2 +- .../synapse/client/federation_whitelist.py | 2 +- synapse/rest/synapse/client/jwks.py | 2 +- .../rest/synapse/client/new_user_consent.py | 2 +- .../oidc/backchannel_logout_resource.py | 2 +- .../synapse/client/oidc/callback_resource.py | 2 +- synapse/rest/synapse/client/password_reset.py | 2 +- synapse/rest/synapse/client/pick_idp.py | 2 +- synapse/rest/synapse/client/pick_username.py | 4 +-- synapse/rest/synapse/client/rendezvous.py | 2 +- .../synapse/client/saml2/response_resource.py | 2 +- synapse/rest/synapse/client/sso_register.py | 2 +- synapse/rest/synapse/client/unsubscribe.py | 2 +- synapse/rest/well_known.py | 2 +- tests/test_server.py | 12 ++++----- 19 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 changelog.d/18600.misc diff --git a/Cargo.lock b/Cargo.lock index 636b4f5a62..b85f5f8b98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -365,7 +371,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", "http", @@ -485,7 +491,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -1053,7 +1059,7 @@ version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -1327,7 +1333,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "blake2", "bytes", "futures", diff --git a/changelog.d/18600.misc b/changelog.d/18600.misc new file mode 100644 index 0000000000..b1a39b43f7 --- /dev/null +++ b/changelog.d/18600.misc @@ -0,0 +1 @@ +Better handling of ratelimited requests. diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index e65af85570..59eae841d5 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -53,7 +53,7 @@ class AdditionalResource(DirectServeJsonResource): hs: homeserver handler: function to be called to handle the request. """ - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._handler = handler async def _async_render(self, request: Request) -> Optional[Tuple[int, Any]]: diff --git a/synapse/http/server.py b/synapse/http/server.py index ec99aa39f9..395d82fd16 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -42,6 +42,7 @@ from typing import ( Protocol, Tuple, Union, + cast, ) import attr @@ -49,8 +50,9 @@ import jinja2 from canonicaljson import encode_canonical_json from zope.interface import implementer -from twisted.internet import defer, interfaces +from twisted.internet import defer, interfaces, reactor from twisted.internet.defer import CancelledError +from twisted.internet.interfaces import IReactorTime from twisted.python import failure from twisted.web import resource @@ -401,8 +403,15 @@ class DirectServeJsonResource(_AsyncResource): """ def __init__( - self, clock: Clock, canonical_json: bool = False, extract_context: bool = False + self, + canonical_json: bool = False, + extract_context: bool = False, + # Clock is optional as this class is exposed to the module API. + clock: Optional[Clock] = None, ): + if clock is None: + clock = Clock(cast(IReactorTime, reactor)) + super().__init__(clock, extract_context) self.canonical_json = canonical_json @@ -460,7 +469,7 @@ class JsonResource(DirectServeJsonResource): extract_context: bool = False, ): self.clock = hs.get_clock() - super().__init__(self.clock, canonical_json, extract_context) + super().__init__(canonical_json, extract_context, clock=self.clock) # Map of path regex -> method -> callback. self._routes: Dict[Pattern[str], Dict[bytes, _PathEntry]] = {} self.hs = hs @@ -573,6 +582,17 @@ class DirectServeHtmlResource(_AsyncResource): # The error template to use for this resource ERROR_TEMPLATE = HTML_ERROR_TEMPLATE + def __init__( + self, + extract_context: bool = False, + # Clock is optional as this class is exposed to the module API. + clock: Optional[Clock] = None, + ): + if clock is None: + clock = Clock(cast(IReactorTime, reactor)) + + super().__init__(clock, extract_context) + def _send_response( self, request: "SynapseRequest", diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index a20eaf70e7..3961f82894 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -81,7 +81,7 @@ class ConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self.hs = hs self.store = hs.get_datastores().main diff --git a/synapse/rest/synapse/client/federation_whitelist.py b/synapse/rest/synapse/client/federation_whitelist.py index bccc3c3f37..f59daf8428 100644 --- a/synapse/rest/synapse/client/federation_whitelist.py +++ b/synapse/rest/synapse/client/federation_whitelist.py @@ -44,7 +44,7 @@ class FederationWhitelistResource(DirectServeJsonResource): PATH = "/_synapse/client/v1/config/federation_whitelist" def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._federation_whitelist = hs.config.federation.federation_domain_whitelist diff --git a/synapse/rest/synapse/client/jwks.py b/synapse/rest/synapse/client/jwks.py index dd70a4015a..e9a7c24e3b 100644 --- a/synapse/rest/synapse/client/jwks.py +++ b/synapse/rest/synapse/client/jwks.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) class JwksResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock(), extract_context=True) + super().__init__(clock=hs.get_clock(), extract_context=True) # Parameters that are allowed to be exposed in the public key. # This is done manually, because authlib's private to public key conversion diff --git a/synapse/rest/synapse/client/new_user_consent.py b/synapse/rest/synapse/client/new_user_consent.py index dfe5dadf60..c7bd8de482 100644 --- a/synapse/rest/synapse/client/new_user_consent.py +++ b/synapse/rest/synapse/client/new_user_consent.py @@ -48,7 +48,7 @@ class NewUserConsentResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._server_name = hs.hostname self._consent_version = hs.config.consent.user_consent_version diff --git a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py index abc39234a7..114c8c2d82 100644 --- a/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py +++ b/synapse/rest/synapse/client/oidc/backchannel_logout_resource.py @@ -35,7 +35,7 @@ class OIDCBackchannelLogoutResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_POST(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/oidc/callback_resource.py b/synapse/rest/synapse/client/oidc/callback_resource.py index 6ce259d0ae..cc0213c50c 100644 --- a/synapse/rest/synapse/client/oidc/callback_resource.py +++ b/synapse/rest/synapse/client/oidc/callback_resource.py @@ -35,7 +35,7 @@ class OIDCCallbackResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._oidc_handler = hs.get_oidc_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/password_reset.py b/synapse/rest/synapse/client/password_reset.py index 28f924f4c7..377578ef8a 100644 --- a/synapse/rest/synapse/client/password_reset.py +++ b/synapse/rest/synapse/client/password_reset.py @@ -47,7 +47,7 @@ class PasswordResetSubmitTokenResource(DirectServeHtmlResource): Args: hs: server """ - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py index e980e1e2ab..9668a09c19 100644 --- a/synapse/rest/synapse/client/pick_idp.py +++ b/synapse/rest/synapse/client/pick_idp.py @@ -44,7 +44,7 @@ class PickIdpResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() self._sso_login_idp_picker_template = ( hs.config.sso.sso_login_idp_picker_template diff --git a/synapse/rest/synapse/client/pick_username.py b/synapse/rest/synapse/client/pick_username.py index e11c3f4302..1727bb63b7 100644 --- a/synapse/rest/synapse/client/pick_username.py +++ b/synapse/rest/synapse/client/pick_username.py @@ -62,7 +62,7 @@ def pick_username_resource(hs: "HomeServer") -> Resource: class AvailabilityCheckResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> Tuple[int, JsonDict]: @@ -78,7 +78,7 @@ class AvailabilityCheckResource(DirectServeJsonResource): class AccountDetailsResource(DirectServeHtmlResource): def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() def template_search_dirs() -> Generator[str, None, None]: diff --git a/synapse/rest/synapse/client/rendezvous.py b/synapse/rest/synapse/client/rendezvous.py index ceb61ef7a1..5278c35572 100644 --- a/synapse/rest/synapse/client/rendezvous.py +++ b/synapse/rest/synapse/client/rendezvous.py @@ -30,7 +30,7 @@ class MSC4108RendezvousSessionResource(DirectServeJsonResource): isLeaf = True def __init__(self, hs: "HomeServer") -> None: - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._handler = hs.get_rendezvous_handler() async def _async_render_GET(self, request: SynapseRequest) -> None: diff --git a/synapse/rest/synapse/client/saml2/response_resource.py b/synapse/rest/synapse/client/saml2/response_resource.py index 74e22f49bc..d2cf4f21f1 100644 --- a/synapse/rest/synapse/client/saml2/response_resource.py +++ b/synapse/rest/synapse/client/saml2/response_resource.py @@ -35,7 +35,7 @@ class SAML2ResponseResource(DirectServeHtmlResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._saml_handler = hs.get_saml_handler() self._sso_handler = hs.get_sso_handler() diff --git a/synapse/rest/synapse/client/sso_register.py b/synapse/rest/synapse/client/sso_register.py index 1a5afddaee..74d9bdee87 100644 --- a/synapse/rest/synapse/client/sso_register.py +++ b/synapse/rest/synapse/client/sso_register.py @@ -43,7 +43,7 @@ class SsoRegisterResource(DirectServeHtmlResource): """ def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._sso_handler = hs.get_sso_handler() async def _async_render_GET(self, request: Request) -> None: diff --git a/synapse/rest/synapse/client/unsubscribe.py b/synapse/rest/synapse/client/unsubscribe.py index b5077d5741..90837611dd 100644 --- a/synapse/rest/synapse/client/unsubscribe.py +++ b/synapse/rest/synapse/client/unsubscribe.py @@ -38,7 +38,7 @@ class UnsubscribeResource(DirectServeHtmlResource): SUCCESS_HTML = b"You have been unsubscribed" def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self.notifier = hs.get_notifier() self.auth = hs.get_auth() self.pusher_pool = hs.get_pusherpool() diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index 45d81af842..b4476e5a69 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -86,7 +86,7 @@ class ClientWellKnownResource(DirectServeJsonResource): isLeaf = 1 def __init__(self, hs: "HomeServer"): - super().__init__(hs.get_clock()) + super().__init__(clock=hs.get_clock()) self._well_known_builder = WellKnownBuilder(hs) async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: diff --git a/tests/test_server.py b/tests/test_server.py index b7e62ba0e6..0aa3584518 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -325,7 +325,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): request.write(b"response") request.finish() - res = WrapHtmlRequestHandlerTests.TestResource(self.clock) + res = WrapHtmlRequestHandlerTests.TestResource(clock=self.clock) res.callback = callback channel = make_request( @@ -345,7 +345,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): async def callback(request: SynapseRequest, **kwargs: object) -> None: raise RedirectException(b"/look/an/eagle", 301) - res = WrapHtmlRequestHandlerTests.TestResource(self.clock) + res = WrapHtmlRequestHandlerTests.TestResource(clock=self.clock) res.callback = callback channel = make_request( @@ -367,7 +367,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): e.cookies.append(b"session=yespls") raise e - res = WrapHtmlRequestHandlerTests.TestResource(self.clock) + res = WrapHtmlRequestHandlerTests.TestResource(clock=self.clock) res.callback = callback channel = make_request( @@ -388,7 +388,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): request.write(b"response") request.finish() - res = WrapHtmlRequestHandlerTests.TestResource(self.clock) + res = WrapHtmlRequestHandlerTests.TestResource(clock=self.clock) res.callback = callback channel = make_request( @@ -401,7 +401,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): class CancellableDirectServeJsonResource(DirectServeJsonResource): def __init__(self, clock: Clock): - super().__init__(clock) + super().__init__(clock=clock) self.clock = clock @cancellable @@ -418,7 +418,7 @@ class CancellableDirectServeHtmlResource(DirectServeHtmlResource): ERROR_TEMPLATE = "{code} {msg}" def __init__(self, clock: Clock): - super().__init__(clock) + super().__init__(clock=clock) self.clock = clock @cancellable From db710cf29b487db917b946b45aa57f2fdc509ef4 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Fri, 27 Jun 2025 22:07:24 +0200 Subject: [PATCH 404/691] Add `forget_forced_upon_leave` capability as per MSC4267 (#18196) This adds the capability from https://github.com/matrix-org/matrix-spec-proposals/pull/4267 under an experimental feature. Signed-off-by: Johannes Marbach --- changelog.d/18196.feature | 1 + synapse/config/experimental.py | 3 ++ synapse/config/room.py | 2 +- synapse/rest/client/capabilities.py | 5 ++++ tests/rest/client/test_capabilities.py | 40 ++++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18196.feature diff --git a/changelog.d/18196.feature b/changelog.d/18196.feature new file mode 100644 index 0000000000..6f4a67b519 --- /dev/null +++ b/changelog.d/18196.feature @@ -0,0 +1 @@ +Add `forget_forced_upon_leave` capability as per [MSC4267](https://github.com/matrix-org/matrix-spec-proposals/pull/4267). diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 259b2c70cb..85eca9c05b 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -567,5 +567,8 @@ class ExperimentalConfig(Config): False, ) + # MSC4267: Automatically forgetting rooms on leave + self.msc4267_enabled: bool = experimental.get("msc4267_enabled", False) + # MSC4155: Invite filtering self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) diff --git a/synapse/config/room.py b/synapse/config/room.py index bad387ecbf..e698c7bafd 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -85,4 +85,4 @@ class RoomConfig(Config): # When enabled, users will forget rooms when they leave them, either via a # leave, kick or ban. - self.forget_on_leave = config.get("forget_rooms_on_leave", False) + self.forget_on_leave: bool = config.get("forget_rooms_on_leave", False) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index ebd5a33ea5..8f3193fb47 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ class CapabilitiesRestServlet(RestServlet): "disallowed" ] = disallowed + if self.config.experimental.msc4267_enabled: + response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { + "enabled": self.config.room.forget_on_leave, + } + return HTTPStatus.OK, response diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 8af00221c2..13831462e8 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -264,3 +264,43 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertTrue(capabilities["m.get_login_token"]["enabled"]) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": True, + } + ) + def test_get_forget_forced_upon_leave_with_auto_forget(self) -> None: + # Server auto-forgets on /leave, expect enabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) + + @override_config( + { + "experimental_features": {"msc4267_enabled": True}, + "forget_rooms_on_leave": False, + } + ) + def test_get_forget_forced_upon_leave_without_auto_forget(self) -> None: + # Server doesn't auto-forget on /leave, expect disabled client capability + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse( + capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] + ) From 8afea3d51d05a395e8cbbb65515f91ca9ba4af8a Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 30 Jun 2025 09:35:23 +0000 Subject: [PATCH 405/691] Improve docstring on `simple_upsert_many`. (#18573) It came up that this was somewhat confusing and an example might help. So here's an example :) --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18573.misc | 1 + synapse/storage/database.py | 46 +++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18573.misc diff --git a/changelog.d/18573.misc b/changelog.d/18573.misc new file mode 100644 index 0000000000..373852638f --- /dev/null +++ b/changelog.d/18573.misc @@ -0,0 +1 @@ +Improve docstring on `simple_upsert_many`. \ No newline at end of file diff --git a/synapse/storage/database.py b/synapse/storage/database.py index a4941e58f6..6188195614 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1478,13 +1478,49 @@ class DatabasePool: """ Upsert, many times. + This executes a query equivalent to `INSERT INTO ... ON CONFLICT DO UPDATE`, + with multiple value rows. + The query may use emulated upserts if the database engine does not support upserts, + or if the table is currently unsafe to upsert. + + If there are no value columns, this instead generates a `ON CONFLICT DO NOTHING`. + Args: table: The table to upsert into - key_names: The key column names. - key_values: A list of each row's key column values. - value_names: The value column names - value_values: A list of each row's value column values. + key_names: The unique key column names. These are the columns used in the ON CONFLICT clause. + key_values: A list of each row's key column values, in the same order as `key_names`. + value_names: The non-unique value column names + value_values: A list of each row's value column values, in the same order as `value_names`. Ignored if value_names is empty. + + Example: + ```python + simple_upsert_many( + "mytable", + key_names=("room_id", "user_id"), + key_values=[ + ("!room1:example.org", "@user1:example.org"), + ("!room2:example.org", "@user2:example.org"), + ], + value_names=("wombat_count", "is_updated"), + value_values=[ + (42, True), + (7, False) + ], + ) + ``` + + gives something equivalent to: + + ```sql + INSERT INTO mytable (room_id, user_id, wombat_count, is_updated) + VALUES + ('!room1:example.org', '@user1:example.org', 42, True), + ('!room2:example.org', '@user2:example.org', 7, False) + ON CONFLICT DO UPDATE SET + wombat_count = EXCLUDED.wombat_count, + is_updated = EXCLUDED.is_updated + ``` """ # We can autocommit if it safe to upsert @@ -1513,6 +1549,8 @@ class DatabasePool: """ Upsert, many times. + See the documentation for `simple_upsert_many` for examples. + Args: table: The table to upsert into key_names: The key column names. From bfb3a6e7005b06b85bb3a66b7d2e0bf99bc423ad Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 30 Jun 2025 09:36:12 +0000 Subject: [PATCH 406/691] Improve performance of device deletion by adding missing index. (#18582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit
  1. Reorder columns in `event_txn_id_device_id_txn_id` index \ This now satisfies the foreign key on `(user_id, device_id)` making reverse lookups, as needed for device deletions, more efficient. This improves device deletion performance by on the order of 8 to 10× on matrix.org.
Rationale: ## On the `event_txn_id_device_id` table: We currently have this index: ```sql -- This ensures that there is only one mapping per (room_id, user_id, device_id, txn_id) tuple. CREATE UNIQUE INDEX IF NOT EXISTS event_txn_id_device_id_txn_id ON event_txn_id_device_id(room_id, user_id, device_id, txn_id); ``` The main way we use this table is ```python return await self.db_pool.simple_select_one_onecol( table="event_txn_id_device_id", keyvalues={ "room_id": room_id, "user_id": user_id, "device_id": device_id, "txn_id": txn_id, }, retcol="event_id", allow_none=True, desc="get_event_id_from_transaction_id_and_device_id", ) ``` But this foreign key is relatively unsupported, making deletions in the devices table inefficient (full index scan on the above index): ```sql FOREIGN KEY (user_id, device_id) REFERENCES devices (user_id, device_id) ON DELETE CASCADE ``` I propose re-ordering the columns in that index to: `(user_id, device_id, room_id, txn_id)` (by replacing it). That way the foreign key back-check can rely on the prefix of this index, but it's still useful for the original purpose it was made for. It doesn't take any extra disk space and does not harm write performance (because the same amount of writing work needs to be performed). --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18582.bugfix | 1 + synapse/storage/databases/main/events_worker.py | 13 +++++++++++++ .../92/07_event_txn_id_device_id_txn_id2.sql | 15 +++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 changelog.d/18582.bugfix create mode 100644 synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql diff --git a/changelog.d/18582.bugfix b/changelog.d/18582.bugfix new file mode 100644 index 0000000000..7091bcc997 --- /dev/null +++ b/changelog.d/18582.bugfix @@ -0,0 +1 @@ +Improve performance of device deletion by adding missing index. \ No newline at end of file diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 3db4460f57..9cc0493307 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -349,6 +349,19 @@ class EventsWorkerStore(SQLBaseStore): where_clause="type = 'm.room.member'", ) + # Added to support efficient reverse lookups on the foreign key + # (user_id, device_id) when deleting devices. + # We already had a UNIQUE index on these 4 columns but out-of-order + # so replace that one. + self.db_pool.updates.register_background_index_update( + update_name="event_txn_id_device_id_txn_id2", + index_name="event_txn_id_device_id_txn_id2", + table="event_txn_id_device_id", + columns=("user_id", "device_id", "room_id", "txn_id"), + unique=True, + replaces_index="event_txn_id_device_id_txn_id", + ) + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( diff --git a/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql new file mode 100644 index 0000000000..b568fc04f2 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9207, 'event_txn_id_device_id_txn_id2', '{}'); From b35c6483d513dcc340961532f53ccaeb35f94f3f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 30 Jun 2025 04:45:23 -0600 Subject: [PATCH 407/691] Skip processing policy server events through policy server (#18605) Co-authored-by: Andrew Morgan --- changelog.d/18605.bugfix | 1 + synapse/handlers/room_policy.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/18605.bugfix diff --git a/changelog.d/18605.bugfix b/changelog.d/18605.bugfix new file mode 100644 index 0000000000..516f09e502 --- /dev/null +++ b/changelog.d/18605.bugfix @@ -0,0 +1 @@ +Ensure policy servers are not asked to scan policy server change events, allowing rooms to disable the use of a policy server while the policy server is down. diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index dcfebb128c..170c477d6f 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -54,6 +54,9 @@ class RoomPolicyHandler: Returns: bool: True if the event is allowed in the room, False otherwise. """ + if event.type == "org.matrix.msc4284.policy" and event.state_key is not None: + return True # always allow policy server change events + policy_event = await self._storage_controllers.state.get_current_state_event( event.room_id, "org.matrix.msc4284.policy", "" ) From 3878699df7a812031da50cfbbb1a76f4b935d045 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 30 Jun 2025 11:48:57 +0100 Subject: [PATCH 408/691] Speed up device deletion (#18602) This is to handle the case of deleting lots of "bot" devices at once. Reviewable commit-by-commit --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/18602.misc | 1 + synapse/handlers/auth.py | 27 ++++++- synapse/handlers/device.py | 47 +++++------ synapse/storage/databases/main/deviceinbox.py | 37 ++++++++- synapse/storage/databases/main/devices.py | 74 +++++++++++++---- .../storage/databases/main/end_to_end_keys.py | 44 +---------- .../storage/databases/main/registration.py | 79 ++++++++++++++++++- 7 files changed, 225 insertions(+), 84 deletions(-) create mode 100644 changelog.d/18602.misc diff --git a/changelog.d/18602.misc b/changelog.d/18602.misc new file mode 100644 index 0000000000..637d842682 --- /dev/null +++ b/changelog.d/18602.misc @@ -0,0 +1 @@ +Speed up bulk device deletion. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index de5b38caca..9e3e70ec1c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -76,7 +76,7 @@ from synapse.storage.databases.main.registration import ( LoginTokenLookupResult, LoginTokenReused, ) -from synapse.types import JsonDict, Requester, UserID +from synapse.types import JsonDict, Requester, StrCollection, UserID from synapse.util import stringutils as stringutils from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.msisdn import phone_number_to_msisdn @@ -1547,6 +1547,31 @@ class AuthHandler: user_id, (token_id for _, token_id, _ in tokens_and_devices) ) + async def delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> None: + """Invalidate access tokens for the devices + + Args: + user_id: ID of user the tokens belong to + device_ids: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + """ + tokens_and_devices = await self.store.user_delete_access_tokens_for_devices( + user_id, + device_ids, + ) + + # see if any modules want to know about this + if self.password_auth_provider.on_logged_out_callbacks: + for token, _, device_id in tokens_and_devices: + await self.password_auth_provider.on_logged_out( + user_id=user_id, device_id=device_id, access_token=token + ) + async def add_threepid( self, user_id: str, medium: str, address: str, validated_at: int ) -> None: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 8f9bf92fda..c6e44dae6a 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -671,12 +671,12 @@ class DeviceHandler(DeviceWorkerHandler): except_device_id: optional device id which should not be deleted """ device_map = await self.store.get_devices_by_user(user_id) - device_ids = list(device_map) if except_device_id is not None: - device_ids = [d for d in device_ids if d != except_device_id] - await self.delete_devices(user_id, device_ids) + device_map.pop(except_device_id, None) + user_device_ids = device_map.keys() + await self.delete_devices(user_id, user_device_ids) - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: """Delete several devices Args: @@ -695,17 +695,10 @@ class DeviceHandler(DeviceWorkerHandler): else: raise - # Delete data specific to each device. Not optimised as it is not - # considered as part of a critical path. - for device_id in device_ids: - await self._auth_handler.delete_access_tokens_for_user( - user_id, device_id=device_id - ) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=device_id - ) - - if self.hs.config.experimental.msc3890_enabled: + # Delete data specific to each device. Not optimised as its an + # experimental MSC. + if self.hs.config.experimental.msc3890_enabled: + for device_id in device_ids: # Remove any local notification settings for this device in accordance # with MSC3890. await self._account_data_handler.remove_account_data_for_user( @@ -713,6 +706,13 @@ class DeviceHandler(DeviceWorkerHandler): f"org.matrix.msc3890.local_notification_settings.{device_id}", ) + # If we're deleting a lot of devices, a bunch of them may not have any + # to-device messages queued up. We filter those out to avoid scheduling + # unnecessary tasks. + devices_with_messages = await self.store.get_devices_with_messages( + user_id, device_ids + ) + for device_id in devices_with_messages: # Delete device messages asynchronously and in batches using the task scheduler # We specify an upper stream id to avoid deleting non delivered messages # if an user re-uses a device ID. @@ -726,6 +726,10 @@ class DeviceHandler(DeviceWorkerHandler): }, ) + await self._auth_handler.delete_access_tokens_for_devices( + user_id, device_ids=device_ids + ) + # Pushers are deleted after `delete_access_tokens_for_user` is called so that # modules using `on_logged_out` hook can use them if needed. await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) @@ -819,10 +823,11 @@ class DeviceHandler(DeviceWorkerHandler): # This should only happen if there are no updates, so we bail. return - for device_id in device_ids: - logger.debug( - "Notifying about update %r/%r, ID: %r", user_id, device_id, position - ) + if logger.isEnabledFor(logging.DEBUG): + for device_id in device_ids: + logger.debug( + "Notifying about update %r/%r, ID: %r", user_id, device_id, position + ) # specify the user ID too since the user should always get their own device list # updates, even if they aren't in any rooms. @@ -922,9 +927,6 @@ class DeviceHandler(DeviceWorkerHandler): # can't call self.delete_device because that will clobber the # access token so call the storage layer directly await self.store.delete_devices(user_id, [old_device_id]) - await self.store.delete_e2e_keys_by_device( - user_id=user_id, device_id=old_device_id - ) # tell everyone that the old device is gone and that the dehydrated # device has a new display name @@ -946,7 +948,6 @@ class DeviceHandler(DeviceWorkerHandler): raise errors.NotFoundError() await self.delete_devices(user_id, [device_id]) - await self.store.delete_e2e_keys_by_device(user_id=user_id, device_id=device_id) @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 579d29ac15..a22eab2474 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -52,10 +52,11 @@ from synapse.storage.database import ( make_in_list_sql_clause, ) from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict +from synapse.types import JsonDict, StrCollection from synapse.util import Duration, json_encoder from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -1027,6 +1028,40 @@ class DeviceInboxWorkerStore(SQLBaseStore): # loop first time we run this. self._clock.sleep(1) + async def get_devices_with_messages( + self, user_id: str, device_ids: StrCollection + ) -> StrCollection: + """Get the matching device IDs that have messages in the device inbox.""" + + def get_devices_with_messages_txn( + txn: LoggingTransaction, + batch_device_ids: StrCollection, + ) -> StrCollection: + clause, args = make_in_list_sql_clause( + self.database_engine, "device_id", batch_device_ids + ) + sql = f""" + SELECT DISTINCT device_id FROM device_inbox + WHERE {clause} AND user_id = ? + """ + args.append(user_id) + txn.execute(sql, args) + return {row[0] for row in txn} + + results: Set[str] = set() + for batch_device_ids in batch_iter(device_ids, 1000): + batch_results = await self.db_pool.runInteraction( + "get_devices_with_messages", + get_devices_with_messages_txn, + batch_device_ids, + # We don't need to run in a transaction as it's a single query + db_autocommit=True, + ) + + results.update(batch_results) + + return results + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 6191f22cd6..941d278e6c 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -282,7 +282,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) - @cached() + @cached(tree=True) async def get_device( self, user_id: str, device_id: str ) -> Optional[Mapping[str, Any]]: @@ -1861,7 +1861,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): ) raise StoreError(500, "Problem storing device.") - async def delete_devices(self, user_id: str, device_ids: List[str]) -> None: + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: """Deletes several devices. Args: @@ -1885,11 +1885,49 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): values=device_ids, keyvalues={"user_id": user_id}, ) - self._invalidate_cache_and_stream_bulk( - txn, self.get_device, [(user_id, device_id) for device_id in device_ids] + + # Also delete associated e2e keys. + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, ) - for batch in batch_iter(device_ids, 100): + # We're bulk deleting potentially many devices at once, so + # let's not invalidate the cache for each device individually. + # Instead, we will invalidate the cache for the user as a whole. + self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id,) + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id,) + ) + + for batch in batch_iter(device_ids, 1000): await self.db_pool.runInteraction( "delete_devices", _delete_devices_txn, batch ) @@ -2061,32 +2099,36 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): context = get_active_span_text_map() def add_device_changes_txn( - txn: LoggingTransaction, stream_ids: List[int] + txn: LoggingTransaction, + batch_device_ids: StrCollection, + stream_ids: List[int], ) -> None: self._add_device_change_to_stream_txn( txn, user_id, - device_ids, + batch_device_ids, stream_ids, ) self._add_device_outbound_room_poke_txn( txn, user_id, - device_ids, + batch_device_ids, room_ids, stream_ids, context, ) - async with self._device_list_id_gen.get_next_mult( - len(device_ids) - ) as stream_ids: - await self.db_pool.runInteraction( - "add_device_change_to_stream", - add_device_changes_txn, - stream_ids, - ) + for batch_device_ids in batch_iter(device_ids, 1000): + async with self._device_list_id_gen.get_next_mult( + len(device_ids) + ) as stream_ids: + await self.db_pool.runInteraction( + "add_device_change_to_stream", + add_device_changes_txn, + batch_device_ids, + stream_ids, + ) return stream_ids[-1] diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 341e7014d6..0700b0087b 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -593,7 +593,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker txn, self.count_e2e_one_time_keys, (user_id, device_id) ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def count_e2e_one_time_keys( self, user_id: str, device_id: str ) -> Mapping[str, int]: @@ -808,7 +808,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker }, ) - @cached(max_entries=10000) + @cached(max_entries=10000, tree=True) async def get_e2e_unused_fallback_key_types( self, user_id: str, device_id: str ) -> Sequence[str]: @@ -1632,46 +1632,6 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): log_kv({"message": "Device keys stored."}) return True - async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: - def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: - log_kv( - { - "message": "Deleting keys for device", - "device_id": device_id, - "user_id": user_id, - } - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_device_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_one_time_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.count_e2e_one_time_keys, (user_id, device_id) - ) - self.db_pool.simple_delete_txn( - txn, - table="dehydrated_devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self.db_pool.simple_delete_txn( - txn, - table="e2e_fallback_keys_json", - keyvalues={"user_id": user_id, "device_id": device_id}, - ) - self._invalidate_cache_and_stream( - txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) - ) - - await self.db_pool.runInteraction( - "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn - ) - def _set_e2e_cross_signing_key_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 40c551bcb4..1e21996b12 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -40,14 +40,16 @@ from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.stats import StatsStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator from synapse.storage.util.sequence import build_sequence_generator -from synapse.types import JsonDict, UserID, UserInfo +from synapse.types import JsonDict, StrCollection, UserID, UserInfo from synapse.util.caches.descriptors import cached +from synapse.util.iterutils import batch_iter if TYPE_CHECKING: from synapse.server import HomeServer @@ -2801,6 +2803,81 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): return await self.db_pool.runInteraction("user_delete_access_tokens", f) + async def user_delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + device_ids: The devices to delete tokens for. + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def user_delete_access_tokens_for_devices_txn( + txn: LoggingTransaction, batch_device_ids: StrCollection + ) -> List[Tuple[str, int, Optional[str]]]: + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + clause, args = make_in_list_sql_clause( + txn.database_engine, "device_id", batch_device_ids + ) + args.append(user_id) + + if self.database_engine.supports_returning: + sql = f""" + DELETE FROM access_tokens + WHERE {clause} AND user_id = ? + RETURNING token, id, device_id + """ + txn.execute(sql, args) + tokens_and_devices = txn.fetchall() + else: + tokens_and_devices = self.db_pool.simple_select_many_txn( + txn, + table="access_tokens", + column="device_id", + iterable=batch_device_ids, + keyvalues={"user_id": user_id}, + retcols=("token", "id", "device_id"), + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="access_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(t[0],) for t in tokens_and_devices], + ) + return tokens_and_devices + + results = [] + for batch_device_ids in batch_iter(device_ids, 1000): + tokens_and_devices = await self.db_pool.runInteraction( + "user_delete_access_tokens_for_devices", + user_delete_access_tokens_for_devices_txn, + batch_device_ids, + ) + results.extend(tokens_and_devices) + + return results + async def delete_access_token(self, access_token: str) -> None: def f(txn: LoggingTransaction) -> None: self.db_pool.simple_delete_one_txn( From a2bee2f255853c6e019e1651a168c12dff64d68f Mon Sep 17 00:00:00 2001 From: Krishan <33421343+kfiven@users.noreply.github.com> Date: Mon, 30 Jun 2025 22:42:14 +1000 Subject: [PATCH 409/691] Add `via` param to hierarchy enpoint (#18070) ### Pull Request Checklist Implementation of [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235) as per suggestion in [pull request 17750](https://github.com/element-hq/synapse/pull/17750#issuecomment-2411248598). * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18070.feature | 1 + synapse/config/experimental.py | 3 ++ synapse/handlers/room_summary.py | 21 +++++++++-- synapse/rest/client/room.py | 11 ++++++ tests/handlers/test_room_summary.py | 56 +++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18070.feature diff --git a/changelog.d/18070.feature b/changelog.d/18070.feature new file mode 100644 index 0000000000..ecf468f0b6 --- /dev/null +++ b/changelog.d/18070.feature @@ -0,0 +1 @@ +Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): via query param for hierarchy endpoint. Contributed by Krishan (@kfiven). diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 85eca9c05b..1ae47d3c16 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -561,6 +561,9 @@ class ExperimentalConfig(Config): # MSC4076: Add `disable_badge_count`` to pusher configuration self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) + # MSC4235: Add `via` param to hierarchy endpoint + self.msc4235_enabled: bool = experimental.get("msc4235_enabled", False) + # MSC4263: Preventing MXID enumeration via key queries self.msc4263_limit_key_queries_to_users_who_share_rooms = experimental.get( "msc4263_limit_key_queries_to_users_who_share_rooms", diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 91b131d09b..1f322ac263 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -111,7 +111,15 @@ class RoomSummaryHandler: # If a user tries to fetch the same page multiple times in quick succession, # only process the first attempt and return its result to subsequent requests. self._pagination_response_cache: ResponseCache[ - Tuple[str, str, bool, Optional[int], Optional[int], Optional[str]] + Tuple[ + str, + str, + bool, + Optional[int], + Optional[int], + Optional[str], + Optional[Tuple[str, ...]], + ] ] = ResponseCache( hs.get_clock(), "get_room_hierarchy", @@ -126,6 +134,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """ Implementation of the room hierarchy C-S API. @@ -143,6 +152,9 @@ class RoomSummaryHandler: limit: An optional limit on the number of rooms to return per page. Must be a positive integer. from_token: An optional pagination token. + remote_room_hosts: An optional list of remote homeserver server names. If defined, + each host will be used to try and fetch the room hierarchy. Must be a tuple so + that it can be hashed by the `RoomSummaryHandler._pagination_response_cache`. Returns: The JSON hierarchy dictionary. @@ -162,6 +174,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ), self._get_room_hierarchy, requester.user.to_string(), @@ -170,6 +183,7 @@ class RoomSummaryHandler: max_depth, limit, from_token, + remote_room_hosts, ) async def _get_room_hierarchy( @@ -180,6 +194,7 @@ class RoomSummaryHandler: max_depth: Optional[int] = None, limit: Optional[int] = None, from_token: Optional[str] = None, + remote_room_hosts: Optional[Tuple[str, ...]] = None, ) -> JsonDict: """See docstring for SpaceSummaryHandler.get_room_hierarchy.""" @@ -199,7 +214,7 @@ class RoomSummaryHandler: if not local_room: room_hierarchy = await self._summarize_remote_room_hierarchy( - _RoomQueueEntry(requested_room_id, ()), + _RoomQueueEntry(requested_room_id, remote_room_hosts or ()), False, ) root_room_entry = room_hierarchy[0] @@ -240,7 +255,7 @@ class RoomSummaryHandler: processed_rooms = set(pagination_session["processed_rooms"]) else: # The queue of rooms to process, the next room is last on the stack. - room_queue = [_RoomQueueEntry(requested_room_id, ())] + room_queue = [_RoomQueueEntry(requested_room_id, remote_room_hosts or ())] # Rooms we have already processed. processed_rooms = set() diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index c64ce1f9c7..4600a87778 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1538,6 +1538,7 @@ class RoomHierarchyRestServlet(RestServlet): super().__init__() self._auth = hs.get_auth() self._room_summary_handler = hs.get_room_summary_handler() + self.msc4235_enabled = hs.config.experimental.msc4235_enabled async def on_GET( self, request: SynapseRequest, room_id: str @@ -1547,6 +1548,15 @@ class RoomHierarchyRestServlet(RestServlet): max_depth = parse_integer(request, "max_depth") limit = parse_integer(request, "limit") + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + remote_room_hosts = None + if self.msc4235_enabled: + args: Dict[bytes, List[bytes]] = request.args # type: ignore + via_param = parse_strings_from_args( + args, "org.matrix.msc4235.via", required=False + ) + remote_room_hosts = tuple(via_param or []) + return 200, await self._room_summary_handler.get_room_hierarchy( requester, room_id, @@ -1554,6 +1564,7 @@ class RoomHierarchyRestServlet(RestServlet): max_depth=max_depth, limit=limit, from_token=parse_string(request, "from"), + remote_room_hosts=remote_room_hosts, ) diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index b55fa1a8fd..bf18c1e72a 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -1080,6 +1080,62 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): self.assertEqual(federation_requests, 2) self._assert_hierarchy(result, expected) + def test_fed_remote_room_hosts(self) -> None: + """ + Test if requested room is available over federation using via's. + """ + fed_hostname = self.hs.hostname + "2" + fed_space = "#fed_space:" + fed_hostname + fed_subroom = "#fed_sub_room:" + fed_hostname + + remote_room_hosts = tuple(fed_hostname) + + requested_room_entry = _RoomEntry( + fed_space, + { + "room_id": fed_space, + "world_readable": True, + "join_rule": "public", + "room_type": RoomTypes.SPACE, + }, + [ + { + "type": EventTypes.SpaceChild, + "room_id": fed_space, + "state_key": fed_subroom, + "content": {"via": [fed_hostname]}, + } + ], + ) + child_room = { + "room_id": fed_subroom, + "world_readable": True, + "join_rule": "public", + } + + async def summarize_remote_room_hierarchy( + _self: Any, room: Any, suggested_only: bool + ) -> Tuple[Optional[_RoomEntry], Dict[str, JsonDict], Set[str]]: + return requested_room_entry, {fed_subroom: child_room}, set() + + expected = [ + (fed_space, [fed_subroom]), + (fed_subroom, ()), + ] + + with mock.patch( + "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy", + new=summarize_remote_room_hierarchy, + ): + result = self.get_success( + self.handler.get_room_hierarchy( + create_requester(self.user), + fed_space, + remote_room_hosts=remote_room_hosts, + ) + ) + self._assert_hierarchy(result, expected) + class RoomSummaryTestCase(unittest.HomeserverTestCase): servlets = [ From 291880012fd947da22952a7c77a44772e8ef57f1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 1 Jul 2025 12:04:23 +0100 Subject: [PATCH 410/691] Stop sending or processing the `origin` field in PDUs (#18418) Co-authored-by: Quentin Gliech Co-authored-by: Eric Eastwood --- changelog.d/18418.removal | 1 + contrib/graph/graph.py | 4 ++++ docs/admin_api/event_reports.md | 1 - synapse/events/__init__.py | 1 - synapse/events/builder.py | 5 ++--- synapse/events/validator.py | 8 -------- synapse/federation/federation_base.py | 3 +-- synapse/util/caches/__init__.py | 2 +- tests/crypto/test_event_signing.py | 14 ++++++-------- tests/events/test_utils.py | 8 +++----- tests/federation/test_federation_server.py | 3 --- 11 files changed, 18 insertions(+), 32 deletions(-) create mode 100644 changelog.d/18418.removal diff --git a/changelog.d/18418.removal b/changelog.d/18418.removal new file mode 100644 index 0000000000..ff1d081d8f --- /dev/null +++ b/changelog.d/18418.removal @@ -0,0 +1 @@ +Stop adding the "origin" field to newly-created events (PDUs). \ No newline at end of file diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index 1d74fee822..9d5f3c7f4f 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -45,6 +45,10 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None: colors = {"red", "green", "blue", "yellow", "purple"} for pdu in pdus: + # TODO: The "origin" field has since been removed from events generated + # by Synapse. We should consider removing it here as well but since this + # is part of `contrib/`, it is left for the community to revise and ensure things + # still work correctly. origins.add(pdu.get("origin")) color_map = {color: color for color in colors if color in origins} diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md index 9075e92882..225b431715 100644 --- a/docs/admin_api/event_reports.md +++ b/docs/admin_api/event_reports.md @@ -117,7 +117,6 @@ It returns a JSON body like the following: "hashes": { "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw" }, - "origin": "matrix.org", "origin_server_ts": 1592291711430, "prev_events": [ "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M" diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index a85e66d6bf..c77d569e2e 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -208,7 +208,6 @@ class EventBase(metaclass=abc.ABCMeta): depth: DictProperty[int] = DictProperty("depth") content: DictProperty[JsonDict] = DictProperty("content") hashes: DictProperty[Dict[str, str]] = DictProperty("hashes") - origin: DictProperty[str] = DictProperty("origin") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") room_id: DictProperty[str] = DictProperty("room_id") sender: DictProperty[str] = DictProperty("sender") diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 76df083d69..afb04881df 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -302,8 +302,8 @@ def create_local_event_from_event_dict( event_dict: JsonDict, internal_metadata_dict: Optional[JsonDict] = None, ) -> EventBase: - """Takes a fully formed event dict, ensuring that fields like `origin` - and `origin_server_ts` have correct values for a locally produced event, + """Takes a fully formed event dict, ensuring that fields like + `origin_server_ts` have correct values for a locally produced event, then signs and hashes it. """ @@ -319,7 +319,6 @@ def create_local_event_from_event_dict( if format_version == EventFormatVersions.ROOM_V1_V2: event_dict["event_id"] = _create_event_id(clock, hostname) - event_dict["origin"] = hostname event_dict.setdefault("origin_server_ts", time_now) event_dict.setdefault("unsigned", {}) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index d1fb026cd6..15095cc4ef 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -67,7 +67,6 @@ class EventValidator: "auth_events", "content", "hashes", - "origin", "prev_events", "sender", "type", @@ -77,13 +76,6 @@ class EventValidator: if k not in event: raise SynapseError(400, "Event does not have key %s" % (k,)) - # Check that the following keys have string values - event_strings = ["origin"] - - for s in event_strings: - if not isinstance(getattr(event, s), str): - raise SynapseError(400, "'%s' not a string type" % (s,)) - # Depending on the room version, ensure the data is spec compliant JSON. if event.room_version.strict_canonicaljson: validate_canonicaljson(event.get_pdu_json()) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 45593430e8..8d1e156dab 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -322,8 +322,7 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB SynapseError: if the pdu is missing required fields or is otherwise not a valid matrix event """ - # we could probably enforce a bunch of other fields here (room_id, sender, - # origin, etc etc) + # we could probably enforce a bunch of other fields here (room_id, sender, etc.) assert_params_in_dict(pdu_json, ("type", "depth")) # Strip any unauthorized values from "unsigned" if they exist diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index df8829baeb..76e6e139c6 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -225,7 +225,7 @@ KNOWN_KEYS = { "depth", "event_id", "hashes", - "origin", + "origin", # old events were created with an origin field. "origin_server_ts", "prev_events", "room_id", diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index d7b9fb8bc6..9cdc1604da 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -48,7 +48,6 @@ class EventSigningTestCase(unittest.TestCase): def test_sign_minimal(self) -> None: event_dict = { "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "signatures": {}, "type": "X", @@ -64,7 +63,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI" + event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" ) self.assertTrue(hasattr(event, "signatures")) @@ -72,15 +71,14 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" - "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", + "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" + "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", ) def test_sign_message(self) -> None: event_dict = { "content": {"body": "Here is the message content"}, "event_id": "$0:domain", - "origin": "domain", "origin_server_ts": 1000000, "type": "m.room.message", "room_id": "!r:domain", @@ -98,7 +96,7 @@ class EventSigningTestCase(unittest.TestCase): self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + event.hashes["sha256"], "rDCeYBepPlI891h/RkI2/Lkf9bt7u0TxFku4tMs7WKk" ) self.assertTrue(hasattr(event, "signatures")) @@ -106,6 +104,6 @@ class EventSigningTestCase(unittest.TestCase): self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" - "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA", + "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc" + "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw", ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 654e6521a2..521251a97e 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -122,7 +122,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", # Also include a key that should be removed. @@ -139,7 +139,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase): "prev_events": "prev_events", "prev_state": "prev_state", "auth_events": "auth_events", - "origin": "domain", + "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", "content": {}, @@ -148,13 +148,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase): }, ) - # As of room versions we now redact the membership, prev_states, and origin keys. + # As of room versions we now redact the membership and prev_states keys. self.run_test( { "type": "A", "prev_state": "prev_state", "membership": "join", - "origin": "example.com", }, {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, room_version=RoomVersions.V11, @@ -238,7 +237,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase): { "type": "m.room.create", "content": {"not_a_real_key": True}, - "origin": "some_homeserver", "nonsense_field": "some_random_garbage", }, { diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index a7b702fbd1..58ead90909 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -535,7 +535,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.member", - "origin": "test.servx", "content": {"membership": "join"}, "auth_events": [], "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, @@ -552,7 +551,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.member", - "origin": "test.servx", "auth_events": [], "content": {"membership": "join"}, "unsigned": { @@ -579,7 +577,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase): "depth": 1000, "origin_server_ts": 1, "type": "m.room.power_levels", - "origin": "test.servx", "content": {}, "auth_events": [], "unsigned": { From c471e8469790a8546798eb26299f0be8bb66a281 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Jul 2025 14:54:33 +0200 Subject: [PATCH 411/691] Bump cibuildwheel to 3.0.0 to fix the building of wheels (#18615) Fixes https://github.com/element-hq/synapse/issues/18614 This upgrade CIBW to 3.0, which now builds using the manylinux_2_28 image, as the previous image is EOL and not supported by some of our dependencies anymore. This also updates the job to use the `ubuntu-24.04` base image instead of `ubuntu-22.04` --- .github/workflows/release-artifacts.yml | 4 ++-- changelog.d/18615.bugfix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18615.bugfix diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 572d73e6ad..c6c252c9c3 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -111,7 +111,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13] + os: [ubuntu-24.04, macos-13] arch: [x86_64, aarch64] # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. @@ -139,7 +139,7 @@ jobs: python-version: "3.x" - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.23.0 + run: python -m pip install cibuildwheel==3.0.0 - name: Set up QEMU to emulate aarch64 if: matrix.arch == 'aarch64' diff --git a/changelog.d/18615.bugfix b/changelog.d/18615.bugfix new file mode 100644 index 0000000000..dbdf1849ca --- /dev/null +++ b/changelog.d/18615.bugfix @@ -0,0 +1 @@ +Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. From 214ac2f00550e5ac0541068c60123424251c2911 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Jul 2025 15:13:42 +0200 Subject: [PATCH 412/691] 1.133.0 --- CHANGES.md | 9 +++++++++ changelog.d/18615.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/18615.bugfix diff --git a/CHANGES.md b/CHANGES.md index 23debccd63..c0c03af72e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +# Synapse 1.133.0 (2025-07-01) + +### Bugfixes + +- Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. ([\#18615](https://github.com/element-hq/synapse/issues/18615)) + + + + # Synapse 1.133.0rc1 (2025-06-24) ### Features diff --git a/changelog.d/18615.bugfix b/changelog.d/18615.bugfix deleted file mode 100644 index dbdf1849ca..0000000000 --- a/changelog.d/18615.bugfix +++ /dev/null @@ -1 +0,0 @@ -Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. diff --git a/debian/changelog b/debian/changelog index 614901e3fd..32070a32d0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.133.0) stable; urgency=medium + + * New synapse release 1.133.0. + + -- Synapse Packaging team Tue, 01 Jul 2025 13:13:24 +0000 + matrix-synapse-py3 (1.133.0~rc1) stable; urgency=medium * New Synapse release 1.133.0rc1. diff --git a/pyproject.toml b/pyproject.toml index a354e26d39..8593ce06f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.133.0rc1" +version = "1.133.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From e3ed93adf35f6246ccd98c9cb5e938d564541fde Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 1 Jul 2025 16:01:28 +0200 Subject: [PATCH 413/691] Add a note in the changelog about the manylinux wheels --- CHANGES.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c0c03af72e..f3e541563f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ # Synapse 1.133.0 (2025-07-01) +Pre-built wheels are now built using the [manylinux_2_28](https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based) base, which is expected to be compatible with distros using glibc 2.28 or later, including: + + - Debian 10+ + - Ubuntu 18.10+ + - Fedora 29+ + - CentOS/RHEL 8+ + +Previously, wheels were built using the [manylinux2014](https://github.com/pypa/manylinux#manylinux2014-centos-7-based-glibc-217) base, which was expected to be compatible with distros using glibc 2.17 or later. + ### Bugfixes - Bump `cibuildwheel` to 3.0.0 to fix the `manylinux` wheel builds. ([\#18615](https://github.com/element-hq/synapse/issues/18615)) From c17fd947f30004745b3191ff0edd76c3cd87e352 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 1 Jul 2025 16:55:38 +0000 Subject: [PATCH 414/691] Fix documentation of the Delete Room Admin API's status field. (#18519) Fixes: #18502 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/18519.doc | 1 + docs/admin_api/rooms.md | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18519.doc diff --git a/changelog.d/18519.doc b/changelog.d/18519.doc new file mode 100644 index 0000000000..b6fde5f42b --- /dev/null +++ b/changelog.d/18519.doc @@ -0,0 +1 @@ +Fix documentation of the Delete Room Admin API's status field. \ No newline at end of file diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index bdda9b47ad..12af87148d 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -806,7 +806,7 @@ A response body like the following is returned: }, { "delete_id": "delete_id2", "room_id": "!roomid:example.com", - "status": "purging", + "status": "active", "shutdown_room": { "kicked_users": [ "@foobar:example.com" @@ -843,7 +843,7 @@ A response body like the following is returned: ```json { - "status": "purging", + "status": "active", "delete_id": "bHkCNQpHqOaFhPtK", "room_id": "!roomid:example.com", "shutdown_room": { @@ -876,8 +876,8 @@ The following fields are returned in the JSON response body: - `delete_id` - The ID for this purge - `room_id` - The ID of the room being deleted - `status` - The status will be one of: - - `shutting_down` - The process is removing users from the room. - - `purging` - The process is purging the room and event data from database. + - `scheduled` - The deletion is waiting to be started + - `active` - The process is purging the room and event data from database. - `complete` - The process has completed successfully. - `failed` - The process is aborted, an error has occurred. - `error` - A string that shows an error message if `status` is `failed`. From cc8da2c5ed0cecc771919d76533704a04de9a41e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Jul 2025 15:02:12 +0100 Subject: [PATCH 415/691] Log the room ID we're purging state for (#18625) So we can see what we're deleting. --- changelog.d/18625.misc | 1 + synapse/storage/controllers/purge_events.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/18625.misc diff --git a/changelog.d/18625.misc b/changelog.d/18625.misc new file mode 100644 index 0000000000..f2672864c2 --- /dev/null +++ b/changelog.d/18625.misc @@ -0,0 +1 @@ +Log the room ID we're purging state for. diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index c2d4bf8290..df3f264b06 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -34,6 +34,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_proces from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases from synapse.types.storage import _BackgroundUpdates +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -167,6 +168,12 @@ class PurgeEventsStorageController: break (room_id, groups_to_sequences) = next_to_delete + + logger.info( + "[purge] deleting state groups for room %s: %s", + room_id, + shortstr(groups_to_sequences.keys(), maxitems=10), + ) made_progress = await self._delete_state_groups( room_id, groups_to_sequences ) From 6ddbb0361222da276cf0f9b76dd2dff92862327e Mon Sep 17 00:00:00 2001 From: V02460 Date: Wed, 2 Jul 2025 17:57:30 +0200 Subject: [PATCH 416/691] Raise poetry-core version cap to 2.1.3 (#18575) Request to raise the defensive version cap for poetry-core from 1.9.1 to 2.1.3. My understanding is that the major version bump of poetry signals the transition to standardized pyproject.toml metadata, but does not affect backwards compatibility. This is a subset of the changes in #18432 Fixes #18200 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18575.misc | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18575.misc diff --git a/changelog.d/18575.misc b/changelog.d/18575.misc new file mode 100644 index 0000000000..0033f27df8 --- /dev/null +++ b/changelog.d/18575.misc @@ -0,0 +1 @@ +Raise poetry-core version cap to 2.1.3. diff --git a/pyproject.toml b/pyproject.toml index 8593ce06f0..9ad489b778 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,7 +374,7 @@ tomli = ">=1.2.3" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"] +requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.10.2"] build-backend = "poetry.core.masonry.api" From 2f9c9d5ebaa6b53d23aba2ec140499ff007e2bab Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 2 Jul 2025 12:18:33 -0400 Subject: [PATCH 417/691] Forbid locked users from using `POST /login` (#18594) Discussed in the [Synapse Dev room](https://matrix.to/#/!vcyiEtMVHIhWXcJAfl:sw1v.org/$K4UojQtvaSpxSe35TWFXtKWGoAuHwHFcKo8qn2lwxSs?via=matrix.org&via=element.io&via=envs.net) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18594.bugfix | 1 + synapse/api/auth/internal.py | 8 ++------ synapse/api/errors.py | 14 ++++++++++++++ synapse/rest/client/login.py | 16 ++++++++++------ tests/rest/admin/test_user.py | 10 ++++++++++ 5 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 changelog.d/18594.bugfix diff --git a/changelog.d/18594.bugfix b/changelog.d/18594.bugfix new file mode 100644 index 0000000000..3293a731f2 --- /dev/null +++ b/changelog.d/18594.bugfix @@ -0,0 +1 @@ +Respond with 401 & `M_USER_LOCKED` when a locked user calls `POST /login`, as per the spec. diff --git a/synapse/api/auth/internal.py b/synapse/api/auth/internal.py index 9fd4db68e1..afc7b5c4a9 100644 --- a/synapse/api/auth/internal.py +++ b/synapse/api/auth/internal.py @@ -29,6 +29,7 @@ from synapse.api.errors import ( InvalidClientTokenError, MissingClientTokenError, UnrecognizedRequestError, + UserLockedError, ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import active_span, force_tracing, start_active_span @@ -162,12 +163,7 @@ class InternalAuth(BaseAuth): if not allow_locked and await self.store.get_user_locked_status( requester.user.to_string() ): - raise AuthError( - 401, - "User account has been locked", - errcode=Codes.USER_LOCKED, - additional_fields={"soft_logout": True}, - ) + raise UserLockedError() # Deny the request if the user account has expired. # This check is only done for regular users, not appservice ones. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 601a09efe1..b832c2f6a1 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -306,6 +306,20 @@ class UserDeactivatedError(SynapseError): ) +class UserLockedError(SynapseError): + """The error returned to the client when the user attempted to access an + authenticated endpoint, but the account has been locked. + """ + + def __init__(self) -> None: + super().__init__( + code=HTTPStatus.UNAUTHORIZED, + msg="User account has been locked", + errcode=Codes.USER_LOCKED, + additional_fields={"soft_logout": True}, + ) + + class FederationDeniedError(SynapseError): """An error raised when the server tries to federate with a server which is not on its federation whitelist. diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 72b219447b..8a781f759c 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -42,6 +42,7 @@ from synapse.api.errors import ( NotApprovedError, SynapseError, UserDeactivatedError, + UserLockedError, ) from synapse.api.ratelimiting import Ratelimiter from synapse.api.urls import CLIENT_API_PREFIX @@ -313,7 +314,7 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token=should_issue_refresh_token, # The user represented by an appservice's configured sender_localpart # is not actually created in Synapse. - should_check_deactivated=qualified_user_id != appservice.sender, + should_check_deactivated_or_locked=qualified_user_id != appservice.sender, request_info=request_info, ) @@ -367,7 +368,7 @@ class LoginRestServlet(RestServlet): auth_provider_id: Optional[str] = None, should_issue_refresh_token: bool = False, auth_provider_session_id: Optional[str] = None, - should_check_deactivated: bool = True, + should_check_deactivated_or_locked: bool = True, *, request_info: RequestInfo, ) -> LoginResponse: @@ -389,8 +390,8 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token: True if this login should issue a refresh token alongside the access token. auth_provider_session_id: The session ID got during login from the SSO IdP. - should_check_deactivated: True if the user should be checked for - deactivation status before logging in. + should_check_deactivated_or_locked: True if the user should be checked for + deactivation or locked status before logging in. This exists purely for appservice's configured sender_localpart which doesn't have an associated user in the database. @@ -415,11 +416,14 @@ class LoginRestServlet(RestServlet): ) user_id = canonical_uid - # If the account has been deactivated, do not proceed with the login. - if should_check_deactivated: + # If the account has been deactivated or locked, do not proceed with the login. + if should_check_deactivated_or_locked: deactivated = await self._main_store.get_user_deactivated_status(user_id) if deactivated: raise UserDeactivatedError("This account has been deactivated") + locked = await self._main_store.get_user_locked_status(user_id) + if locked: + raise UserLockedError() device_id = login_submission.get("device_id") diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 5f73dbdc4a..e9c631e2aa 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -2846,6 +2846,16 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) self.assertTrue(channel.json_body["soft_logout"]) + # User is not authorized to log in anymore + channel = self.make_request( + "POST", + "/_matrix/client/r0/login", + {"type": "m.login.password", "user": "user", "password": "pass"}, + ) + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.USER_LOCKED, channel.json_body["errcode"]) + self.assertTrue(channel.json_body["soft_logout"]) + @override_config({"user_directory": {"enabled": True, "search_all_users": True}}) def test_locked_user_not_in_user_dir(self) -> None: # User is available in the user dir From 82757144e93fe8e4f3365fcdec2f40c7f2e8cb8f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:21:51 +0200 Subject: [PATCH 418/691] Bump stefanzweifel/git-auto-commit-action from 5.2.0 to 6.0.1 (#18607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 5.2.0 to 6.0.1.
Release notes

Sourced from stefanzweifel/git-auto-commit-action's releases.

v6.0.1

Fixed

v6.0.0

Added

  • Throw error early if repository is in a detached state (#357)

Fixed

Removed

  • Remove support for create_branch, skip_checkout, skip_Fetch (#314)
Changelog

Sourced from stefanzweifel/git-auto-commit-action's changelog.

Changelog

All notable changes to this project will be documented in this file.

The format is based on Keep a Changelog and this project adheres to Semantic Versioning.

Unreleased

TBD

v6.0.1 - 2025-06-11

Fixed

v6.0.0 - 2025-06-10

Added

  • Throw error early if repository is in a detached state (#357)

Fixed

Removed

  • Remove support for create_branch, skip_checkout, skip_Fetch (#314)

v5.2.0 - 2025-04-19

Added

Fixed

v5.1.0 - 2025-01-11

Changed

Fixed

... (truncated)

Commits
  • 778341a Merge pull request #379 from stefanzweifel/disable-detached-state-check
  • 33b203d Disable Check if Repo is in Detached State
  • a82d80a Update CHANGELOG
  • 3cc016c Merge pull request #375 from stefanzweifel/v6-next
  • ddb7ae4 Merge pull request #376 from Dreamsorcerer/patch-1
  • b001e5f Apply suggestions from code review
  • 6494dc6 Fix PAT instructions with Dependabot
  • 7618051 Add deprecated inputs to fix unbound variable issue
  • ae11462 Merge pull request #371 from stefanzweifel/dependabot/npm_and_yarn/bats-1.12.0
  • 3058f91 Bump bats from 1.11.1 to 1.12.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=stefanzweifel/git-auto-commit-action&package-manager=github_actions&previous-version=5.2.0&new-version=6.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/fix_lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 923e96a624..1547eb6197 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -44,6 +44,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@b863ae1933cb653a53c021fe36dbb774e1fb9403 # v5.2.0 + - uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1 with: commit_message: "Attempt to fix linting" From 2372f3e6b72e42a88a3d8d3e0ecd53609a844b91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:22:41 +0200 Subject: [PATCH 419/691] Bump sigstore/cosign-installer from 3.9.0 to 3.9.1 (#18608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sigstore/cosign-installer](https://github.com/sigstore/cosign-installer) from 3.9.0 to 3.9.1.
Release notes

Sourced from sigstore/cosign-installer's releases.

v3.9.1

What's Changed

Full Changelog: https://github.com/sigstore/cosign-installer/compare/v3.9.0...v3.9.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sigstore/cosign-installer&package-manager=github_actions&previous-version=3.9.0&new-version=3.9.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e31b6aea90..67d7df0b23 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: run: docker buildx inspect - name: Install Cosign - uses: sigstore/cosign-installer@fb28c2b6339dcd94da6e4cbcbc5e888961f6f8c3 # v3.9.0 + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From 81ca2923d1c9dd7ef0682641d0921708bc449f24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:25:50 +0200 Subject: [PATCH 420/691] Bump types-jsonschema from 4.23.0.20250516 to 4.24.0.20250528 (#18611) Bumps [types-jsonschema](https://github.com/typeshed-internal/stub_uploader) from 4.23.0.20250516 to 4.24.0.20250528.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-jsonschema&package-manager=pip&previous-version=4.23.0.20250516&new-version=4.24.0.20250528)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 55 +++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index e3463aba32..f515638f43 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.5.2-py2.py3-none-any.whl", hash = "sha256:8804dd4402ac5e4a0435ac49e0b6e19e395357cfa632a3f624dcb4f6df13b4b1"}, {file = "authlib-1.5.2.tar.gz", hash = "sha256:fe85ec7e50c5f86f1e2603518bb3b4f632985eb4a355e52256530790e326c512"}, @@ -450,7 +450,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -493,7 +493,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -543,7 +543,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, @@ -889,7 +889,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1027,7 +1027,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1043,7 +1043,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, @@ -1323,7 +1323,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1544,7 +1544,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1713,7 +1713,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1721,6 +1721,7 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -1733,7 +1734,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1749,7 +1750,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1972,7 +1973,7 @@ description = "Python extension wrapping the ICU C++ API" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"user-search\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"user-search\"" files = [ {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, ] @@ -2021,7 +2022,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2081,7 +2082,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2106,7 +2107,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2134,7 +2135,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2498,7 +2499,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, @@ -2686,7 +2687,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2702,7 +2703,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2764,7 +2765,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2900,7 +2901,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -2966,14 +2967,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.23.0.20250516" +version = "4.24.0.20250528" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_jsonschema-4.23.0.20250516-py3-none-any.whl", hash = "sha256:e7d0dd7db7e59e63c26e3230e26ffc64c4704cc5170dc21270b366a35ead1618"}, - {file = "types_jsonschema-4.23.0.20250516.tar.gz", hash = "sha256:9ace09d9d35c4390a7251ccd7d833b92ccc189d24d1b347f26212afce361117e"}, + {file = "types_jsonschema-4.24.0.20250528-py3-none-any.whl", hash = "sha256:6a906b5ff73ac11c8d1e0b6c30a9693e1e4e1ab56c56c932b3a7e081b86d187b"}, + {file = "types_jsonschema-4.24.0.20250528.tar.gz", hash = "sha256:7e28c64e0ae7980eeb158105b20663fc6a6b8f81d5f86ea6614aa0014417bd1e"}, ] [package.dependencies] @@ -3243,7 +3244,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From a3ec2d3b3fb7f6d4fba253e551c4ee205c3b38e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:29:37 +0200 Subject: [PATCH 421/691] Bump pillow from 11.2.1 to 11.3.0 (#18624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pillow](https://github.com/python-pillow/Pillow) from 11.2.1 to 11.3.0.
Release notes

Sourced from pillow's releases.

11.3.0

https://pillow.readthedocs.io/en/stable/releasenotes/11.3.0.html

Deprecations

Documentation

Dependencies

Testing

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pillow&package-manager=pip&previous-version=11.2.1&new-version=11.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 173 +++++++++++++++++++++++++++------------------------- 1 file changed, 89 insertions(+), 84 deletions(-) diff --git a/poetry.lock b/poetry.lock index f515638f43..64570b1ba0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1593,101 +1593,106 @@ files = [ [[package]] name = "pillow" -version = "11.2.1" +version = "11.3.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pillow-11.2.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:d57a75d53922fc20c165016a20d9c44f73305e67c351bbc60d1adaf662e74047"}, - {file = "pillow-11.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:127bf6ac4a5b58b3d32fc8289656f77f80567d65660bc46f72c0d77e6600cc95"}, - {file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ba4be812c7a40280629e55ae0b14a0aafa150dd6451297562e1764808bbe61"}, - {file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bd62331e5032bc396a93609982a9ab6b411c05078a52f5fe3cc59234a3abd1"}, - {file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:562d11134c97a62fe3af29581f083033179f7ff435f78392565a1ad2d1c2c45c"}, - {file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c97209e85b5be259994eb5b69ff50c5d20cca0f458ef9abd835e262d9d88b39d"}, - {file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0c3e6d0f59171dfa2e25d7116217543310908dfa2770aa64b8f87605f8cacc97"}, - {file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc1c3bc53befb6096b84165956e886b1729634a799e9d6329a0c512ab651e579"}, - {file = "pillow-11.2.1-cp310-cp310-win32.whl", hash = "sha256:312c77b7f07ab2139924d2639860e084ec2a13e72af54d4f08ac843a5fc9c79d"}, - {file = "pillow-11.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9bc7ae48b8057a611e5fe9f853baa88093b9a76303937449397899385da06fad"}, - {file = "pillow-11.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:2728567e249cdd939f6cc3d1f049595c66e4187f3c34078cbc0a7d21c47482d2"}, - {file = "pillow-11.2.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35ca289f712ccfc699508c4658a1d14652e8033e9b69839edf83cbdd0ba39e70"}, - {file = "pillow-11.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0409af9f829f87a2dfb7e259f78f317a5351f2045158be321fd135973fff7bf"}, - {file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e5c5edee874dce4f653dbe59db7c73a600119fbea8d31f53423586ee2aafd7"}, - {file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b93a07e76d13bff9444f1a029e0af2964e654bfc2e2c2d46bfd080df5ad5f3d8"}, - {file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e6def7eed9e7fa90fde255afaf08060dc4b343bbe524a8f69bdd2a2f0018f600"}, - {file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8f4f3724c068be008c08257207210c138d5f3731af6c155a81c2b09a9eb3a788"}, - {file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0a6709b47019dff32e678bc12c63008311b82b9327613f534e496dacaefb71e"}, - {file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f6b0c664ccb879109ee3ca702a9272d877f4fcd21e5eb63c26422fd6e415365e"}, - {file = "pillow-11.2.1-cp311-cp311-win32.whl", hash = "sha256:cc5d875d56e49f112b6def6813c4e3d3036d269c008bf8aef72cd08d20ca6df6"}, - {file = "pillow-11.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:0f5c7eda47bf8e3c8a283762cab94e496ba977a420868cb819159980b6709193"}, - {file = "pillow-11.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:4d375eb838755f2528ac8cbc926c3e31cc49ca4ad0cf79cff48b20e30634a4a7"}, - {file = "pillow-11.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:78afba22027b4accef10dbd5eed84425930ba41b3ea0a86fa8d20baaf19d807f"}, - {file = "pillow-11.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78092232a4ab376a35d68c4e6d5e00dfd73454bd12b230420025fbe178ee3b0b"}, - {file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a5f306095c6780c52e6bbb6109624b95c5b18e40aab1c3041da3e9e0cd3e2d"}, - {file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c7b29dbd4281923a2bfe562acb734cee96bbb129e96e6972d315ed9f232bef4"}, - {file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e645b020f3209a0181a418bffe7b4a93171eef6c4ef6cc20980b30bebf17b7d"}, - {file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2dbea1012ccb784a65349f57bbc93730b96e85b42e9bf7b01ef40443db720b4"}, - {file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3104c57bbd72948d75f6a9389e6727d2ab6333c3617f0a89d72d4940aa0443"}, - {file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:598174aef4589af795f66f9caab87ba4ff860ce08cd5bb447c6fc553ffee603c"}, - {file = "pillow-11.2.1-cp312-cp312-win32.whl", hash = "sha256:1d535df14716e7f8776b9e7fee118576d65572b4aad3ed639be9e4fa88a1cad3"}, - {file = "pillow-11.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:14e33b28bf17c7a38eede290f77db7c664e4eb01f7869e37fa98a5aa95978941"}, - {file = "pillow-11.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:21e1470ac9e5739ff880c211fc3af01e3ae505859392bf65458c224d0bf283eb"}, - {file = "pillow-11.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fdec757fea0b793056419bca3e9932eb2b0ceec90ef4813ea4c1e072c389eb28"}, - {file = "pillow-11.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0e130705d568e2f43a17bcbe74d90958e8a16263868a12c3e0d9c8162690830"}, - {file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bdb5e09068332578214cadd9c05e3d64d99e0e87591be22a324bdbc18925be0"}, - {file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d189ba1bebfbc0c0e529159631ec72bb9e9bc041f01ec6d3233d6d82eb823bc1"}, - {file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:191955c55d8a712fab8934a42bfefbf99dd0b5875078240943f913bb66d46d9f"}, - {file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad275964d52e2243430472fc5d2c2334b4fc3ff9c16cb0a19254e25efa03a155"}, - {file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:750f96efe0597382660d8b53e90dd1dd44568a8edb51cb7f9d5d918b80d4de14"}, - {file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fe15238d3798788d00716637b3d4e7bb6bde18b26e5d08335a96e88564a36b6b"}, - {file = "pillow-11.2.1-cp313-cp313-win32.whl", hash = "sha256:3fe735ced9a607fee4f481423a9c36701a39719252a9bb251679635f99d0f7d2"}, - {file = "pillow-11.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:74ee3d7ecb3f3c05459ba95eed5efa28d6092d751ce9bf20e3e253a4e497e691"}, - {file = "pillow-11.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:5119225c622403afb4b44bad4c1ca6c1f98eed79db8d3bc6e4e160fc6339d66c"}, - {file = "pillow-11.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8ce2e8411c7aaef53e6bb29fe98f28cd4fbd9a1d9be2eeea434331aac0536b22"}, - {file = "pillow-11.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ee66787e095127116d91dea2143db65c7bb1e232f617aa5957c0d9d2a3f23a7"}, - {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9622e3b6c1d8b551b6e6f21873bdcc55762b4b2126633014cea1803368a9aa16"}, - {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63b5dff3a68f371ea06025a1a6966c9a1e1ee452fc8020c2cd0ea41b83e9037b"}, - {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:31df6e2d3d8fc99f993fd253e97fae451a8db2e7207acf97859732273e108406"}, - {file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:062b7a42d672c45a70fa1f8b43d1d38ff76b63421cbbe7f88146b39e8a558d91"}, - {file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4eb92eca2711ef8be42fd3f67533765d9fd043b8c80db204f16c8ea62ee1a751"}, - {file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f91ebf30830a48c825590aede79376cb40f110b387c17ee9bd59932c961044f9"}, - {file = "pillow-11.2.1-cp313-cp313t-win32.whl", hash = "sha256:e0b55f27f584ed623221cfe995c912c61606be8513bfa0e07d2c674b4516d9dd"}, - {file = "pillow-11.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:36d6b82164c39ce5482f649b437382c0fb2395eabc1e2b1702a6deb8ad647d6e"}, - {file = "pillow-11.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:225c832a13326e34f212d2072982bb1adb210e0cc0b153e688743018c94a2681"}, - {file = "pillow-11.2.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:7491cf8a79b8eb867d419648fff2f83cb0b3891c8b36da92cc7f1931d46108c8"}, - {file = "pillow-11.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b02d8f9cb83c52578a0b4beadba92e37d83a4ef11570a8688bbf43f4ca50909"}, - {file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:014ca0050c85003620526b0ac1ac53f56fc93af128f7546623cc8e31875ab928"}, - {file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3692b68c87096ac6308296d96354eddd25f98740c9d2ab54e1549d6c8aea9d79"}, - {file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:f781dcb0bc9929adc77bad571b8621ecb1e4cdef86e940fe2e5b5ee24fd33b35"}, - {file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:2b490402c96f907a166615e9a5afacf2519e28295f157ec3a2bb9bd57de638cb"}, - {file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dd6b20b93b3ccc9c1b597999209e4bc5cf2853f9ee66e3fc9a400a78733ffc9a"}, - {file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4b835d89c08a6c2ee7781b8dd0a30209a8012b5f09c0a665b65b0eb3560b6f36"}, - {file = "pillow-11.2.1-cp39-cp39-win32.whl", hash = "sha256:b10428b3416d4f9c61f94b494681280be7686bda15898a3a9e08eb66a6d92d67"}, - {file = "pillow-11.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:6ebce70c3f486acf7591a3d73431fa504a4e18a9b97ff27f5f47b7368e4b9dd1"}, - {file = "pillow-11.2.1-cp39-cp39-win_arm64.whl", hash = "sha256:c27476257b2fdcd7872d54cfd119b3a9ce4610fb85c8e32b70b42e3680a29a1e"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9b7b0d4fd2635f54ad82785d56bc0d94f147096493a79985d0ab57aedd563156"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:aa442755e31c64037aa7c1cb186e0b369f8416c567381852c63444dd666fb772"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0d3348c95b766f54b76116d53d4cb171b52992a1027e7ca50c81b43b9d9e363"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85d27ea4c889342f7e35f6d56e7e1cb345632ad592e8c51b693d7b7556043ce0"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bf2c33d6791c598142f00c9c4c7d47f6476731c31081331664eb26d6ab583e01"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e616e7154c37669fc1dfc14584f11e284e05d1c650e1c0f972f281c4ccc53193"}, - {file = "pillow-11.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:39ad2e0f424394e3aebc40168845fee52df1394a4673a6ee512d840d14ab3013"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:80f1df8dbe9572b4b7abdfa17eb5d78dd620b1d55d9e25f834efdbee872d3aed"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ea926cfbc3957090becbcbbb65ad177161a2ff2ad578b5a6ec9bb1e1cd78753c"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:738db0e0941ca0376804d4de6a782c005245264edaa253ffce24e5a15cbdc7bd"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db98ab6565c69082ec9b0d4e40dd9f6181dab0dd236d26f7a50b8b9bfbd5076"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:036e53f4170e270ddb8797d4c590e6dd14d28e15c7da375c18978045f7e6c37b"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:14f73f7c291279bd65fda51ee87affd7c1e097709f7fdd0188957a16c264601f"}, - {file = "pillow-11.2.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:208653868d5c9ecc2b327f9b9ef34e0e42a4cdd172c2988fd81d62d2bc9bc044"}, - {file = "pillow-11.2.1.tar.gz", hash = "sha256:a64dd61998416367b7ef979b73d3a85853ba9bec4c2925f74e588879a58716b6"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e"}, + {file = "pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6"}, + {file = "pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f"}, + {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94"}, + {file = "pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0"}, + {file = "pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac"}, + {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d"}, + {file = "pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149"}, + {file = "pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d"}, + {file = "pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b"}, + {file = "pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3"}, + {file = "pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51"}, + {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c"}, + {file = "pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788"}, + {file = "pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31"}, + {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a"}, + {file = "pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214"}, + {file = "pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635"}, + {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b"}, + {file = "pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12"}, + {file = "pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db"}, + {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d"}, + {file = "pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71"}, + {file = "pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada"}, + {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, + {file = "pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523"}, ] [package.extras] -docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] fpx = ["olefile"] mic = ["olefile"] test-arrow = ["pyarrow"] -tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] From cb4b5585a4574c95970be3d87d798dc1cd4adf19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:39:13 +0000 Subject: [PATCH 422/691] Bump prometheus-client from 0.21.0 to 0.22.1 (#18609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [prometheus-client](https://github.com/prometheus/client_python) from 0.21.0 to 0.22.1.
Release notes

Sourced from prometheus-client's releases.

v0.22.1

What's Changed

Full Changelog: https://github.com/prometheus/client_python/compare/v0.22.0...v0.22.1

v0.22.0

What's Changed

Full Changelog: https://github.com/prometheus/client_python/compare/v0.21.0...v0.22.0

0.21.1 / 2024-12-03

What's Changed

[BUGFIX] Revert incorrect use of reentrant locks. #1076

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=prometheus-client&package-manager=pip&previous-version=0.21.0&new-version=0.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 64570b1ba0..964cd38bf1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1698,14 +1698,14 @@ xmp = ["defusedxml"] [[package]] name = "prometheus-client" -version = "0.21.0" +version = "0.22.1" description = "Python client for the Prometheus monitoring system." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"}, - {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"}, + {file = "prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094"}, + {file = "prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28"}, ] [package.extras] From c2108948a370d01fac8692a1908d4e9d418b5dc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:51:43 +0000 Subject: [PATCH 423/691] Bump treq from 24.9.1 to 25.5.0 (#18610) Bumps [treq](https://github.com/twisted/treq) from 24.9.1 to 25.5.0.
Changelog

Sourced from treq's changelog.

25.5.0 (2025-05-31)

Features

  • treq is packaged with Hatchling, and consequently no longer directly depends on setuptools. ([#388](https://github.com/twisted/treq/issues/388) <https://github.com/twisted/treq/issues/388>__)

Improved Documentation

  • Update documentation to use async/await syntax ([#409](https://github.com/twisted/treq/issues/409) <https://github.com/twisted/treq/issues/409>__)

Deprecations and Removals

  • Support for Python 3.8, which has reached end of support, is deprecated. This is the last release with support for Python 3.8. ([#407](https://github.com/twisted/treq/issues/407) <https://github.com/twisted/treq/issues/407>__)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=treq&package-manager=pip&previous-version=24.9.1&new-version=25.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 964cd38bf1..04cc71de79 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1436,6 +1436,22 @@ files = [ {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, ] +[[package]] +name = "multipart" +version = "1.2.1" +description = "Parser for multipart/form-data" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "multipart-1.2.1-py3-none-any.whl", hash = "sha256:c03dc203bc2e67f6b46a599467ae0d87cf71d7530504b2c1ff4a9ea21d8b8c8c"}, + {file = "multipart-1.2.1.tar.gz", hash = "sha256:829b909b67bc1ad1c6d4488fcdc6391c2847842b08323addf5200db88dbe9480"}, +] + +[package.extras] +dev = ["build", "pytest", "pytest-cov", "twine"] +docs = ["sphinx (>=8,<9)", "sphinx-autobuild"] + [[package]] name = "mypy" version = "1.13.0" @@ -2810,27 +2826,28 @@ dev = ["furo (>=2024.05.06)", "nox", "packaging", "sphinx (>=5)", "twisted"] [[package]] name = "treq" -version = "24.9.1" +version = "25.5.0" description = "High-level Twisted HTTP Client API" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8.0" groups = ["main"] files = [ - {file = "treq-24.9.1-py3-none-any.whl", hash = "sha256:eee4756fd9a857c77f180fd5202b52c518f2d3e2826dce28b89066c03bfc45d0"}, - {file = "treq-24.9.1.tar.gz", hash = "sha256:15da7fc404f3e4ed59d0abe5f8eef4966fabbe618039a2a23bc7c15305cefea8"}, + {file = "treq-25.5.0-py3-none-any.whl", hash = "sha256:e99d4e66cacaa1f0da82bb60b317d104c29dbd8ac0a75d7f657b348178d830f4"}, + {file = "treq-25.5.0.tar.gz", hash = "sha256:25dde3a55ae85ec2f2c56332c99aef255ab14f997d0d00552ebff13538a9804a"}, ] [package.dependencies] attrs = "*" hyperlink = ">=21.0.0" -incremental = "*" +incremental = ">=24.7.2" +multipart = "*" requests = ">=2.1.0" -Twisted = {version = ">=22.10.0", extras = ["tls"]} +twisted = {version = ">=22.10.0", extras = ["tls"]} typing-extensions = ">=3.10.0" [package.extras] dev = ["httpbin (==0.7.0)", "pep8", "pyflakes", "werkzeug (==2.0.3)"] -docs = ["sphinx (<7.0.0)"] +docs = ["sphinx", "sphinx-rtd-theme"] [[package]] name = "twine" From 717f67f3d304b9ee1d034d2c3ee6060256f666e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:54:29 +0000 Subject: [PATCH 424/691] Bump Swatinem/rust-cache from 2.7.8 to 2.8.0 (#18612) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) from 2.7.8 to 2.8.0.
Release notes

Sourced from Swatinem/rust-cache's releases.

v2.8.0

What's Changed

New Contributors

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.7.8...v2.8.0

Changelog

Sourced from Swatinem/rust-cache's changelog.

Changelog

2.8.0

  • Add support for warpbuild cache provider
  • Add new cache-workspace-crates feature

2.7.8

  • Include CPU arch in the cache key

2.7.7

  • Also cache cargo install metadata

2.7.6

  • Allow opting out of caching $CARGO_HOME/bin
  • Add runner OS in cache key
  • Adds an option to do lookup-only of the cache

2.7.5

  • Support Cargo.lock format cargo-lock v4
  • Only run macOsWorkaround() on macOS

2.7.3

  • Work around upstream problem that causes cache saving to hang for minutes.

2.7.2

  • Only key by Cargo.toml and Cargo.lock files of workspace members.

2.7.1

  • Update toml parser to fix parsing errors.

2.7.0

  • Properly cache trybuild tests.

2.6.2

  • Fix toml parsing.

2.6.1

  • Fix hash contributions of Cargo.lock/Cargo.toml files.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Swatinem/rust-cache&package-manager=github_actions&previous-version=2.7.8&new-version=2.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 6 +++--- .github/workflows/tests.yml | 24 ++++++++++++------------ .github/workflows/twisted_trunk.yml | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 1547eb6197..56be48fd65 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -22,7 +22,7 @@ jobs: # clippy correctly fixes up the benchmarks. toolchain: nightly-2022-12-01 components: clippy, rustfmt - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index ee0dac3beb..917131acaa 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -76,7 +76,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -149,7 +149,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9156e81ba..e2de0ac2a9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -150,7 +150,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -211,7 +211,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: poetry-version: "2.1.1" @@ -230,7 +230,7 @@ jobs: uses: dtolnay/rust-toolchain@0d72692bcfbf448b1e2afa01a67f71b455a9dcec # 1.86.0 with: components: clippy - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy -- -D warnings @@ -249,7 +249,7 @@ jobs: with: toolchain: nightly-2025-04-23 components: clippy - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy --all-features -- -D warnings @@ -267,7 +267,7 @@ jobs: # We use nightly so that it correctly groups together imports toolchain: nightly-2025-04-23 components: rustfmt - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo fmt --check @@ -363,7 +363,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -405,7 +405,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -520,7 +520,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Run SyTest run: /bootstrap.sh synapse @@ -664,7 +664,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh @@ -696,7 +696,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo test @@ -716,7 +716,7 @@ jobs: uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo bench --no-run diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 5638029b39..0d3a65c1bc 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -44,7 +44,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -70,7 +70,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -114,7 +114,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8 + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, From 3bd476eb0d448ea35c258ddd64662a3b5a3e7078 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:44:22 +0000 Subject: [PATCH 425/691] Bump tokio from 1.45.1 to 1.46.0 (#18628) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.45.1 to 1.46.0.
Release notes

Sourced from tokio's releases.

Tokio v1.46.0

1.46.0 (July 2nd, 2025)

Fixed

  • net: fixed TcpStream::shutdown incorrectly returning an error on macOS (#7290)

Added

  • sync: mpsc::OwnedPermit::{same_channel, same_channel_as_sender} methods (#7389)
  • macros: biased option for join! and try_join!, similar to select! (#7307)
  • net: support for cygwin (#7393)
  • net: support pope::OpenOptions::read_write on Android (#7426)
  • net: add Clone implementation for net::unix::SocketAddr (#7422)

Changed

  • runtime: eliminate unnecessary lfence while operating on queue::Local<T> (#7340)
  • task: disallow blocking in LocalSet::{poll,drop} (#7372)

Unstable

  • runtime: add TaskMeta::spawn_location tracking where a task was spawned (#7417)
  • runtime: removed borrow from LocalOptions parameter to runtime::Builder::build_local (#7346)

Documented

  • io: clarify behavior of seeking when start_seek is not used (#7366)
  • io: document cancellation safety of AsyncWriteExt::flush (#7364)
  • net: fix docs for recv_buffer_size method (#7336)
  • net: fix broken link of RawFd in TcpSocket docs (#7416)
  • net: update AsRawFd doc link to current Rust stdlib location (#7429)
  • readme: fix double period in reactor description (#7363)
  • runtime: add doc note that on_*_task_poll is unstable (#7311)
  • sync: update broadcast docs on allocation failure (#7352)
  • time: add a missing panic scenario of time::advance (#7394)

#7290: tokio-rs/tokio#7290 #7307: tokio-rs/tokio#7307 #7311: tokio-rs/tokio#7311 #7336: tokio-rs/tokio#7336 #7340: tokio-rs/tokio#7340 #7346: tokio-rs/tokio#7346 #7352: tokio-rs/tokio#7352 #7363: tokio-rs/tokio#7363 #7364: tokio-rs/tokio#7364 #7366: tokio-rs/tokio#7366 #7372: tokio-rs/tokio#7372 #7389: tokio-rs/tokio#7389 #7393: tokio-rs/tokio#7393

... (truncated)

Commits
  • 3f1f268 chore: prepare Tokio v1.46.0 (#7437)
  • 3e890cc rt(unstable): add spawn Location to TaskMeta (#7417)
  • 69290a6 net: derive Clone for net::unix::SocketAddr (#7422)
  • e2b1758 fuzz: cfg fuzz tests under cfg(test) (#7428)
  • b7a75b5 net: update AsRawFd doc link to current Rust stdlib location (#7429)
  • 6b705b3 net: allow pipe::OpenOptions::read_write on Android (#7426)
  • 3636fd0 net: fix broken link of RawFd in TcpSocket docs (#7416)
  • 2506c9f benches: revert "properly gate unix benches" (#7412)
  • b3a1448 sync: improve docs of tokio_util::sync::CancellationToken (#7408)
  • 013f323 docs: add a missing panic scenario of time::advance (#7394)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tokio&package-manager=cargo&previous-version=1.45.1&new-version=1.46.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b85f5f8b98..c9ba99e29c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,6 +664,17 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1429,15 +1440,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", + "slab", "socket2", "windows-sys 0.52.0", ] From 87fc518e0c25f0285fff2be7e2e09b72ca951f3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:44:42 +0000 Subject: [PATCH 426/691] Bump base64 from 0.21.7 to 0.22.1 (#18629) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.21.7 to 0.22.1.
Changelog

Sourced from base64's changelog.

0.22.1

  • Correct the symbols used for the predefined alphabet::BIN_HEX.

0.22.0

  • DecodeSliceError::OutputSliceTooSmall is now conservative rather than precise. That is, the error will only occur if the decoded output cannot fit, meaning that Engine::decode_slice can now be used with exactly-sized output slices. As part of this, Engine::internal_decode now returns DecodeSliceError instead of DecodeError, but that is not expected to affect any external callers.
  • DecodeError::InvalidLength now refers specifically to the number of valid symbols being invalid (i.e. len % 4 == 1), rather than just the number of input bytes. This avoids confusing scenarios when based on interpretation you could make a case for either InvalidLength or InvalidByte being appropriate.
  • Decoding is somewhat faster (5-10%)
Commits
  • e144006 v0.22.1
  • 64cca59 Merge pull request #271 from JobanSD/patch-1
  • 838355e Correct BinHex 4.0 alphabet according to specifications
  • bf15ccf Merge pull request #270 from marshallpierce/mp/clippy
  • fc6aabe Appease clippy
  • 9a518a2 Merge pull request #267 from bdura/patch-1
  • d96c80f Merge branch 'marshallpierce:master' into patch-1
  • 5d70ba7 Merge pull request #269 from marshallpierce/mp/decode-precisely
  • efb6c00 Release notes
  • 2b91084 Add some tests to boost coverage
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=base64&package-manager=cargo&previous-version=0.21.7&new-version=0.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c9ba99e29c..de2b4e68cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,12 +65,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -371,7 +365,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "headers-core", "http", @@ -491,7 +485,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -1070,7 +1064,7 @@ version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-util", @@ -1344,7 +1338,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.7", + "base64", "blake2", "bytes", "futures", From e6b8eedd0290e1a66dd1567b752cbff57c67b52c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 2 Jul 2025 19:48:28 +0100 Subject: [PATCH 427/691] Update Rust in CI to v1.87.0 as well as `dtolnay/rust-toolchain` GitHub Action pinned commit hash (#18596) --- .github/workflows/fix_lint.yaml | 11 +++++--- .github/workflows/latest_deps.yml | 15 ++++++++-- .github/workflows/tests.yml | 44 +++++++++++++++++++++-------- .github/workflows/twisted_trunk.yml | 15 ++++++++-- changelog.d/18596.misc | 1 + 5 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 changelog.d/18596.misc diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 56be48fd65..845e90d27c 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -6,6 +6,11 @@ name: Attempt to automatically fix linting errors on: workflow_dispatch: +env: + # We use nightly so that `fmt` correctly groups together imports, and + # clippy correctly fixes up the benchmarks. + RUST_VERSION: nightly-2025-06-24 + jobs: fixup: name: Fix up @@ -16,11 +21,9 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: - # We use nightly so that `fmt` correctly groups together imports, and - # clippy correctly fixes up the benchmarks. - toolchain: nightly-2022-12-01 + toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 917131acaa..702f1a5381 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -21,6 +21,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -41,7 +44,9 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # The dev dependencies aren't exposed in the wheel metadata (at least with current @@ -75,7 +80,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: sudo apt-get -qq install xmlsec1 @@ -148,7 +155,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Ensure sytest runs `pip install` diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e2de0ac2a9..f6250c8ed2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: # Job to detect what has changed so we don't run e.g. Rust checks on PRs that # don't modify Rust code. @@ -85,7 +88,9 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -149,7 +154,9 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Setup Poetry @@ -210,7 +217,9 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -227,9 +236,10 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@0d72692bcfbf448b1e2afa01a67f71b455a9dcec # 1.86.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: components: clippy + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo clippy -- -D warnings @@ -245,7 +255,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: toolchain: nightly-2025-04-23 components: clippy @@ -262,7 +272,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: # We use nightly so that it correctly groups together imports toolchain: nightly-2025-04-23 @@ -362,7 +372,9 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -404,7 +416,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 # There aren't wheels for some of the older deps, so we need to install @@ -519,7 +533,9 @@ jobs: run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Run SyTest @@ -663,7 +679,9 @@ jobs: path: synapse - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Prepare Complement's Prerequisites @@ -695,7 +713,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@c1678930c21fb233e4987c4ae12158f9125e5762 # 1.81.0 + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - run: cargo test @@ -713,7 +733,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: toolchain: nightly-2022-12-01 - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 0d3a65c1bc..15aa1886af 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -20,6 +20,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: 1.87.0 + jobs: check_repo: # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is @@ -43,7 +46,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -69,7 +74,9 @@ jobs: - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -113,7 +120,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1) + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 - name: Patch dependencies diff --git a/changelog.d/18596.misc b/changelog.d/18596.misc new file mode 100644 index 0000000000..ce7bfdd129 --- /dev/null +++ b/changelog.d/18596.misc @@ -0,0 +1 @@ +Update to Rust 1.87.0 in CI, and bump the pinned commit of the `dtolnay/rust-toolchain` GitHub Action to `b3b07ba8b418998c39fb20f53e8b695cdcc8de1b`. \ No newline at end of file From dc974fdf83927070fa140a7227ec4b457f383322 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 07:35:31 +0000 Subject: [PATCH 428/691] Bump sentry-sdk from 2.22.0 to 2.32.0 (#18633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 2.22.0 to 2.32.0.
Release notes

Sourced from sentry-sdk's releases.

2.32.0

Various fixes & improvements

2.31.0

Various fixes & improvements

import sentry_sdk
from sentry_sdk.integrations.openai_agents import
OpenAIAgentsIntegration

Add the OpenAIAgentsIntegration to your sentry_sdk.init call:


sentry_sdk.init(

dsn="...",

integrations=[

OpenAIAgentsIntegration(),

]

)

For more information see the OpenAI Agents integrations documentation.

2.30.0

Various fixes & improvements

  • New beta feature: Sentry logs for Loguru (#4445) by @​sentrivana

    We can now capture Loguru logs and send them to Sentry.

... (truncated)

Changelog

Sourced from sentry-sdk's changelog.

2.32.0

Various fixes & improvements

2.31.0

Various fixes & improvements

import sentry_sdk
from sentry_sdk.integrations.openai_agents import
OpenAIAgentsIntegration

Add the OpenAIAgentsIntegration to your sentry_sdk.init call:


sentry_sdk.init(

dsn="...",

integrations=[

OpenAIAgentsIntegration(),

]

)

For more information see the OpenAI Agents integrations documentation.

2.30.0

Various fixes & improvements

... (truncated)

Commits
  • 4b022dc Merge branch 'master' into release/2.32.0
  • 2634a52 Pin zope.event (#4531)
  • c815a32 updated changelog
  • bca8816 release: 2.32.0
  • 546ce1f Set tool span to failed if an error is raised in the tool (#4527)
  • ab2e3f0 fix(integrations/ray): Correctly pass keyword arguments to ray.remote functio...
  • 7804260 fix(langchain): Make span_map an instance variable (#4476)
  • 0a2d858 fix(langchain): Ensure no duplicate SentryLangchainCallback (#4485)
  • dae0218 fix(Litestar): Apply failed_request_status_codes to exceptions raised in mi...
  • 65d31af Merge branch 'release/2.31.0'
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sentry-sdk&package-manager=pip&previous-version=2.22.0&new-version=2.32.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 04cc71de79..61423bd8de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2515,15 +2515,15 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.22.0" +version = "2.32.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] markers = "extra == \"all\" or extra == \"sentry\"" files = [ - {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"}, - {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"}, + {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, + {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, ] [package.dependencies] From 9d9e14034a69dc5f18877ce2277de5ef531fc5a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 07:40:30 +0000 Subject: [PATCH 429/691] Bump jsonschema from 4.23.0 to 4.24.0 (#18630) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.23.0 to 4.24.0.
Release notes

Sourced from jsonschema's releases.

v4.24.0

What's Changed

New Contributors

Full Changelog: https://github.com/python-jsonschema/jsonschema/compare/v4.23.0...v4.24.0

Changelog

Sourced from jsonschema's changelog.

v4.24.0

  • Fix improper handling of unevaluatedProperties in the presence of additionalProperties (#1351).
  • Support for Python 3.8 has been dropped, as it is end-of-life.
Commits
  • 3e23ee5 Add the bugfix to the changelog.
  • 8917e85 Stop running CIFuzz.
  • 737e5ed Rely on ruff in pre-commit.
  • 57e5e03 Test via PyPy 3.11.
  • d6c2ad7 Add the zizmor setup here as well.
  • af9a857 Drop a dead pyproject section.
  • c64ef84 This is less true than it once was...
  • 9ff926f Merge branch 'additional-evaluated'
  • 8290667 We still need to ditch pip-licenses...
  • 6d973b5 Update pre-commit hooks.
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=jsonschema&package-manager=pip&previous-version=4.23.0&new-version=4.24.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 61423bd8de..98efb5a4c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -960,14 +960,14 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jsonschema" -version = "4.23.0" +version = "4.24.0" description = "An implementation of JSON Schema validation for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, - {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, + {file = "jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d"}, + {file = "jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196"}, ] [package.dependencies] From fb9d737250e4c6588450fd919930c9ed7784486f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 08:17:26 +0000 Subject: [PATCH 430/691] Bump types-bleach from 6.2.0.20241123 to 6.2.0.20250514 (#18634) Bumps [types-bleach](https://github.com/typeshed-internal/stub_uploader) from 6.2.0.20241123 to 6.2.0.20250514.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-bleach&package-manager=pip&previous-version=6.2.0.20241123&new-version=6.2.0.20250514)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 98efb5a4c1..0256ab3da1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2935,14 +2935,14 @@ twisted = "*" [[package]] name = "types-bleach" -version = "6.2.0.20241123" +version = "6.2.0.20250514" description = "Typing stubs for bleach" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_bleach-6.2.0.20241123-py3-none-any.whl", hash = "sha256:c6e58b3646665ca7c6b29890375390f4569e84f0cf5c171e0fe1ddb71a7be86a"}, - {file = "types_bleach-6.2.0.20241123.tar.gz", hash = "sha256:dac5fe9015173514da3ac810c1a935619a3ccbcc5d66c4cbf4707eac00539057"}, + {file = "types_bleach-6.2.0.20250514-py3-none-any.whl", hash = "sha256:380cb74f0db1e3c3b2e0cde217221108e975e07e95ef0970c9d41f7cd4e8ea3c"}, + {file = "types_bleach-6.2.0.20250514.tar.gz", hash = "sha256:38c2e51d9cac51dc70c1b66121a11f4dad8bbf47fbad494bb7a77d8b8f3c4323"}, ] [package.dependencies] From 777c3f56ded45320a26ed91822ca97ebd4f13a3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 08:31:48 +0000 Subject: [PATCH 431/691] Bump reqwest from 0.12.20 to 0.12.22 (#18627) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.20 to 0.12.22.
Release notes

Sourced from reqwest's releases.

v0.12.22

tl;dr

  • Fix socks proxies when resolving IPv6 destinations.

What's Changed

Full Changelog: https://github.com/seanmonstar/reqwest/compare/v0.12.21...v0.12.22

v0.12.21

tl;dr

  • Fix socks proxy to use socks4a:// instead of socks4h://.
  • Fix Error::is_timeout() to check for hyper and IO timeouts too.
  • Fix request Error to again include URLs when possible.
  • Fix socks connect error to include more context.
  • (wasm) implement Default for Body.

What's Changed

New Contributors

Full Changelog: https://github.com/seanmonstar/reqwest/compare/v0.12.20...v0.12.21

Changelog

Sourced from reqwest's changelog.

v0.12.22

  • Fix socks proxies when resolving IPv6 destinations.

v0.12.21

  • Fix socks proxy to use socks4a:// instead of socks4h://.
  • Fix Error::is_timeout() to check for hyper and IO timeouts too.
  • Fix request Error to again include URLs when possible.
  • Fix socks connect error to include more context.
  • (wasm) implement Default for Body.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=reqwest&package-manager=cargo&previous-version=0.12.20&new-version=0.12.22)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de2b4e68cf..0ac5280c17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1060,9 +1060,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.20" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "base64", "bytes", From ac268051f2535c446f85722bc9e164b283cb9bfa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 08:38:40 +0000 Subject: [PATCH 432/691] Bump hiredis from 3.1.0 to 3.2.1 (#18638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [hiredis](https://github.com/redis/hiredis-py) from 3.1.0 to 3.2.1.
Release notes

Sourced from hiredis's releases.

3.2.1

Changes

  • Fix assertion in reader.c (#212)

Contributors

We'd like to thank all the contributors who worked on this release!

@​uglide

3.2.0

Changes

  • Introduce new type for RESP3 PUSH notifications (#208)
  • Run integration workflow on version branches (#210)
  • Update hiredis to 1.3.0 (#203)

Contributors

We'd like to thank all the contributors who worked on this release!

@​uglide

3.1.1

Summary

This is a maintenance release that addresses issues in RESP3 parsing.

Changes

  • Fix memory leaks and segfaults in RESP3 map parsing (#204, #205, #206)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=hiredis&package-manager=pip&previous-version=3.1.0&new-version=3.2.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 220 ++++++++++++++++++++++++++-------------------------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0256ab3da1..22953577fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -538,122 +538,122 @@ test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3. [[package]] name = "hiredis" -version = "3.1.0" +version = "3.2.1" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"all\" or extra == \"redis\"" files = [ - {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:2892db9db21f0cf7cc298d09f85d3e1f6dc4c4c24463ab67f79bc7a006d51867"}, - {file = "hiredis-3.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:93cfa6cc25ee2ceb0be81dc61eca9995160b9e16bdb7cca4a00607d57e998918"}, - {file = "hiredis-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2af62070aa9433802cae7be7364d5e82f76462c6a2ae34e53008b637aaa9a156"}, - {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:072c162260ebb1d892683107da22d0d5da7a1414739eae4e185cac22fe89627f"}, - {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6b232c43e89755ba332c2745ddab059c0bc1a0f01448a3a14d506f8448b1ce6"}, - {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb5316c9a65c4dde80796aa245b76011bab64eb84461a77b0a61c1bf2970bcc9"}, - {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e812a4e656bbd1c1c15c844b28259c49e26bb384837e44e8d2aa55412c91d2f7"}, - {file = "hiredis-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93a6c9230e5a5565847130c0e1005c8d3aa5ca681feb0ed542c4651323d32feb"}, - {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a5f65e89ce50a94d9490d5442a649c6116f53f216c8c14eb37cf9637956482b2"}, - {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9b2d6e33601c67c074c367fdccdd6033e642284e7a56adc130f18f724c378ca8"}, - {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bad3b1e0c83849910f28c95953417106f539277035a4b515d1425f93947bc28f"}, - {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9646de31f5994e6218311dcf216e971703dbf804c510fd3f84ddb9813c495824"}, - {file = "hiredis-3.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:59a9230f3aa38a33d09d8171400de202f575d7a38869e5ce2947829bca6fe359"}, - {file = "hiredis-3.1.0-cp310-cp310-win32.whl", hash = "sha256:0322d70f3328b97da14b6e98b18f0090a12ed8a8bf7ae20932e2eb9d1bb0aa2c"}, - {file = "hiredis-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:802474c18e878b3f9905e160a8b7df87d57885758083eda76c5978265acb41aa"}, - {file = "hiredis-3.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:c339ff4b4739b2a40da463763dd566129762f72926bca611ad9a457a9fe64abd"}, - {file = "hiredis-3.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:0ffa2552f704a45954627697a378fc2f559004e53055b82f00daf30bd4305330"}, - {file = "hiredis-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9acf7f0e7106f631cd618eb60ec9bbd6e43045addd5310f66ba1177209567e59"}, - {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea4f5ecf9dbea93c827486f59c606684c3496ea71c7ba9a8131932780696e61a"}, - {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39efab176fca3d5111075f6ba56cd864f18db46d858289d39360c5672e0e5c3e"}, - {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1110eae007f30e70a058d743e369c24430327cd01fd97d99519d6794a58dd587"}, - {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b390f63191bcccbb6044d4c118acdf4fa55f38e5658ac4cfd5a33a6f0c07659"}, - {file = "hiredis-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72a98ccc7b8ec9ce0100ecf59f45f05d2023606e8e3676b07a316d1c1c364072"}, - {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7c76e751fd1e2f221dec09cdc24040ee486886e943d5d7ffc256e8cf15c75e51"}, - {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7d3880f213b6f14e9c69ce52beffd1748eecc8669698c4782761887273b6e1bd"}, - {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:87c2b3fe7e7c96eba376506a76e11514e07e848f737b254e0973e4b5c3a491e9"}, - {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d3cfb4089e96f8f8ee9554da93148a9261aa6612ad2cc202c1a494c7b712e31f"}, - {file = "hiredis-3.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f12018e5c5f866a1c3f7017cb2d88e5c6f9440df2281e48865a2b6c40f247f4"}, - {file = "hiredis-3.1.0-cp311-cp311-win32.whl", hash = "sha256:107b66ce977bb2dff8f2239e68344360a75d05fed3d9fa0570ac4d3020ce2396"}, - {file = "hiredis-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8f1240bde53d3d1676f0aba61b3661560dc9a681cae24d9de33e650864029aa4"}, - {file = "hiredis-3.1.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:f7c7f89e0bc4246115754e2eda078a111282f6d6ecc6fb458557b724fe6f2aac"}, - {file = "hiredis-3.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:3dbf9163296fa45fbddcfc4c5900f10e9ddadda37117dbfb641e327e536b53e0"}, - {file = "hiredis-3.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af46a4be0e82df470f68f35316fa16cd1e134d1c5092fc1082e1aad64cce716d"}, - {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc63d698c43aea500a84d8b083f830c03808b6cf3933ae4d35a27f0a3d881652"}, - {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:676b3d88674134bfaaf70dac181d1790b0f33b3187bfb9da9221e17e0e624f83"}, - {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aed10d9df1e2fb0011db2713ac64497462e9c2c0208b648c97569da772b959ca"}, - {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b5bd8adfe8742e331a94cccd782bffea251fa70d9a709e71f4510f50794d700"}, - {file = "hiredis-3.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fc4e35b4afb0af6da55495dd0742ad32ab88150428a6ecdbb3085cbd60714e8"}, - {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:89b83e76eb00ab0464e7b0752a3ffcb02626e742e9509bc141424a9c3202e8dc"}, - {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:98ebf08c907836b70a8f40e030df8ab6f174dc7f6fa765251d813e89f14069d8"}, - {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6c840b9cec086328f2ee2cfee0038b5d6bbb514bac7b5e579da6e346eaac056c"}, - {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:c5c44e9fa6f4462d0330cb5f5d46fa652512fc86b41d4d1974d0356f263e9105"}, - {file = "hiredis-3.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e665b14ab50aa175cfa306fcb00fffd4e3ff02ceb36ca6a4df00b1246d6a73c4"}, - {file = "hiredis-3.1.0-cp312-cp312-win32.whl", hash = "sha256:bd33db977ac7af97e8d035ffadb163b00546be22e5f1297b2123f5f9bf0f8a21"}, - {file = "hiredis-3.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:37aed4aa9348600145e2d019c7be27855e503ecc4906c6976ff2f3b52e3d5d97"}, - {file = "hiredis-3.1.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:b87cddd8107487863fed6994de51e5594a0be267b0b19e213694e99cdd614623"}, - {file = "hiredis-3.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:d302deff8cb63a7feffc1844e4dafc8076e566bbf10c5aaaf0f4fe791b8a6bd0"}, - {file = "hiredis-3.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a018340c073cf88cb635b2bedff96619df2f666018c655e7911f46fa2c1c178"}, - {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e8ba6414ac1ae536129e18c069f3eb497df5a74e136e3566471620a4fa5f95"}, - {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a86b9fef256c2beb162244791fdc025aa55f936d6358e86e2020e512fe2e4972"}, - {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7acdc68e29a446ad17aadaff19c981a36b3bd8c894c3520412c8a7ab1c3e0de7"}, - {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7e06baea05de57e1e7548064f505a6964e992674fe61b8f274afe2ac93b6371"}, - {file = "hiredis-3.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b5fc061c8a0dbfdb440053280504d6aaa8d9726bd4d1d0e1cfcbbdf0d60b73"}, - {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c89d2dcb271d24c44f02264233b75d5db8c58831190fa92456a90b87fa17b748"}, - {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:aa36688c10a08f626fddcf68c2b1b91b0e90b070c26e550a4151a877f5c2d431"}, - {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f3982a9c16c1c4bc05a00b65d01ffb8d80ea1a7b6b533be2f1a769d3e989d2c0"}, - {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d1a6f889514ee2452300c9a06862fceedef22a2891f1c421a27b1ba52ef130b2"}, - {file = "hiredis-3.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8a45ff7915392a55d9386bb235ea1d1eb9960615f301979f02143fc20036b699"}, - {file = "hiredis-3.1.0-cp313-cp313-win32.whl", hash = "sha256:539e5bb725b62b76a5319a4e68fc7085f01349abc2316ef3df608ea0883c51d2"}, - {file = "hiredis-3.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9020fd7e58f489fda6a928c31355add0e665fd6b87b21954e675cf9943eafa32"}, - {file = "hiredis-3.1.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:b621a89fc29b3f4b01be6640ec81a6a94b5382bc78fecb876408d57a071e45aa"}, - {file = "hiredis-3.1.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:363e21fba55e1a26349dc9ca7da6b14332123879b6359bcee4a9acecb40ca33b"}, - {file = "hiredis-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c156156798729eadc9ab76ffee96c88b93cc1c3b493f4dd0a4341f53939194ee"}, - {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e38d8a325f9a6afac1b1c72d996d1add9e1b99696ce9410538ba5e9aa8fdba02"}, - {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3004ef7436feb7bfa61c0b36d422b8fb8c29aaa1a514c9405f0fdee5e9694dd3"}, - {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13f5b16f97d0bbd1c04ce367c49097d1214d60e11f9fee7ef2a9b54e0a6645c8"}, - {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:230dd0e77cb0f525f58a1306a7b4aaf078037fc5229110922332ca46f90821bb"}, - {file = "hiredis-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d968116caddd19d63120d1298e62b1bbc694db3360ed0d5df8c3a97edbc12552"}, - {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:511e36a6fa41d3efab3cd5cd70ac388ed825993b9e66fa3b0e47cf27a2f5ffee"}, - {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c5cd20804e3cb0d31e7d899d8dd091f569c33fe40d4bade670a067ab7d31c2ac"}, - {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:09e89e7d34cfe5ca8f7a869fca827d1af0afe8aaddb26b38c01058730edb79ad"}, - {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:570cbf31413c77fe5e7c157f2943ca4400493ddd9cf2184731cfcafc753becd7"}, - {file = "hiredis-3.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:b9b4da8162cf289781732d6a5ba01d820c42c05943fcdb7de307d03639961db3"}, - {file = "hiredis-3.1.0-cp38-cp38-win32.whl", hash = "sha256:bc117a04bcb461d3bb1b2c5b417aee3442e1e8aa33ebc800481431f4c09fe0c5"}, - {file = "hiredis-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:34f3f5f0354db2d6797a6fb08d2c036a50af62a1d919d122c1c784304ef49347"}, - {file = "hiredis-3.1.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:a26fa888025badb5563f283cc19594c215a413e905729e59a5f7cf3f46d66c32"}, - {file = "hiredis-3.1.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:f50763cd819d4a52a47b5966d4bb47dee34b637c5fa6402509800eee6ecb61e6"}, - {file = "hiredis-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b6d1c9e1fce5e0a94072667ae2bf0142b89ebbb1917d3531184e060a43f3ee11"}, - {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e38d7a56b1a79ed0bbb9e6fe376d82e3f4dcc646ae47472f2c858e19a597c112"}, - {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ef5ad8b91530e4d10a68562b0a380ea22705a60e88cecee086d7c63a38564ce"}, - {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf3d2299b054e57a9f97ca08704c2843e44f29b57dc69b76a2592ecd212efe1a"}, - {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93811d60b0f73d0f049c86f4373a3833b4a38fce374ab151074d929553eb4304"}, - {file = "hiredis-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e703ff860c1d83abbcf57012b309ead02b56b60e85150c6c3bfb37cbb16ebf"}, - {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f9ea0678806c53d96758e74c6a898f9d506a2e3367a344757f768bef9e069366"}, - {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cf6844035abf47d52a1c3f4257255af3bf3b0f14d559b08eaa45885418c6c55d"}, - {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7acf35cfa7ec9e1e7559c04e7095628f7d06049b5f24dcb58c1a55ef6dc689f8"}, - {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b885695dce7a39b1fd9a609ed9c4cf312e53df2ec028d5a78af7a891b5fbea4d"}, - {file = "hiredis-3.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1c22fa74ddd063396b19fe8445a1ae8b4190eff755d5750dda48e860a45b2ee7"}, - {file = "hiredis-3.1.0-cp39-cp39-win32.whl", hash = "sha256:0614e16339f1784df3bbd2800322e20b4127d3f3a3509f00a5562efddb2521aa"}, - {file = "hiredis-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:c2bc713ee73ab9de4a0d68b0ab0f29612342b63173714742437b977584adb2d8"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:07ab990d0835f36bf358dbb84db4541ac0a8f533128ec09af8f80a576eef2e88"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c54a88eb9d8ebc4e5eefaadbe2102a4f7499f9e413654172f40aefd25350959"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8095ef159896e5999a795b0f80e4d64281301a109e442a8d29cd750ca6bd8303"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f8ca13e2476ffd6d5be4763f5868133506ddcfa5ce54b4dac231ebdc19be6c6"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34d25aa25c10f966d5415795ed271da84605044dbf436c054966cea5442451b3"}, - {file = "hiredis-3.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4180dc5f646b426e5fa1212e1348c167ee2a864b3a70d56579163d64a847dd1e"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d92144e0cd6e6e841a6ad343e9d58631626eeb4ac96b0322649379b5d4527447"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb91ba42903de637b94a1b64477f381f94ad82c0742c264f9245be76a7a3cbc"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ce71a797b5bc02c51da082428c00251ed6a7a67a03acbda5fbf9e8d028725f6"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e04c7feb9467e3170cd4d5bee381775783d81bbc45d6147c1c0ce3b50dc04f9"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a31806306a60f3565c04c964d6bee0e9d4a5120e1da589e41976b53972edf635"}, - {file = "hiredis-3.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:bc51f594c2c0863ded6501642dc96701ca8bbea9ced4fa3af0a1aeda8aa634cb"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4663a319ab7d22c597b9421e5ea384fd583e044f2f1ca9a1b98d4fef8a0fea2f"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:8060fa256862b0c3de64a73ab45bc1ccf381caca464f2647af9075b200828948"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9445b7f117a9c8c8ccad97cb44daa55ddccff3cbc9079984eac56d982ba01f"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:732cf1c5cf1324f7bf3b6086976fe62a2ca98f0bf6316f31063c2c67be8797bc"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2102a94063d878c40df92f55199637a74f535e3a0b79ceba4a00538853a21be3"}, - {file = "hiredis-3.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d968dde69e3fe903bf9ef00667669dcf04a3e096e33aaf138775106ead138bc8"}, - {file = "hiredis-3.1.0.tar.gz", hash = "sha256:51d40ac3611091020d7dea6b05ed62cb152bff595fa4f931e7b6479d777acf7c"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, + {file = "hiredis-3.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f9ad63cd9065820a43fb1efb8ed5ae85bb78f03ef5eb53f6bde47914708f5718"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e7f9e5fdba08841d78d4e1450cae03a4dbed2eda8a4084673cafa5615ce24a"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1dce2508eca5d4e47ef38bc7c0724cb45abcdb0089f95a2ef49baf52882979a8"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:186428bf353e4819abae15aa2ad64c3f40499d596ede280fe328abb9e98e72ce"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74f2500d90a0494843aba7abcdc3e77f859c502e0892112d708c02e1dcae8f90"}, + {file = "hiredis-3.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32822a94d2fdd1da96c05b22fdeef6d145d8fdbd865ba2f273f45eb949e4a805"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ead809fb08dd4fdb5b4b6e2999c834e78c3b0c450a07c3ed88983964432d0c64"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b90fada20301c3a257e868dd6a4694febc089b2b6d893fa96a3fc6c1f9ab4340"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6d8bff53f526da3d9db86c8668011e4f7ca2958ee3a46c648edab6fe2cd1e709"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:043d929ae262d03e1db0f08616e14504a9119c1ff3de13d66f857d85cd45caff"}, + {file = "hiredis-3.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8d470fef39d02dbe5c541ec345cc4ffd7d2baec7d6e59c92bd9d9545dc221829"}, + {file = "hiredis-3.2.1-cp310-cp310-win32.whl", hash = "sha256:efa4c76c45cc8c42228c7989b279fa974580e053b5e6a4a834098b5324b9eafa"}, + {file = "hiredis-3.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:cbac5ec3a620b095c46ef3a8f1f06da9c86c1cdc411d44a5f538876c39a2b321"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:e4ae0be44cab5e74e6e4c4a93d04784629a45e781ff483b136cc9e1b9c23975c"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:24647e84c9f552934eb60b7f3d2116f8b64a7020361da9369e558935ca45914d"}, + {file = "hiredis-3.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6fb3e92d1172da8decc5f836bf8b528c0fc9b6d449f1353e79ceeb9dc1801132"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ba7a32e51e518b6b3e470142e52ed2674558e04d7d73d86eb19ebcb37d7d40"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4fc632be73174891d6bb71480247e57b2fd8f572059f0a1153e4d0339e919779"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f03e6839ff21379ad3c195e0700fc9c209e7f344946dea0f8a6d7b5137a2a141"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99983873e37c71bb71deb544670ff4f9d6920dab272aaf52365606d87a4d6c73"}, + {file = "hiredis-3.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffd982c419f48e3a57f592678c72474429465bb4bfc96472ec805f5d836523f0"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bc993f4aa4abc029347f309e722f122e05a3b8a0c279ae612849b5cc9dc69f2d"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dde790d420081f18b5949227649ccb3ed991459df33279419a25fcae7f97cd92"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b0c8cae7edbef860afcf3177b705aef43e10b5628f14d5baf0ec69668247d08d"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e8a90eaca7e1ce7f175584f07a2cdbbcab13f4863f9f355d7895c4d28805f65b"}, + {file = "hiredis-3.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:476031958fa44e245e803827e0787d49740daa4de708fe514370293ce519893a"}, + {file = "hiredis-3.2.1-cp311-cp311-win32.whl", hash = "sha256:eb3f5df2a9593b4b4b676dce3cea53b9c6969fc372875188589ddf2bafc7f624"}, + {file = "hiredis-3.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:1402e763d8a9fdfcc103bbf8b2913971c0a3f7b8a73deacbda3dfe5f3a9d1e0b"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:3742d8b17e73c198cabeab11da35f2e2a81999d406f52c6275234592256bf8e8"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9c2f3176fb617a79f6cccf22cb7d2715e590acb534af6a82b41f8196ad59375d"}, + {file = "hiredis-3.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8bd46189c7fa46174e02670dc44dfecb60f5bd4b67ed88cb050d8f1fd842f09"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f86ee4488c8575b58139cdfdddeae17f91e9a893ffee20260822add443592e2f"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3717832f4a557b2fe7060b9d4a7900e5de287a15595e398c3f04df69019ca69d"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5cb12c21fb9e2403d28c4e6a38120164973342d34d08120f2d7009b66785644"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:080fda1510bbd389af91f919c11a4f2aa4d92f0684afa4709236faa084a42cac"}, + {file = "hiredis-3.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1252e10a1f3273d1c6bf2021e461652c2e11b05b83e0915d6eb540ec7539afe2"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d9e320e99ab7d2a30dc91ff6f745ba38d39b23f43d345cdee9881329d7b511d6"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:641668f385f16550fdd6fdc109b0af6988b94ba2acc06770a5e06a16e88f320c"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1e1f44208c39d6c345ff451f82f21e9eeda6fe9af4ac65972cc3eeb58d41f7cb"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f882a0d6415fffe1ffcb09e6281d0ba8b1ece470e866612bbb24425bf76cf397"}, + {file = "hiredis-3.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b4e78719a0730ebffe335528531d154bc8867a246418f74ecd88adbc4d938c49"}, + {file = "hiredis-3.2.1-cp312-cp312-win32.whl", hash = "sha256:33c4604d9f79a13b84da79950a8255433fca7edaf292bbd3364fd620864ed7b2"}, + {file = "hiredis-3.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7b9749375bf9d171aab8813694f379f2cff0330d7424000f5e92890ad4932dc9"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:7cabf7f1f06be221e1cbed1f34f00891a7bdfad05b23e4d315007dd42148f3d4"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:db85cb86f8114c314d0ec6d8de25b060a2590b4713135240d568da4f7dea97ac"}, + {file = "hiredis-3.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c9a592a49b7b8497e4e62c3ff40700d0c7f1a42d145b71e3e23c385df573c964"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0079ef1e03930b364556b78548e67236ab3def4e07e674f6adfc52944aa972dd"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d6a290ed45d9c14f4c50b6bda07afb60f270c69b5cb626fd23a4c2fde9e3da1"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79dd5fe8c0892769f82949adeb021342ca46871af26e26945eb55d044fcdf0d0"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998a82281a159f4aebbfd4fb45cfe24eb111145206df2951d95bc75327983b58"}, + {file = "hiredis-3.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41fc3cd52368ffe7c8e489fb83af5e99f86008ed7f9d9ba33b35fec54f215c0a"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8d10df3575ce09b0fa54b8582f57039dcbdafde5de698923a33f601d2e2a246c"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ab010d04be33735ad8e643a40af0d68a21d70a57b1d0bff9b6a66b28cca9dbf"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ec3b5f9ea34f70aaba3e061cbe1fa3556fea401d41f5af321b13e326792f3017"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:158dfb505fff6bffd17f823a56effc0c2a7a8bc4fb659d79a52782f22eefc697"}, + {file = "hiredis-3.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d632cd0ddd7895081be76748e6fb9286f81d2a51c371b516541c6324f2fdac9"}, + {file = "hiredis-3.2.1-cp313-cp313-win32.whl", hash = "sha256:e9726d03e7df068bf755f6d1ecc61f7fc35c6b20363c7b1b96f39a14083df940"}, + {file = "hiredis-3.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:b5b1653ad7263a001f2e907e81a957d6087625f9700fa404f1a2268c0a4f9059"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:ef27728a8ceaa038ef4b6efc0e4473b7643b5c873c2fff5475e2c8b9c8d2e0d5"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:1039d8d2e1d2a1528ad9f9e289e8aa8eec9bf4b4759be4d453a2ab406a70a800"}, + {file = "hiredis-3.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83a8cd0eb6e535c93aad9c21e3e85bcb7dd26d3ff9b8ab095287be86e8af2f59"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6fc1e8f78bcdc7e25651b7d96d19b983b843b575904d96642f97ae157797ae4"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ddfa9a10fda3bea985a3b371a64553731141aaa0a20cbcc62a0e659f05e6c01"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e789ee008752b9be82a7bed82e36b62053c7cc06a0179a5a403ba5b2acba5bd8"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bf271877947a0f3eb9dc331688404a2e4cc246bca61bc5a1e2d62da9a1caad8"}, + {file = "hiredis-3.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9ad404fd0fdbdfe74e55ebb0592ab4169eecfe70ccf0db80eedc1d9943dd6d7"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:979572c602bdea0c3df255545c8c257f2163dd6c10d1f172268ffa7a6e1287d6"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f74e3d899be057fb00444ea5f7ae1d7389d393bddf0f3ed698997aa05563483b"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:a015666d5fdc3ca704f68db9850d0272ddcfb27e9f26a593013383f565ed2ad7"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:259a3389dfe3390e356c2796b6bc96a778695e9d7d40c82121096a6b8a2dd3c6"}, + {file = "hiredis-3.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:39f469891d29f0522712265de76018ab83a64b85ac4b4f67e1f692cbd42a03f9"}, + {file = "hiredis-3.2.1-cp38-cp38-win32.whl", hash = "sha256:73aa0508f26cd6cb4dfdbe189b28fb3162fd171532e526e90a802363b88027f8"}, + {file = "hiredis-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:2b910f12d7bcaf5ffc056087fc7b2d23e688f166462c31b73a0799d12891378d"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:523a241d9f268bc0c7306792f58f9c633185f939a19abc0356c55f078d3901c5"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:fec453a038c262e18d7de4919220b2916e0b17d1eadd12e7a800f09f78f84f39"}, + {file = "hiredis-3.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e75a49c5927453c316665cfa39f4274081d00ce69b137b393823eb90c66a8371"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd974cbe8b3ae8d3e7f60675e6da10383da69f029147c2c93d1a7e44b36d1290"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12d3b8fff9905e44f357417159d64138a32500dbd0d5cffaddbb2600d3ce33b1"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e21985804a40cb91e69e35ae321eb4e3610cd61a2cbc0328ab73a245f608fa1c"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e26e2b49a9569f44a2a2d743464ff0786b46fb1124ed33d2a1bd8b1c660c25b"}, + {file = "hiredis-3.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef1ebf9ee8e0b4a895b86a02a8b7e184b964c43758393532966ecb8a256f37c"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c936b690dd31d7af74f707fc9003c500315b4c9ad70fa564aff73d1283b3b37a"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4909666bcb73270bb806aa00d0eee9e81f7a1aca388aafb4ba7dfcf5d344d23a"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d74a2ad25bc91ca9639e4485099852e6263b360b2c3650fdd3cc47762c5db3fa"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e99910088df446ee64d64b160835f592fb4d36189fcc948dd204e903d91fffa3"}, + {file = "hiredis-3.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:54423bd7af93a773edc6f166341cfb0e5f35ef42ca07b93f568f672a6f445e40"}, + {file = "hiredis-3.2.1-cp39-cp39-win32.whl", hash = "sha256:4a5365cb6d7be82d3c6d523b369bc0bc1a64987e88ed6ecfabadda2aa1cf4fa4"}, + {file = "hiredis-3.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a2eb02b6aaf4f1425a408e892c0378ba6cb6b45b1412c30dd258df1322d88c0"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:73913d2fa379e722d17ba52f21ce12dd578140941a08efd73e73b6fab1dea4d8"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:15a3dff3eca31ecbf3d7d6d104cf1b318dc2b013bad3f4bdb2839cb9ea2e1584"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78258032c2f9fc6f39fee7b07882ce26de281e09178266ce535992572132d95"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578d6a881e64e46db065256355594e680202c3bacf3270be3140057171d2c23e"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b7f34b170093c077c972b8cc0ceb15d8ff88ad0079751a8ae9733e94d77e733"}, + {file = "hiredis-3.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:291a18b228fc90f6720d178de2fac46522082c96330b4cc2d3dd8cb2c1cb2815"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f53d2af5a7cd33a4b4d7ba632dce80c17823df6814ef5a8d328ed44c815a68e7"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:20bdf6dbdf77eb43b98bc53950f7711983042472199245d4c36448e6b4cb460f"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f43e5c50d76da15118c72b757216cf26c643d55bb1b3c86cad1ae49173971780"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e5bb5fe9834851d56c8543e52dcd2ac5275fb6772ebc97876e18c2e05a3300b"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e348438b6452e3d14dddb95d071fe8eaf6f264f641cba999c10bf6359cf1d2"}, + {file = "hiredis-3.2.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e305f6c63a2abcbde6ce28958de2bb4dd0fd34c6ab3bde5a4410befd5df8c6b2"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:33f24b1152f684b54d6b9d09135d849a6df64b6982675e8cf972f8adfa2de9aa"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:01dd8ea88bf8363751857ca2eb8f13faad0c7d57a6369663d4d1160f225ab449"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b16946533535cbb5cc7d4b6fc009d32d22b0f9ac58e8eb6f144637b64f9a61d"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9a03886cad1076e9f7e9e411c402826a8eac6f56ba426ee84b88e6515574b7b"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4f6340f1c378bce17c195d46288a796fcf213dd3e2a008c2c942b33ab58993"}, + {file = "hiredis-3.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d64ddf29016d34e7e3bc4b3d36ca9ac8a94f9b2c13ac4b9d8a486862d91b95c"}, + {file = "hiredis-3.2.1.tar.gz", hash = "sha256:5a5f64479bf04dd829fe7029fad0ea043eac4023abc6e946668cbbec3493a78d"}, ] [[package]] From 5898271365995d2bf5de8493d271292a5cca91ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 10:46:03 +0200 Subject: [PATCH 433/691] Bump pydantic from 2.11.4 to 2.11.7 (#18639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.11.4 to 2.11.7.
Release notes

Sourced from pydantic's releases.

v2.11.7 2025-06-14

What's Changed

Fixes

Full Changelog: https://github.com/pydantic/pydantic/compare/v2.11.6...v2.11.7

v2.11.6 2025-06-13

v2.11.6 (2025-06-13)

What's Changed

Fixes

Full Changelog: https://github.com/pydantic/pydantic/compare/v2.11.5...v2.11.6

v2.11.5 2025-05-22

What's Changed

Fixes

Full Changelog: https://github.com/pydantic/pydantic/compare/v2.11.4...v2.11.5

Changelog

Sourced from pydantic's changelog.

v2.11.7 (2025-06-14)

GitHub release

What's Changed

Fixes

  • Copy FieldInfo instance if necessary during FieldInfo build by @​Viicos in #11898

v2.11.6 (2025-06-13)

GitHub release

What's Changed

Fixes

v2.11.5 (2025-05-22)

GitHub release

What's Changed

Fixes

Commits
  • 5f033e4 Prepare release v2.11.7
  • c3368b8 Copy FieldInfo instance if necessary during FieldInfo build (#11980)
  • 3987b23 Prepare release v2.11.6
  • dc7a9d2 Always store the original field assignment on FieldInfo
  • c284c27 Rebuild dataclass fields before schema generation
  • 5e6d1dc Prepare release v2.11.5
  • 1b63218 Do not duplicate metadata on model rebuild (#11902)
  • 5aefad8 Do not delete mock validator/serializer in model_rebuild()
  • 8fbe658 Check if FieldInfo is complete after applying type variable map
  • 12b371a Update documentation about @dataclass_transform support
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pydantic&package-manager=pip&previous-version=2.11.4&new-version=2.11.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 22953577fe..0f908c2008 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1820,14 +1820,14 @@ files = [ [[package]] name = "pydantic" -version = "2.11.4" +version = "2.11.7" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "pydantic-2.11.4-py3-none-any.whl", hash = "sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb"}, - {file = "pydantic-2.11.4.tar.gz", hash = "sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d"}, + {file = "pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b"}, + {file = "pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db"}, ] [package.dependencies] From 8075c963d8a28fba11448d032dab3a2695539c1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 09:12:06 +0000 Subject: [PATCH 434/691] Bump mypy-zope from 1.0.11 to 1.0.12 (#18640) Bumps [mypy-zope](https://github.com/Shoobx/mypy-zope) from 1.0.11 to 1.0.12.
Changelog

Sourced from mypy-zope's changelog.

1.0.12 (2025-06-03)

  • Support mypy-1.16
Commits
  • fb4d6b3 Preparing release 1.0.12
  • 782aa68 Maintain changelog
  • dadddcf Merge pull request #129 from Shoobx/dependabot/pip/mypy-gte-1.0.0-and-lt-1.17.0
  • 1c2e229 Update mypy requirement from <1.16.0,>=1.0.0 to >=1.0.0,<1.17.0
  • 77eb8fc Back to development: 1.0.12
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=mypy-zope&package-manager=pip&previous-version=1.0.11&new-version=1.0.12)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0f908c2008..11b0f2de5e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1520,18 +1520,18 @@ files = [ [[package]] name = "mypy-zope" -version = "1.0.11" +version = "1.0.12" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" groups = ["dev"] files = [ - {file = "mypy_zope-1.0.11-py3-none-any.whl", hash = "sha256:4395d716b43ab89916edf6d0b5761655b4d4a43b2692fce806bbd733829977ee"}, - {file = "mypy_zope-1.0.11.tar.gz", hash = "sha256:1c95e49e9dcdf070a0858f067dac55e8e4e47519fdc15dfdab9b7eee273a0e01"}, + {file = "mypy_zope-1.0.12-py3-none-any.whl", hash = "sha256:f2ecf169f886fbc266e9339db0c2f3818528a7536b9bb4f5ece1d5854dc2f27c"}, + {file = "mypy_zope-1.0.12.tar.gz", hash = "sha256:d6f8f99eb5644885553b4ec7afc8d68f5daf412c9bf238ec3c36b65d97df6cbe"}, ] [package.dependencies] -mypy = ">=1.0.0,<1.16.0" +mypy = ">=1.0.0,<1.17.0" "zope.interface" = "*" "zope.schema" = "*" From 8f4caeeaf6aafd757368dbe41ccc9e5b0c7fa57d Mon Sep 17 00:00:00 2001 From: V02460 Date: Thu, 3 Jul 2025 11:12:49 +0200 Subject: [PATCH 435/691] Raise setuptools_rust version cap to 1.11.1 (#18576) Request to raise the defensive version cap for setuptools_rust from 1.10.2 to 1.11.1. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/18576.misc | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18576.misc diff --git a/changelog.d/18576.misc b/changelog.d/18576.misc new file mode 100644 index 0000000000..46cc78ec2d --- /dev/null +++ b/changelog.d/18576.misc @@ -0,0 +1 @@ +Raise setuptools_rust version cap to 1.11.1. diff --git a/pyproject.toml b/pyproject.toml index 9ad489b778..da9bbc6151 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,7 +374,7 @@ tomli = ">=1.2.3" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.10.2"] +requires = ["poetry-core>=1.1.0,<=2.1.3", "setuptools_rust>=1.3,<=1.11.1"] build-backend = "poetry.core.masonry.api" From 0d0f966b31144f0cacd8d35b60f6089cda3e2039 Mon Sep 17 00:00:00 2001 From: Alex Durham Date: Thu, 3 Jul 2025 11:59:45 +0200 Subject: [PATCH 436/691] Fix GET /_matrix/federation/v1/query/profile response (#18593) Don't send the fields `avatar_url` and `displayname` when they are not defined for the queried user. Before this change they would be sent and set to null in the JSON response object, which would violate the OpenAPI definitions (https://spec.matrix.org/v1.11/server-server-api/#get_matrixfederationv1queryprofile). Fixes: #18442 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/18593.bugfix | 1 + synapse/handlers/profile.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18593.bugfix diff --git a/changelog.d/18593.bugfix b/changelog.d/18593.bugfix new file mode 100644 index 0000000000..9079b9534a --- /dev/null +++ b/changelog.d/18593.bugfix @@ -0,0 +1 @@ +Fix `avatar_url` and `displayname` being sent on federation profile queries when they are not set. \ No newline at end of file diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index cdc388b4ab..76aa90e11b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -539,11 +539,17 @@ class ProfileHandler: response: JsonDict = {} try: if just_field is None or just_field == ProfileFields.DISPLAYNAME: - response["displayname"] = await self.store.get_profile_displayname(user) - + displayname = await self.store.get_profile_displayname(user) + # do not set the displayname field if it is None, + # since then we send a null in the JSON response + if displayname is not None: + response["displayname"] = displayname if just_field is None or just_field == ProfileFields.AVATAR_URL: - response["avatar_url"] = await self.store.get_profile_avatar_url(user) - + avatar_url = await self.store.get_profile_avatar_url(user) + # do not set the avatar_url field if it is None, + # since then we send a null in the JSON response + if avatar_url is not None: + response["avatar_url"] = avatar_url if self.hs.config.experimental.msc4133_enabled: if just_field is None: response.update(await self.store.get_profile_fields(user)) From 832690e7466988be109b2a298b0a4f064a8b3e0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 10:02:24 +0000 Subject: [PATCH 437/691] Bump types-pyyaml from 6.0.12.20241230 to 6.0.12.20250516 (#18643) Bumps [types-pyyaml](https://github.com/typeshed-internal/stub_uploader) from 6.0.12.20241230 to 6.0.12.20250516.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-pyyaml&package-manager=pip&previous-version=6.0.12.20241230&new-version=6.0.12.20250516)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 11b0f2de5e..73cafc414c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3068,14 +3068,14 @@ types-cffi = "*" [[package]] name = "types-pyyaml" -version = "6.0.12.20241230" +version = "6.0.12.20250516" description = "Typing stubs for PyYAML" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"}, - {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"}, + {file = "types_pyyaml-6.0.12.20250516-py3-none-any.whl", hash = "sha256:8478208feaeb53a34cb5d970c56a7cd76b72659442e733e268a94dc72b2d0530"}, + {file = "types_pyyaml-6.0.12.20250516.tar.gz", hash = "sha256:9f21a70216fc0fa1b216a8176db5f9e0af6eb35d2f2932acb87689d03a5bf6ba"}, ] [[package]] From be4c95baf11f16a6108da5b73f7140d36de6f55d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 3 Jul 2025 11:12:12 +0100 Subject: [PATCH 438/691] Replace PyICU with Rust `icu_segmenter` crate (#18553) Co-authored-by: anoa's Codex Agent Co-authored-by: Quentin Gliech --- changelog.d/18553.misc | 1 + docs/development/contributing_guide.md | 2 - docs/development/dependencies.md | 5 +- docs/setup/installation.md | 26 ++-------- docs/upgrade.md | 7 +++ docs/user_directory.md | 13 ++--- flake.nix | 1 - poetry.lock | 17 +------ pyproject.toml | 7 --- rust/Cargo.toml | 1 + rust/src/lib.rs | 2 + rust/src/segmenter.rs | 33 +++++++++++++ .../storage/databases/main/user_directory.py | 49 +++---------------- synapse/synapse_rust/segmenter.pyi | 3 ++ tests/storage/test_user_directory.py | 39 +-------------- 15 files changed, 70 insertions(+), 136 deletions(-) create mode 100644 changelog.d/18553.misc create mode 100644 rust/src/segmenter.rs create mode 100644 synapse/synapse_rust/segmenter.pyi diff --git a/changelog.d/18553.misc b/changelog.d/18553.misc new file mode 100644 index 0000000000..bb143aacfc --- /dev/null +++ b/changelog.d/18553.misc @@ -0,0 +1 @@ +Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index d6efab96cf..eb6f04e301 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -29,8 +29,6 @@ easiest way of installing the latest version is to use [rustup](https://rustup.r Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`. -Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`. - The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git). For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/). diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index fa5ff4dcf7..e381b3d155 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -164,10 +164,7 @@ $ poetry cache clear --all . # including the wheel artifacts which is not covered by the above command # (see https://github.com/python-poetry/poetry/issues/10304) # -# This is necessary in order to rebuild or fetch new wheels. For example, if you update -# the `icu` library in on your system, you will need to rebuild the PyICU Python package -# in order to incorporate the correct dynamically linked library locations otherwise you -# will run into errors like: `ImportError: libicui18n.so.75: cannot open shared object file: No such file or directory` +# This is necessary in order to rebuild or fetch new wheels. $ rm -rf $(poetry config cache-dir) ``` diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 0853496ab7..0840f532b0 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -286,7 +286,7 @@ Installing prerequisites on Ubuntu or Debian: ```sh sudo apt install build-essential python3-dev libffi-dev \ python3-pip python3-setuptools sqlite3 \ - libssl-dev virtualenv libjpeg-dev libxslt1-dev libicu-dev + libssl-dev virtualenv libjpeg-dev libxslt1-dev ``` ##### ArchLinux @@ -295,7 +295,7 @@ Installing prerequisites on ArchLinux: ```sh sudo pacman -S base-devel python python-pip \ - python-setuptools python-virtualenv sqlite3 icu + python-setuptools python-virtualenv sqlite3 ``` ##### CentOS/Fedora @@ -305,8 +305,7 @@ Installing prerequisites on CentOS or Fedora Linux: ```sh sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ libwebp-devel libxml2-devel libxslt-devel libpq-devel \ - python3-virtualenv libffi-devel openssl-devel python3-devel \ - libicu-devel + python3-virtualenv libffi-devel openssl-devel python3-devel sudo dnf group install "Development Tools" ``` @@ -333,7 +332,7 @@ dnf install python3.12 python3.12-devel ``` Finally, install common prerequisites ```bash -dnf install libicu libicu-devel libpq5 libpq5-devel lz4 pkgconf +dnf install libpq5 libpq5-devel lz4 pkgconf dnf group install "Development Tools" ``` ###### Using venv module instead of virtualenv command @@ -365,20 +364,6 @@ xcode-select --install Some extra dependencies may be needed. You can use Homebrew (https://brew.sh) for them. -You may need to install icu, and make the icu binaries and libraries accessible. -Please follow [the official instructions of PyICU](https://pypi.org/project/PyICU/) to do so. - -If you're struggling to get icu discovered, and see: -``` - RuntimeError: - Please install pkg-config on your system or set the ICU_VERSION environment - variable to the version of ICU you have installed. -``` -despite it being installed and having your `PATH` updated, you can omit this dependency by -not specifying `--extras all` to `poetry`. If using postgres, you can install Synapse via -`poetry install --extras saml2 --extras oidc --extras postgres --extras opentracing --extras redis --extras sentry`. -ICU is not a hard dependency on getting a working installation. - On ARM-based Macs you may also need to install libjpeg and libpq: ```sh brew install jpeg libpq @@ -400,8 +385,7 @@ Installing prerequisites on openSUSE: ```sh sudo zypper in -t pattern devel_basis sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \ - python-devel libffi-devel libopenssl-devel libjpeg62-devel \ - libicu-devel + python-devel libffi-devel libopenssl-devel libjpeg62-devel ``` ##### OpenBSD diff --git a/docs/upgrade.md b/docs/upgrade.md index d508e2231e..ca9ca121f2 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,13 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.134.0 + +## ICU bundled with Synapse + +Synapse now uses the Rust `icu` library for improved user search. Installing the +native ICU library on your system is no longer required. + # Upgrading to v1.130.0 ## Documented endpoint which can be delegated to a federation worker diff --git a/docs/user_directory.md b/docs/user_directory.md index be8664a016..75d32af44f 100644 --- a/docs/user_directory.md +++ b/docs/user_directory.md @@ -77,14 +77,11 @@ The user provided search term is lowercased and normalized using [NFKC](https:// this treats the string as case-insensitive, canonicalizes different forms of the same text, and maps some "roughly equivalent" characters together. -The search term is then split into words: - -* If [ICU](https://en.wikipedia.org/wiki/International_Components_for_Unicode) is - available, then the system's [default locale](https://unicode-org.github.io/icu/userguide/locale/#default-locales) - will be used to break the search term into words. (See the - [installation instructions](setup/installation.md) for how to install ICU.) -* If unavailable, then runs of ASCII characters, numbers, underscores, and hyphens - are considered words. +The search term is then split into segments using the [`icu_segmenter` +Rust crate](https://crates.io/crates/icu_segmenter). This crate ships with its +own dictionary and Long Short Term-Memory (LSTM) machine learning models +per-language to segment words. Read more [in the crate's +documentation](https://docs.rs/icu/latest/icu/segmenter/struct.WordSegmenter.html#method.new_auto). The queries for PostgreSQL and SQLite are detailed below, but their overall goal is to find matching users, preferring users who are "real" (e.g. not bots, diff --git a/flake.nix b/flake.nix index 749c10da1d..4ff6518aed 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,6 @@ gnumake # Native dependencies for running Synapse. - icu libffi libjpeg libpqxx diff --git a/poetry.lock b/poetry.lock index 73cafc414c..329c373d12 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1987,18 +1987,6 @@ files = [ [package.extras] plugins = ["importlib-metadata ; python_version < \"3.8\""] -[[package]] -name = "pyicu" -version = "2.14" -description = "Python extension wrapping the ICU C++ API" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"all\" or extra == \"user-search\"" -files = [ - {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"}, -] - [[package]] name = "pyjwt" version = "2.6.0" @@ -3393,7 +3381,7 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pyicu", "pysaml2", "sentry-sdk", "txredisapi"] +all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "sentry-sdk", "txredisapi"] cache-memory = ["Pympler"] jwt = ["authlib"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -3406,9 +3394,8 @@ sentry = ["sentry-sdk"] systemd = ["systemd-python"] test = ["idna", "parameterized"] url-preview = ["lxml"] -user-search = ["pyicu"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "9824e42dfc0e128129ee0c8641f7fe639bf47574cdd3f052dd995941abc6e44b" +content-hash = "457f188ae22af9663b2ed21f2586720ce5014edc7c34a697787f16aad733ea41" diff --git a/pyproject.toml b/pyproject.toml index da9bbc6151..ecd601f03b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,7 +254,6 @@ hiredis = { version = "*", optional = true } Pympler = { version = "*", optional = true } parameterized = { version = ">=0.7.4", optional = true } idna = { version = ">=2.5", optional = true } -pyicu = { version = ">=2.10.2", optional = true } [tool.poetry.extras] # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified @@ -277,10 +276,6 @@ redis = ["txredisapi", "hiredis"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler"] test = ["parameterized", "idna"] -# Allows for better search for international characters in the user directory. This -# requires libicu's development headers installed on the system (e.g. libicu-dev on -# Debian-based distributions). -user-search = ["pyicu"] # The duplication here is awful. I hate hate hate hate hate it. However, for now I want # to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations: @@ -312,8 +307,6 @@ all = [ "txredisapi", "hiredis", # cache-memory "pympler", - # improved user search - "pyicu", # omitted: # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3a47f7ddf7..dab32c8952 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -43,6 +43,7 @@ sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } serde_json = "1.0.85" ulid = "1.1.2" +icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ "http2", "stream", diff --git a/rust/src/lib.rs b/rust/src/lib.rs index e33a8cc44c..6522148fa1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -13,6 +13,7 @@ pub mod identifier; pub mod matrix_const; pub mod push; pub mod rendezvous; +pub mod segmenter; lazy_static! { static ref LOGGING_HANDLE: ResetHandle = pyo3_log::init(); @@ -53,6 +54,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { events::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; + segmenter::register_module(py, m)?; Ok(()) } diff --git a/rust/src/segmenter.rs b/rust/src/segmenter.rs new file mode 100644 index 0000000000..135b3c1779 --- /dev/null +++ b/rust/src/segmenter.rs @@ -0,0 +1,33 @@ +use icu_segmenter::options::WordBreakInvariantOptions; +use icu_segmenter::WordSegmenter; +use pyo3::prelude::*; + +#[pyfunction] +pub fn parse_words(text: &str) -> PyResult> { + let segmenter = WordSegmenter::new_auto(WordBreakInvariantOptions::default()); + let mut parts = Vec::new(); + let mut last = 0usize; + + // `segment_str` gives us word boundaries as a vector of indexes. Use that + // to build a vector of words, and return. + for boundary in segmenter.segment_str(text) { + if boundary > last { + parts.push(text[last..boundary].to_string()); + } + last = boundary; + } + Ok(parts) +} + +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "segmenter")?; + child_module.add_function(wrap_pyfunction!(parse_words, m)?)?; + + m.add_submodule(&child_module)?; + + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.segmenter", child_module)?; + + Ok(()) +} diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 09671abcad..9deb9ab73c 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -37,16 +37,8 @@ from typing import ( import attr -try: - # Figure out if ICU support is available for searching users. - import icu - - USE_ICU = True -except ModuleNotFoundError: - # except ModuleNotFoundError: - USE_ICU = False - from synapse.api.errors import StoreError +from synapse.synapse_rust import segmenter as icu from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: @@ -1226,7 +1218,7 @@ def _filter_text_for_index(text: str) -> str: def _parse_query_sqlite(search_term: str) -> str: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. @@ -1242,7 +1234,7 @@ def _parse_query_sqlite(search_term: str) -> str: def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. + that can be passed to the database. We use this so that we can add prefix matching, which isn't something that is supported by default. """ @@ -1272,12 +1264,7 @@ def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]: def _parse_words(search_term: str) -> List[str]: - """Split the provided search string into a list of its words. - - If support for ICU (International Components for Unicode) is available, use it. - Otherwise, fall back to using a regex to detect word boundaries. This latter - solution works well enough for most latin-based languages, but doesn't work as well - with other languages. + """Split the provided search string into a list of its words using ICU. Args: search_term: The search string. @@ -1285,18 +1272,7 @@ def _parse_words(search_term: str) -> List[str]: Returns: A list of the words in the search string. """ - if USE_ICU: - return _parse_words_with_icu(search_term) - - return _parse_words_with_regex(search_term) - - -def _parse_words_with_regex(search_term: str) -> List[str]: - """ - Break down search term into words, when we don't have ICU available. - See: `_parse_words` - """ - return re.findall(r"([\w-]+)", search_term, re.UNICODE) + return _parse_words_with_icu(search_term) def _parse_words_with_icu(search_term: str) -> List[str]: @@ -1310,22 +1286,13 @@ def _parse_words_with_icu(search_term: str) -> List[str]: A list of the words in the search string. """ results = [] - breaker = icu.BreakIterator.createWordInstance(icu.Locale.getDefault()) - breaker.setText(search_term) - i = 0 - while True: - j = breaker.nextBoundary() - if j < 0: - break - + for part in icu.parse_words(search_term): # We want to make sure that we split on `@` and `:` specifically, as # they occur in user IDs. - for result in re.split(r"[@:]+", search_term[i:j]): + for result in re.split(r"[@:]+", part): results.append(result.strip()) - i = j - - # libicu will break up words that have punctuation in them, but to handle + # icu will break up words that have punctuation in them, but to handle # cases where user IDs have '-', '.' and '_' in them we want to *not* break # those into words and instead allow the DB to tokenise them how it wants. # diff --git a/synapse/synapse_rust/segmenter.pyi b/synapse/synapse_rust/segmenter.pyi new file mode 100644 index 0000000000..5f36765947 --- /dev/null +++ b/synapse/synapse_rust/segmenter.pyi @@ -0,0 +1,3 @@ +from typing import List + +def parse_words(text: str) -> List[str]: ... diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index c26932069f..f97ca12d84 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -32,10 +32,8 @@ from synapse.rest.client import login, register, room from synapse.server import HomeServer from synapse.storage import DataStore from synapse.storage.background_updates import _BackgroundUpdateHandler -from synapse.storage.databases.main import user_directory from synapse.storage.databases.main.user_directory import ( _parse_words_with_icu, - _parse_words_with_regex, ) from synapse.storage.roommember import ProfileInfo from synapse.util import Clock @@ -44,12 +42,6 @@ from tests.server import ThreadedMemoryReactorClock from tests.test_utils.event_injection import inject_member_event from tests.unittest import HomeserverTestCase, override_config -try: - import icu -except ImportError: - icu = None # type: ignore - - ALICE = "@alice:a" BOB = "@bob:b" BOBBY = "@bobby:a" @@ -438,8 +430,6 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): class UserDirectoryStoreTestCase(HomeserverTestCase): - use_icu = False - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main @@ -451,12 +441,6 @@ class UserDirectoryStoreTestCase(HomeserverTestCase): self.get_success(self.store.update_profile_in_user_dir(BELA, "Bela", None)) self.get_success(self.store.add_users_in_public_rooms("!room:id", (ALICE, BOB))) - self._restore_use_icu = user_directory.USE_ICU - user_directory.USE_ICU = self.use_icu - - def tearDown(self) -> None: - user_directory.USE_ICU = self._restore_use_icu - def test_search_user_dir(self) -> None: # normally when alice searches the directory she should just find # bob because bobby doesn't share a room with her. @@ -648,24 +632,14 @@ class UserDirectoryStoreTestCase(HomeserverTestCase): test_search_user_dir_accent_insensitivity.skip = "not supported yet" # type: ignore -class UserDirectoryStoreTestCaseWithIcu(UserDirectoryStoreTestCase): - use_icu = True - - if not icu: - skip = "Requires PyICU" - - class UserDirectoryICUTestCase(HomeserverTestCase): - if not icu: - skip = "Requires PyICU" - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main self.user_dir_helper = GetUserDirectoryTables(self.store) def test_icu_word_boundary(self) -> None: - """Tests that we correctly detect word boundaries when ICU (International - Components for Unicode) support is available. + """Tests that we correctly detect word boundaries with ICU + (International Components for Unicode). """ display_name = "Gáo" @@ -714,12 +688,3 @@ class UserDirectoryICUTestCase(HomeserverTestCase): self.assertEqual(_parse_words_with_icu("user-1"), ["user-1"]) self.assertEqual(_parse_words_with_icu("user-ab"), ["user-ab"]) self.assertEqual(_parse_words_with_icu("user.--1"), ["user", "-1"]) - - def test_regex_word_boundary_punctuation(self) -> None: - """ - Tests the behaviour of punctuation with the non-ICU tokeniser - """ - self.assertEqual( - _parse_words_with_regex("lazy'fox jumped:over the.dog"), - ["lazy", "fox", "jumped", "over", "the", "dog"], - ) From a06d5cef322aa74fd36b1e7da9d8fc3cc98b884f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 3 Jul 2025 12:28:11 +0100 Subject: [PATCH 439/691] Update Cargo.lock (#18646) --- Cargo.lock | 254 ++++++++++++++++++++++++++++++++++++----- changelog.d/18646.misc | 1 + 2 files changed, 227 insertions(+), 28 deletions(-) create mode 100644 changelog.d/18646.misc diff --git a/Cargo.lock b/Cargo.lock index 0ac5280c17..24994225e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -144,6 +150,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -365,7 +380,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", "http", @@ -485,7 +500,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -510,11 +525,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ "displaydoc", - "yoke", + "yoke 0.7.5", "zerofrom", - "zerovec", + "zerovec 0.10.4", ] +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke 0.8.0", + "zerofrom", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ae5921528335e91da1b6c695dbf1ec37df5ac13faa3f91e5640be93aa2fbefd" +dependencies = [ + "displaydoc", + "icu_collections 2.0.0", + "icu_locale_core", + "icu_locale_data", + "icu_provider 2.0.0", + "potential_utf", + "tinystr 0.8.1", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap 0.8.0", + "tinystr 0.8.1", + "writeable 0.6.1", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_locale_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdef0c124749d06a743c69e938350816554eb63ac979166590e2b4ee4252765" + [[package]] name = "icu_locid" version = "1.5.0" @@ -522,10 +585,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "litemap 0.7.5", + "tinystr 0.7.6", + "writeable 0.5.5", + "zerovec 0.10.4", ] [[package]] @@ -537,9 +600,9 @@ dependencies = [ "displaydoc", "icu_locid", "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -555,15 +618,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_normalizer_data", "icu_properties", - "icu_provider", + "icu_provider 1.5.0", "smallvec", "utf16_iter", "utf8_iter", "write16", - "zerovec", + "zerovec 0.10.4", ] [[package]] @@ -579,12 +642,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_locid_transform", "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -603,11 +666,28 @@ dependencies = [ "icu_locid", "icu_provider_macros", "stable_deref_trait", - "tinystr", - "writeable", - "yoke", + "tinystr 0.7.6", + "writeable 0.5.5", + "yoke 0.7.5", "zerofrom", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr 0.8.1", + "writeable 0.6.1", + "yoke 0.8.0", + "zerofrom", + "zerotrie", + "zerovec 0.11.2", ] [[package]] @@ -621,6 +701,30 @@ dependencies = [ "syn", ] +[[package]] +name = "icu_segmenter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e185fc13b6401c138cf40db12b863b35f5edf31b88192a545857b41aeaf7d3d3" +dependencies = [ + "core_maths", + "displaydoc", + "icu_collections 2.0.0", + "icu_locale", + "icu_locale_core", + "icu_provider 2.0.0", + "icu_segmenter_data", + "potential_utf", + "utf8_iter", + "zerovec 0.11.2", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5360a2fbe97f617c4f8b944356dedb36d423f7da7f13c070995cf89e59f01220" + [[package]] name = "idna" version = "1.0.3" @@ -713,12 +817,24 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "litemap" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "log" version = "0.4.27" @@ -811,6 +927,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "serde", + "zerovec 0.11.2", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1064,7 +1190,7 @@ version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -1338,7 +1464,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "blake2", "bytes", "futures", @@ -1346,6 +1472,7 @@ dependencies = [ "hex", "http", "http-body-util", + "icu_segmenter", "lazy_static", "log", "mime", @@ -1414,7 +1541,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec 0.11.2", ] [[package]] @@ -1836,6 +1973,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "yoke" version = "0.7.5" @@ -1844,7 +1987,19 @@ checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive", + "yoke-derive 0.7.5", + "zerofrom", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive 0.8.0", "zerofrom", ] @@ -1860,6 +2015,18 @@ dependencies = [ "synstructure", ] +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.17" @@ -1907,15 +2074,35 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", +] + [[package]] name = "zerovec" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "yoke", + "yoke 0.7.5", "zerofrom", - "zerovec-derive", + "zerovec-derive 0.10.3", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke 0.8.0", + "zerofrom", + "zerovec-derive 0.11.1", ] [[package]] @@ -1928,3 +2115,14 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/changelog.d/18646.misc b/changelog.d/18646.misc new file mode 100644 index 0000000000..bb143aacfc --- /dev/null +++ b/changelog.d/18646.misc @@ -0,0 +1 @@ +Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file From 6c4f8521d4dfbace6ad6e7abbfe70aeded372886 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 11:58:44 +0000 Subject: [PATCH 440/691] Bump ruff from 0.11.11 to 0.12.1 (#18645) Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.11 to 0.12.1.
Release notes

Sourced from ruff's releases.

0.12.1

Release Notes

Preview features

  • [flake8-errmsg] Extend EM101 to support byte strings (#18867)
  • [flake8-use-pathlib] Add autofix for PTH202 (#18763)
  • [pygrep-hooks] Add AsyncMock methods to invalid-mock-access (PGH005) (#18547)
  • [pylint] Ignore __init__.py files in (PLC0414) (#18400)
  • [ruff] Trigger RUF037 for empty string and byte strings (#18862)
  • [formatter] Fix missing blank lines before decorated classes in .pyi files (#18888)

Bug fixes

  • Avoid generating diagnostics with per-file ignores (#18801)
  • Handle parenthesized arguments in remove_argument (#18805)
  • [flake8-logging] Avoid false positive for exc_info=True outside logger.exception (LOG014) (#18737)
  • [flake8-pytest-style] Enforce pytest import for decorators (#18779)
  • [flake8-pytest-style] Mark autofix for PT001 and PT023 as unsafe if there's comments in the decorator (#18792)
  • [flake8-pytest-style] PT001/PT023 fix makes syntax error on parenthesized decorator (#18782)
  • [flake8-raise] Make fix unsafe if it deletes comments (RSE102) (#18788)
  • [flake8-simplify] Fix SIM911 autofix creating a syntax error (#18793)
  • [flake8-simplify] Fix false negatives for shadowed bindings (SIM910, SIM911) (#18794)
  • [flake8-simplify] Preserve original behavior for except () and bare except (SIM105) (#18213)
  • [flake8-pyi] Fix PYI041's fix causing TypeError with None | None | ... (#18637)
  • [perflint] Fix PERF101 autofix creating a syntax error and mark autofix as unsafe if there are comments in the list call expr (#18803)
  • [perflint] Fix false negative in PERF401 (#18866)
  • [pylint] Avoid flattening nested min/max when outer call has single argument (PLW3301) (#16885)
  • [pylint] Fix PLC2801 autofix creating a syntax error (#18857)
  • [pylint] Mark PLE0241 autofix as unsafe if there's comments in the base classes (#18832)
  • [pylint] Suppress PLE2510/PLE2512/PLE2513/PLE2514/PLE2515 autofix if the text contains an odd number of backslashes (#18856)
  • [refurb] Detect more exotic float literals in FURB164 (#18925)
  • [refurb] Fix FURB163 autofix creating a syntax error for yield expressions (#18756)
  • [refurb] Mark FURB129 autofix as unsafe if there's comments in the readlines call (#18858)
  • [ruff] Fix false positives and negatives in RUF010 (#18690)
  • Fix casing of analyze.direction variant names (#18892)

Rule changes

  • Fix f-string interpolation escaping in generated fixes (#18882)
  • [flake8-return] Mark RET501 fix unsafe if comments are inside (#18780)
  • [flake8-async] Fix detection for large integer sleep durations in ASYNC116 rule (#18767)
  • [flake8-async] Mark autofix for ASYNC115 as unsafe if the call expression contains comments (#18753)
  • [flake8-bugbear] Mark autofix for B004 as unsafe if the hasattr call expr contains comments (#18755)
  • [flake8-comprehension] Mark autofix for C420 as unsafe if there's comments inside the dict comprehension (#18768)
  • [flake8-comprehensions] Handle template strings for comprehension fixes (#18710)
  • [flake8-future-annotations] Add autofix (FA100) (#18903)
  • [pyflakes] Mark F504/F522/F523 autofix as unsafe if there's a call with side effect (#18839)
  • [pylint] Allow fix with comments and document performance implications (PLW3301) (#18936)
  • [pylint] Detect more exotic NaN literals in PLW0177 (#18630)

... (truncated)

Changelog

Sourced from ruff's changelog.

0.12.1

Preview features

  • [flake8-errmsg] Extend EM101 to support byte strings (#18867)
  • [flake8-use-pathlib] Add autofix for PTH202 (#18763)
  • [pygrep-hooks] Add AsyncMock methods to invalid-mock-access (PGH005) (#18547)
  • [pylint] Ignore __init__.py files in (PLC0414) (#18400)
  • [ruff] Trigger RUF037 for empty string and byte strings (#18862)
  • [formatter] Fix missing blank lines before decorated classes in .pyi files (#18888)

Bug fixes

  • Avoid generating diagnostics with per-file ignores (#18801)
  • Handle parenthesized arguments in remove_argument (#18805)
  • [flake8-logging] Avoid false positive for exc_info=True outside logger.exception (LOG014) (#18737)
  • [flake8-pytest-style] Enforce pytest import for decorators (#18779)
  • [flake8-pytest-style] Mark autofix for PT001 and PT023 as unsafe if there's comments in the decorator (#18792)
  • [flake8-pytest-style] PT001/PT023 fix makes syntax error on parenthesized decorator (#18782)
  • [flake8-raise] Make fix unsafe if it deletes comments (RSE102) (#18788)
  • [flake8-simplify] Fix SIM911 autofix creating a syntax error (#18793)
  • [flake8-simplify] Fix false negatives for shadowed bindings (SIM910, SIM911) (#18794)
  • [flake8-simplify] Preserve original behavior for except () and bare except (SIM105) (#18213)
  • [flake8-pyi] Fix PYI041's fix causing TypeError with None | None | ... (#18637)
  • [perflint] Fix PERF101 autofix creating a syntax error and mark autofix as unsafe if there are comments in the list call expr (#18803)
  • [perflint] Fix false negative in PERF401 (#18866)
  • [pylint] Avoid flattening nested min/max when outer call has single argument (PLW3301) (#16885)
  • [pylint] Fix PLC2801 autofix creating a syntax error (#18857)
  • [pylint] Mark PLE0241 autofix as unsafe if there's comments in the base classes (#18832)
  • [pylint] Suppress PLE2510/PLE2512/PLE2513/PLE2514/PLE2515 autofix if the text contains an odd number of backslashes (#18856)
  • [refurb] Detect more exotic float literals in FURB164 (#18925)
  • [refurb] Fix FURB163 autofix creating a syntax error for yield expressions (#18756)
  • [refurb] Mark FURB129 autofix as unsafe if there's comments in the readlines call (#18858)
  • [ruff] Fix false positives and negatives in RUF010 (#18690)
  • Fix casing of analyze.direction variant names (#18892)

Rule changes

  • Fix f-string interpolation escaping in generated fixes (#18882)
  • [flake8-return] Mark RET501 fix unsafe if comments are inside (#18780)
  • [flake8-async] Fix detection for large integer sleep durations in ASYNC116 rule (#18767)
  • [flake8-async] Mark autofix for ASYNC115 as unsafe if the call expression contains comments (#18753)
  • [flake8-bugbear] Mark autofix for B004 as unsafe if the hasattr call expr contains comments (#18755)
  • [flake8-comprehension] Mark autofix for C420 as unsafe if there's comments inside the dict comprehension (#18768)
  • [flake8-comprehensions] Handle template strings for comprehension fixes (#18710)
  • [flake8-future-annotations] Add autofix (FA100) (#18903)
  • [pyflakes] Mark F504/F522/F523 autofix as unsafe if there's a call with side effect (#18839)
  • [pylint] Allow fix with comments and document performance implications (PLW3301) (#18936)
  • [pylint] Detect more exotic NaN literals in PLW0177 (#18630)
  • [pylint] Fix PLC1802 autofix creating a syntax error and mark autofix as unsafe if there's comments in the len call (#18836)

... (truncated)

Commits
  • 32c5418 Bump 0.12.1 (#18969)
  • b85c219 [FastAPI] Add fix safety section to FAST002 (#18940)
  • b1d1cf1 [ty] Add regression test for leading tab mis-alignment in diagnostic renderin...
  • 1dcdf7f [ty] Resolve python environment in Options::to_program_settings (#18960)
  • d006976 [ruff] Fix false positives and negatives in RUF010 (#18690)
  • 76619b9 [ty] Fix rendering of long lines that are indented with tabs
  • 6e25cfb [ty] Add regression test for diagnostic rendering panic
  • 7638729 [ty] Move venv and conda env discovery to SearchPath::from_settings (#18938)
  • d04e63a [ty] Add regression-benchmark for attribute-assignment hang (#18957)
  • 86fd9b6 [ty] Format conflicting types as an enumeration (#18956)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ruff&package-manager=pip&previous-version=0.11.11&new-version=0.12.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index 329c373d12..96ebc7cf52 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2442,30 +2442,30 @@ files = [ [[package]] name = "ruff" -version = "0.11.11" +version = "0.12.1" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.11.11-py3-none-linux_armv6l.whl", hash = "sha256:9924e5ae54125ed8958a4f7de320dab7380f6e9fa3195e3dc3b137c6842a0092"}, - {file = "ruff-0.11.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c8a93276393d91e952f790148eb226658dd275cddfde96c6ca304873f11d2ae4"}, - {file = "ruff-0.11.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d6e333dbe2e6ae84cdedefa943dfd6434753ad321764fd937eef9d6b62022bcd"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7885d9a5e4c77b24e8c88aba8c80be9255fa22ab326019dac2356cff42089fc6"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b5ab797fcc09121ed82e9b12b6f27e34859e4227080a42d090881be888755d4"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e231ff3132c1119ece836487a02785f099a43992b95c2f62847d29bace3c75ac"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a97c9babe1d4081037a90289986925726b802d180cca784ac8da2bbbc335f709"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8c4ddcbe8a19f59f57fd814b8b117d4fcea9bee7c0492e6cf5fdc22cfa563c8"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6224076c344a7694c6fbbb70d4f2a7b730f6d47d2a9dc1e7f9d9bb583faf390b"}, - {file = "ruff-0.11.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:882821fcdf7ae8db7a951df1903d9cb032bbe838852e5fc3c2b6c3ab54e39875"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:dcec2d50756463d9df075a26a85a6affbc1b0148873da3997286caf1ce03cae1"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99c28505ecbaeb6594701a74e395b187ee083ee26478c1a795d35084d53ebd81"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9263f9e5aa4ff1dec765e99810f1cc53f0c868c5329b69f13845f699fe74f639"}, - {file = "ruff-0.11.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:64ac6f885e3ecb2fdbb71de2701d4e34526651f1e8503af8fb30d4915a3fe345"}, - {file = "ruff-0.11.11-py3-none-win32.whl", hash = "sha256:1adcb9a18802268aaa891ffb67b1c94cd70578f126637118e8099b8e4adcf112"}, - {file = "ruff-0.11.11-py3-none-win_amd64.whl", hash = "sha256:748b4bb245f11e91a04a4ff0f96e386711df0a30412b9fe0c74d5bdc0e4a531f"}, - {file = "ruff-0.11.11-py3-none-win_arm64.whl", hash = "sha256:6c51f136c0364ab1b774767aa8b86331bd8e9d414e2d107db7a2189f35ea1f7b"}, - {file = "ruff-0.11.11.tar.gz", hash = "sha256:7774173cc7c1980e6bf67569ebb7085989a78a103922fb83ef3dfe230cd0687d"}, + {file = "ruff-0.12.1-py3-none-linux_armv6l.whl", hash = "sha256:6013a46d865111e2edb71ad692fbb8262e6c172587a57c0669332a449384a36b"}, + {file = "ruff-0.12.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b3f75a19e03a4b0757d1412edb7f27cffb0c700365e9d6b60bc1b68d35bc89e0"}, + {file = "ruff-0.12.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9a256522893cb7e92bb1e1153283927f842dea2e48619c803243dccc8437b8be"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:069052605fe74c765a5b4272eb89880e0ff7a31e6c0dbf8767203c1fbd31c7ff"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a684f125a4fec2d5a6501a466be3841113ba6847827be4573fddf8308b83477d"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdecdef753bf1e95797593007569d8e1697a54fca843d78f6862f7dc279e23bd"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:70d52a058c0e7b88b602f575d23596e89bd7d8196437a4148381a3f73fcd5010"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84d0a69d1e8d716dfeab22d8d5e7c786b73f2106429a933cee51d7b09f861d4e"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cc32e863adcf9e71690248607ccdf25252eeeab5193768e6873b901fd441fed"}, + {file = "ruff-0.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fd49a4619f90d5afc65cf42e07b6ae98bb454fd5029d03b306bd9e2273d44cc"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ed5af6aaaea20710e77698e2055b9ff9b3494891e1b24d26c07055459bb717e9"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:801d626de15e6bf988fbe7ce59b303a914ff9c616d5866f8c79eb5012720ae13"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2be9d32a147f98a1972c1e4df9a6956d612ca5f5578536814372113d09a27a6c"}, + {file = "ruff-0.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:49b7ce354eed2a322fbaea80168c902de9504e6e174fd501e9447cad0232f9e6"}, + {file = "ruff-0.12.1-py3-none-win32.whl", hash = "sha256:d973fa626d4c8267848755bd0414211a456e99e125dcab147f24daa9e991a245"}, + {file = "ruff-0.12.1-py3-none-win_amd64.whl", hash = "sha256:9e1123b1c033f77bd2590e4c1fe7e8ea72ef990a85d2484351d408224d603013"}, + {file = "ruff-0.12.1-py3-none-win_arm64.whl", hash = "sha256:78ad09a022c64c13cc6077707f036bab0fac8cd7088772dcd1e5be21c5002efc"}, + {file = "ruff-0.12.1.tar.gz", hash = "sha256:806bbc17f1104fd57451a98a58df35388ee3ab422e029e8f5cf30aa4af2c138c"}, ] [[package]] @@ -3398,4 +3398,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "457f188ae22af9663b2ed21f2586720ce5014edc7c34a697787f16aad733ea41" +content-hash = "4f24cf959d33ebacbcbd791990d42e5fea2ab74bfad32cf2115d2aadd1c46d21" diff --git a/pyproject.toml b/pyproject.toml index ecd601f03b..be5bff31b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -317,7 +317,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.11.11" +ruff = "0.12.1" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From b7d54b7f3c8e4b17c55f01c8643980add7f1e060 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 12:15:22 +0000 Subject: [PATCH 441/691] Bump types-setuptools from 75.2.0.20241019 to 80.9.0.20250529 (#18644) Bumps [types-setuptools](https://github.com/typeshed-internal/stub_uploader) from 75.2.0.20241019 to 80.9.0.20250529.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-setuptools&package-manager=pip&previous-version=75.2.0.20241019&new-version=80.9.0.20250529)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 96ebc7cf52..43be4098d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3083,14 +3083,14 @@ urllib3 = ">=2" [[package]] name = "types-setuptools" -version = "75.2.0.20241019" +version = "80.9.0.20250529" description = "Typing stubs for setuptools" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types-setuptools-75.2.0.20241019.tar.gz", hash = "sha256:86ea31b5f6df2c6b8f2dc8ae3f72b213607f62549b6fa2ed5866e5299f968694"}, - {file = "types_setuptools-75.2.0.20241019-py3-none-any.whl", hash = "sha256:2e48ff3acd4919471e80d5e3f049cce5c177e108d5d36d2d4cee3fa4d4104258"}, + {file = "types_setuptools-80.9.0.20250529-py3-none-any.whl", hash = "sha256:00dfcedd73e333a430e10db096e4d46af93faf9314f832f13b6bbe3d6757e95f"}, + {file = "types_setuptools-80.9.0.20250529.tar.gz", hash = "sha256:79e088ba0cba2186c8d6499cbd3e143abb142d28a44b042c28d3148b1e353c91"}, ] [[package]] From d57a2f7372239a5014fa953cace7f0e08640a0c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:02:27 +0200 Subject: [PATCH 442/691] Bump msgpack from 1.1.0 to 1.1.1 (#18651) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [msgpack](https://github.com/msgpack/msgpack-python) from 1.1.0 to 1.1.1.
Release notes

Sourced from msgpack's releases.

v1.1.1

What's Changed

New Contributors

Full Changelog: https://github.com/msgpack/msgpack-python/compare/v1.1.0...v1.1.1

v1.1.1rc1

What's Changed

New Contributors

Full Changelog: https://github.com/msgpack/msgpack-python/compare/v1.1.0...v1.1.1rc1

Changelog

Sourced from msgpack's changelog.

1.1.1

Release Date: 2025-06-13

  • No change from 1.1.1rc1.

1.1.1rc1

Release Date: 2025-06-06

  • Update Cython to 3.1.1 and cibuildwheel to 2.23.3.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=msgpack&package-manager=pip&previous-version=1.1.0&new-version=1.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 125 +++++++++++++++++++++++++--------------------------- 1 file changed, 60 insertions(+), 65 deletions(-) diff --git a/poetry.lock b/poetry.lock index 43be4098d4..193e56dac6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1364,76 +1364,71 @@ files = [ [[package]] name = "msgpack" -version = "1.1.0" +version = "1.1.1" description = "MessagePack serializer" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b"}, - {file = "msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044"}, - {file = "msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5"}, - {file = "msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88"}, - {file = "msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b"}, - {file = "msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b"}, - {file = "msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c"}, - {file = "msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc"}, - {file = "msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f"}, - {file = "msgpack-1.1.0-cp38-cp38-win32.whl", hash = "sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b"}, - {file = "msgpack-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8"}, - {file = "msgpack-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd"}, - {file = "msgpack-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325"}, - {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, + {file = "msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed"}, + {file = "msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338"}, + {file = "msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd"}, + {file = "msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752"}, + {file = "msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295"}, + {file = "msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a"}, + {file = "msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c"}, + {file = "msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5"}, + {file = "msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323"}, + {file = "msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bba1be28247e68994355e028dcd668316db30c1f758d3241a7b903ac78dcd285"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8f93dcddb243159c9e4109c9750ba5b335ab8d48d9522c5308cd05d7e3ce600"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fbbc0b906a24038c9958a1ba7ae0918ad35b06cb449d398b76a7d08470b0ed9"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:61e35a55a546a1690d9d09effaa436c25ae6130573b6ee9829c37ef0f18d5e78"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:1abfc6e949b352dadf4bce0eb78023212ec5ac42f6abfd469ce91d783c149c2a"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:996f2609ddf0142daba4cefd767d6db26958aac8439ee41db9cc0db9f4c4c3a6"}, + {file = "msgpack-1.1.1-cp38-cp38-win32.whl", hash = "sha256:4d3237b224b930d58e9d83c81c0dba7aacc20fcc2f89c1e5423aa0529a4cd142"}, + {file = "msgpack-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:da8f41e602574ece93dbbda1fab24650d6bf2a24089f9e9dbb4f5730ec1e58ad"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5be6b6bc52fad84d010cb45433720327ce886009d862f46b26d4d154001994b"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3a89cd8c087ea67e64844287ea52888239cbd2940884eafd2dcd25754fb72232"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d75f3807a9900a7d575d8d6674a3a47e9f227e8716256f35bc6f03fc597ffbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d182dac0221eb8faef2e6f44701812b467c02674a322c739355c39e94730cdbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b13fe0fb4aac1aa5320cd693b297fe6fdef0e7bea5518cbc2dd5299f873ae90"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:435807eeb1bc791ceb3247d13c79868deb22184e1fc4224808750f0d7d1affc1"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4835d17af722609a45e16037bb1d4d78b7bdf19d6c0128116d178956618c4e88"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8ef6e342c137888ebbfb233e02b8fbd689bb5b5fcc59b34711ac47ebd504478"}, + {file = "msgpack-1.1.1-cp39-cp39-win32.whl", hash = "sha256:61abccf9de335d9efd149e2fff97ed5974f2481b3353772e8e2dd3402ba2bd57"}, + {file = "msgpack-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:40eae974c873b2992fd36424a5d9407f93e97656d999f43fca9d29f820899084"}, + {file = "msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd"}, ] [[package]] From acabececc48c9f0b9a927582ea0376ec70627c87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:18:49 +0000 Subject: [PATCH 443/691] Bump ijson from 3.3.0 to 3.4.0 (#18650) Bumps [ijson](https://github.com/ICRAR/ijson) from 3.3.0 to 3.4.0.
Changelog

Sourced from ijson's changelog.

[3.4.0]

  • Added support for PEP 489 multi-phase initialisation and per-module state for our C extension, allowing us to support sub-interpreters with per-interpreter GIL.
  • Advertise support for free-threading python mode.
  • Removed support for Python < 3.9.
  • Enhanced generators so they yield all possible results to users before errors are raised (#123).
  • Added ijson.ALL_BACKENDS constant listing all supported backends (which might or not be available at runtime).
  • Added a capabilities constant to each backend describing which capabilities it supports.
  • Exposing backend's name under <backend>.backend_name, and default backend's name under ijson.backend_name. This is similar to the already existing name constant, only slightly better named to hopefully avoid confusion.
  • Restructured source code so all code lives under src/, and the ijson.backends._yajl2 extension under src/ijson/backends/ext/_yajl2. This allows C backend tests to actually run on cibuildwheel.
  • Improved performance of parse routine in C backend by ~4%.
  • Fixed several potential stability issues in C backend around correct error handling.
  • Fixed corner-case wrong behaviour of yajl2_c backend, which didn't work correctly with user-provided event names.
  • Pointing to our own fork of yajl (for when we build it ourselves) that contains fixes for all known CVEs (#126).
  • Removed leftover compatibility bits in the C backend.
  • Fixed potential issue with yajl and yajl2 backends where crashes could occur at interpreter shutdown.
  • Removed tox.
  • Moved static project metadata to pyproject.toml.
Commits
  • 36701be Release ijson 3.4.0
  • cfb044a Modernize packaging (#138)
  • 044cf9b Bump pypa/cibuildwheel from 2.23.2 to 2.23.3 (#140)
  • 81e24b4 Allow building embedded yajl with cmake 4.0
  • b3f9647 Bump pypa/cibuildwheel from 2.23.1 to 2.23.2 (#137)
  • d8fd6d2 Bump pypa/cibuildwheel from 2.23.0 to 2.23.1 (#134)
  • caebc6f Bump pypa/cibuildwheel from 2.22.0 to 2.23.0 (#133)
  • 698b114 Downgrade to ubuntu-22.04 to avoid gcc segfault
  • 9f28dc0 Allow certain branches to trigger a full CI build
  • 901fd3b Add custom event name interning to yajl2_c
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ijson&package-manager=pip&previous-version=3.3.0&new-version=3.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 183 +++++++++++++++++++++++++--------------------------- 1 file changed, 87 insertions(+), 96 deletions(-) diff --git a/poetry.lock b/poetry.lock index 193e56dac6..30a61a5538 100644 --- a/poetry.lock +++ b/poetry.lock @@ -708,106 +708,97 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 [[package]] name = "ijson" -version = "3.3.0" +version = "3.4.0" description = "Iterative JSON parser with standard Python iterator interfaces" optional = false -python-versions = "*" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7f7a5250599c366369fbf3bc4e176f5daa28eb6bc7d6130d02462ed335361675"}, - {file = "ijson-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f87a7e52f79059f9c58f6886c262061065eb6f7554a587be7ed3aa63e6b71b34"}, - {file = "ijson-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b73b493af9e947caed75d329676b1b801d673b17481962823a3e55fe529c8b8b"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5576415f3d76290b160aa093ff968f8bf6de7d681e16e463a0134106b506f49"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e9ffe358d5fdd6b878a8a364e96e15ca7ca57b92a48f588378cef315a8b019e"}, - {file = "ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8643c255a25824ddd0895c59f2319c019e13e949dc37162f876c41a283361527"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:df3ab5e078cab19f7eaeef1d5f063103e1ebf8c26d059767b26a6a0ad8b250a3"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3dc1fb02c6ed0bae1b4bf96971258bf88aea72051b6e4cebae97cff7090c0607"}, - {file = "ijson-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e9afd97339fc5a20f0542c971f90f3ca97e73d3050cdc488d540b63fae45329a"}, - {file = "ijson-3.3.0-cp310-cp310-win32.whl", hash = "sha256:844c0d1c04c40fd1b60f148dc829d3f69b2de789d0ba239c35136efe9a386529"}, - {file = "ijson-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:d654d045adafdcc6c100e8e911508a2eedbd2a1b5f93f930ba13ea67d7704ee9"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:501dce8eaa537e728aa35810656aa00460a2547dcb60937c8139f36ec344d7fc"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:658ba9cad0374d37b38c9893f4864f284cdcc7d32041f9808fba8c7bcaadf134"}, - {file = "ijson-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2636cb8c0f1023ef16173f4b9a233bcdb1df11c400c603d5f299fac143ca8d70"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd174b90db68c3bcca273e9391934a25d76929d727dc75224bf244446b28b03b"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97a9aea46e2a8371c4cf5386d881de833ed782901ac9f67ebcb63bb3b7d115af"}, - {file = "ijson-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c594c0abe69d9d6099f4ece17763d53072f65ba60b372d8ba6de8695ce6ee39e"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e0ff16c224d9bfe4e9e6bd0395826096cda4a3ef51e6c301e1b61007ee2bd24"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0015354011303175eae7e2ef5136414e91de2298e5a2e9580ed100b728c07e51"}, - {file = "ijson-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:034642558afa57351a0ffe6de89e63907c4cf6849070cc10a3b2542dccda1afe"}, - {file = "ijson-3.3.0-cp311-cp311-win32.whl", hash = "sha256:192e4b65495978b0bce0c78e859d14772e841724d3269fc1667dc6d2f53cc0ea"}, - {file = "ijson-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:72e3488453754bdb45c878e31ce557ea87e1eb0f8b4fc610373da35e8074ce42"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:988e959f2f3d59ebd9c2962ae71b97c0df58323910d0b368cc190ad07429d1bb"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b2f73f0d0fce5300f23a1383d19b44d103bb113b57a69c36fd95b7c03099b181"}, - {file = "ijson-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0ee57a28c6bf523d7cb0513096e4eb4dac16cd935695049de7608ec110c2b751"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0155a8f079c688c2ccaea05de1ad69877995c547ba3d3612c1c336edc12a3a5"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ab00721304af1ae1afa4313ecfa1bf16b07f55ef91e4a5b93aeaa3e2bd7917c"}, - {file = "ijson-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40ee3821ee90be0f0e95dcf9862d786a7439bd1113e370736bfdf197e9765bfb"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3b6987a0bc3e6d0f721b42c7a0198ef897ae50579547b0345f7f02486898f5"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:63afea5f2d50d931feb20dcc50954e23cef4127606cc0ecf7a27128ed9f9a9e6"}, - {file = "ijson-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5c3e285e0735fd8c5a26d177eca8b52512cdd8687ca86ec77a0c66e9c510182"}, - {file = "ijson-3.3.0-cp312-cp312-win32.whl", hash = "sha256:907f3a8674e489abdcb0206723e5560a5cb1fa42470dcc637942d7b10f28b695"}, - {file = "ijson-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f890d04ad33262d0c77ead53c85f13abfb82f2c8f078dfbf24b78f59534dfdd"}, - {file = "ijson-3.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b9d85a02e77ee8ea6d9e3fd5d515bcc3d798d9c1ea54817e5feb97a9bc5d52fe"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6576cdc36d5a09b0c1a3d81e13a45d41a6763188f9eaae2da2839e8a4240bce"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5589225c2da4bb732c9c370c5961c39a6db72cf69fb2a28868a5413ed7f39e6"}, - {file = "ijson-3.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad04cf38164d983e85f9cba2804566c0160b47086dcca4cf059f7e26c5ace8ca"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:a3b730ef664b2ef0e99dec01b6573b9b085c766400af363833e08ebc1e38eb2f"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:4690e3af7b134298055993fcbea161598d23b6d3ede11b12dca6815d82d101d5"}, - {file = "ijson-3.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:aaa6bfc2180c31a45fac35d40e3312a3d09954638ce0b2e9424a88e24d262a13"}, - {file = "ijson-3.3.0-cp36-cp36m-win32.whl", hash = "sha256:44367090a5a876809eb24943f31e470ba372aaa0d7396b92b953dda953a95d14"}, - {file = "ijson-3.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7e2b3e9ca957153557d06c50a26abaf0d0d6c0ddf462271854c968277a6b5372"}, - {file = "ijson-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47c144117e5c0e2babb559bc8f3f76153863b8dd90b2d550c51dab5f4b84a87f"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ce02af5fbf9ba6abb70765e66930aedf73311c7d840478f1ccecac53fefbf3"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac6c3eeed25e3e2cb9b379b48196413e40ac4e2239d910bb33e4e7f6c137745"}, - {file = "ijson-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d92e339c69b585e7b1d857308ad3ca1636b899e4557897ccd91bb9e4a56c965b"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:8c85447569041939111b8c7dbf6f8fa7a0eb5b2c4aebb3c3bec0fb50d7025121"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:542c1e8fddf082159a5d759ee1412c73e944a9a2412077ed00b303ff796907dc"}, - {file = "ijson-3.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:30cfea40936afb33b57d24ceaf60d0a2e3d5c1f2335ba2623f21d560737cc730"}, - {file = "ijson-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:6b661a959226ad0d255e49b77dba1d13782f028589a42dc3172398dd3814c797"}, - {file = "ijson-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0b003501ee0301dbf07d1597482009295e16d647bb177ce52076c2d5e64113e0"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e8d8de44effe2dbd0d8f3eb9840344b2d5b4cc284a14eb8678aec31d1b6bea8"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9cd5c03c63ae06d4f876b9844c5898d0044c7940ff7460db9f4cd984ac7862b5"}, - {file = "ijson-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04366e7e4a4078d410845e58a2987fd9c45e63df70773d7b6e87ceef771b51ee"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de7c1ddb80fa7a3ab045266dca169004b93f284756ad198306533b792774f10a"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8851584fb931cffc0caa395f6980525fd5116eab8f73ece9d95e6f9c2c326c4c"}, - {file = "ijson-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdcfc88347fd981e53c33d832ce4d3e981a0d696b712fbcb45dcc1a43fe65c65"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3917b2b3d0dbbe3296505da52b3cb0befbaf76119b2edaff30bd448af20b5400"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:e10c14535abc7ddf3fd024aa36563cd8ab5d2bb6234a5d22c77c30e30fa4fb2b"}, - {file = "ijson-3.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3aba5c4f97f4e2ce854b5591a8b0711ca3b0c64d1b253b04ea7b004b0a197ef6"}, - {file = "ijson-3.3.0-cp38-cp38-win32.whl", hash = "sha256:b325f42e26659df1a0de66fdb5cde8dd48613da9c99c07d04e9fb9e254b7ee1c"}, - {file = "ijson-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:ff835906f84451e143f31c4ce8ad73d83ef4476b944c2a2da91aec8b649570e1"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3c556f5553368dff690c11d0a1fb435d4ff1f84382d904ccc2dc53beb27ba62e"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e4396b55a364a03ff7e71a34828c3ed0c506814dd1f50e16ebed3fc447d5188e"}, - {file = "ijson-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6850ae33529d1e43791b30575070670070d5fe007c37f5d06aebc1dd152ab3f"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36aa56d68ea8def26778eb21576ae13f27b4a47263a7a2581ab2ef58b8de4451"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7ec759c4a0fc820ad5dc6a58e9c391e7b16edcb618056baedbedbb9ea3b1524"}, - {file = "ijson-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b51bab2c4e545dde93cb6d6bb34bf63300b7cd06716f195dd92d9255df728331"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:92355f95a0e4da96d4c404aa3cff2ff033f9180a9515f813255e1526551298c1"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8795e88adff5aa3c248c1edce932db003d37a623b5787669ccf205c422b91e4a"}, - {file = "ijson-3.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8f83f553f4cde6d3d4eaf58ec11c939c94a0ec545c5b287461cafb184f4b3a14"}, - {file = "ijson-3.3.0-cp39-cp39-win32.whl", hash = "sha256:ead50635fb56577c07eff3e557dac39533e0fe603000684eea2af3ed1ad8f941"}, - {file = "ijson-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:c8a9befb0c0369f0cf5c1b94178d0d78f66d9cebb9265b36be6e4f66236076b8"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2af323a8aec8a50fa9effa6d640691a30a9f8c4925bd5364a1ca97f1ac6b9b5c"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f64f01795119880023ba3ce43072283a393f0b90f52b66cc0ea1a89aa64a9ccb"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a716e05547a39b788deaf22725490855337fc36613288aa8ae1601dc8c525553"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473f5d921fadc135d1ad698e2697025045cd8ed7e5e842258295012d8a3bc702"}, - {file = "ijson-3.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd26b396bc3a1e85f4acebeadbf627fa6117b97f4c10b177d5779577c6607744"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:25fd49031cdf5fd5f1fd21cb45259a64dad30b67e64f745cc8926af1c8c243d3"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b72178b1e565d06ab19319965022b36ef41bcea7ea153b32ec31194bec032a2"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d0b6b637d05dbdb29d0bfac2ed8425bb369e7af5271b0cc7cf8b801cb7360c2"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5378d0baa59ae422905c5f182ea0fd74fe7e52a23e3821067a7d58c8306b2191"}, - {file = "ijson-3.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:99f5c8ab048ee4233cc4f2b461b205cbe01194f6201018174ac269bf09995749"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:45ff05de889f3dc3d37a59d02096948ce470699f2368b32113954818b21aa74a"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1efb521090dd6cefa7aafd120581947b29af1713c902ff54336b7c7130f04c47"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87c727691858fd3a1c085d9980d12395517fcbbf02c69fbb22dede8ee03422da"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0420c24e50389bc251b43c8ed379ab3e3ba065ac8262d98beb6735ab14844460"}, - {file = "ijson-3.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8fdf3721a2aa7d96577970f5604bd81f426969c1822d467f07b3d844fa2fecc7"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:891f95c036df1bc95309951940f8eea8537f102fa65715cdc5aae20b8523813b"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed1336a2a6e5c427f419da0154e775834abcbc8ddd703004108121c6dd9eba9d"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0c819f83e4f7b7f7463b2dc10d626a8be0c85fbc7b3db0edc098c2b16ac968e"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33afc25057377a6a43c892de34d229a86f89ea6c4ca3dd3db0dcd17becae0dbb"}, - {file = "ijson-3.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7914d0cf083471856e9bc2001102a20f08e82311dfc8cf1a91aa422f9414a0d6"}, - {file = "ijson-3.3.0.tar.gz", hash = "sha256:7f172e6ba1bee0d4c8f8ebd639577bfe429dee0f3f96775a067b8bae4492d8a0"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e27e50f6dcdee648f704abc5d31b976cd2f90b4642ed447cf03296d138433d09"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a753be681ac930740a4af9c93cfb4edc49a167faed48061ea650dc5b0f406f1"}, + {file = "ijson-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a07c47aed534e0ec198e6a2d4360b259d32ac654af59c015afc517ad7973b7fb"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c55f48181e11c597cd7146fb31edc8058391201ead69f8f40d2ecbb0b3e4fc6"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd5669f96f79d8a2dd5ae81cbd06770a4d42c435fd4a75c74ef28d9913b697d"}, + {file = "ijson-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e3ddd46d16b8542c63b1b8af7006c758d4e21cc1b86122c15f8530fae773461"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1504cec7fe04be2bb0cc33b50c9dd3f83f98c0540ad4991d4017373b7853cfe6"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2f2ff456adeb216603e25d7915f10584c1b958b6eafa60038d76d08fc8a5fb06"}, + {file = "ijson-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0ab00d75d61613a125fbbb524551658b1ad6919a52271ca16563ca5bc2737bb1"}, + {file = "ijson-3.4.0-cp310-cp310-win32.whl", hash = "sha256:ada421fd59fe2bfa4cfa64ba39aeba3f0753696cdcd4d50396a85f38b1d12b01"}, + {file = "ijson-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:8c75e82cec05d00ed3a4af5f4edf08f59d536ed1a86ac7e84044870872d82a33"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e369bf5a173ca51846c243002ad8025d32032532523b06510881ecc8723ee54"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26e7da0a3cd2a56a1fde1b34231867693f21c528b683856f6691e95f9f39caec"}, + {file = "ijson-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c28c7f604729be22aa453e604e9617b665fa0c24cd25f9f47a970e8130c571a"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bed8bcb84d3468940f97869da323ba09ae3e6b950df11dea9b62e2b231ca1e3"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:296bc824f4088f2af814aaf973b0435bc887ce3d9f517b1577cc4e7d1afb1cb7"}, + {file = "ijson-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8145f8f40617b6a8aa24e28559d0adc8b889e56a203725226a8a60fa3501073f"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b674a97bd503ea21bc85103e06b6493b1b2a12da3372950f53e1c664566a33a4"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8bc731cf1c3282b021d3407a601a5a327613da9ad3c4cecb1123232623ae1826"}, + {file = "ijson-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:42ace5e940e0cf58c9de72f688d6829ddd815096d07927ee7e77df2648006365"}, + {file = "ijson-3.4.0-cp311-cp311-win32.whl", hash = "sha256:5be39a0df4cd3f02b304382ea8885391900ac62e95888af47525a287c50005e9"}, + {file = "ijson-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:0b1be1781792291e70d2e177acf564ec672a7907ba74f313583bdf39fe81f9b7"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:956b148f88259a80a9027ffbe2d91705fae0c004fbfba3e5a24028fbe72311a9"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06b89960f5c721106394c7fba5760b3f67c515b8eb7d80f612388f5eca2f4621"}, + {file = "ijson-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a0bb591cf250dd7e9dfab69d634745a7f3272d31cfe879f9156e0a081fd97ee"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72e92de999977f4c6b660ffcf2b8d59604ccd531edcbfde05b642baf283e0de8"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e9602157a5b869d44b6896e64f502c712a312fcde044c2e586fccb85d3e316e"}, + {file = "ijson-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e83660edb931a425b7ff662eb49db1f10d30ca6d4d350e5630edbed098bc01"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:49bf8eac1c7b7913073865a859c215488461f7591b4fa6a33c14b51cb73659d0"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:160b09273cb42019f1811469508b0a057d19f26434d44752bde6f281da6d3f32"}, + {file = "ijson-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2019ff4e6f354aa00c76c8591bd450899111c61f2354ad55cc127e2ce2492c44"}, + {file = "ijson-3.4.0-cp312-cp312-win32.whl", hash = "sha256:931c007bf6bb8330705429989b2deed6838c22b63358a330bf362b6e458ba0bf"}, + {file = "ijson-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:71523f2b64cb856a820223e94d23e88369f193017ecc789bb4de198cc9d349eb"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e8d96f88d75196a61c9d9443de2b72c2d4a7ba9456ff117b57ae3bba23a54256"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c45906ce2c1d3b62f15645476fc3a6ca279549127f01662a39ca5ed334a00cf9"}, + {file = "ijson-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4ab4bc2119b35c4363ea49f29563612237cae9413d2fbe54b223be098b97bc9e"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97b0a9b5a15e61dfb1f14921ea4e0dba39f3a650df6d8f444ddbc2b19b479ff1"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3047bb994dabedf11de11076ed1147a307924b6e5e2df6784fb2599c4ad8c60"}, + {file = "ijson-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68c83161b052e9f5dc8191acbc862bb1e63f8a35344cb5cd0db1afd3afd487a6"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1eebd9b6c20eb1dffde0ae1f0fbb4aeacec2eb7b89adb5c7c0449fc9fd742760"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:13fb6d5c35192c541421f3ee81239d91fc15a8d8f26c869250f941f4b346a86c"}, + {file = "ijson-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28b7196ff7b37c4897c547a28fa4876919696739fc91c1f347651c9736877c69"}, + {file = "ijson-3.4.0-cp313-cp313-win32.whl", hash = "sha256:3c2691d2da42629522140f77b99587d6f5010440d58d36616f33bc7bdc830cc3"}, + {file = "ijson-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:c4554718c275a044c47eb3874f78f2c939f300215d9031e785a6711cc51b83fc"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:915a65e3f3c0eee2ea937bc62aaedb6c14cc1e8f0bb9f3f4fb5a9e2bbfa4b480"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:afbe9748707684b6c5adc295c4fdcf27765b300aec4d484e14a13dca4e5c0afa"}, + {file = "ijson-3.4.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d823f8f321b4d8d5fa020d0a84f089fec5d52b7c0762430476d9f8bf95bbc1a9"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0a2c54f3becf76881188beefd98b484b1d3bd005769a740d5b433b089fa23"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ced19a83ab09afa16257a0b15bc1aa888dbc555cb754be09d375c7f8d41051f2"}, + {file = "ijson-3.4.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8100f9885eff1f38d35cef80ef759a1bbf5fc946349afa681bd7d0e681b7f1a0"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d7bcc3f7f21b0f703031ecd15209b1284ea51b2a329d66074b5261de3916c1eb"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2dcb190227b09dd171bdcbfe4720fddd574933c66314818dfb3960c8a6246a77"}, + {file = "ijson-3.4.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:eda4cfb1d49c6073a901735aaa62e39cb7ab47f3ad7bb184862562f776f1fa8a"}, + {file = "ijson-3.4.0-cp313-cp313t-win32.whl", hash = "sha256:0772638efa1f3b72b51736833404f1cbd2f5beeb9c1a3d392e7d385b9160cba7"}, + {file = "ijson-3.4.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3d8a0d67f36e4fb97c61a724456ef0791504b16ce6f74917a31c2e92309bbeb9"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8a990401dc7350c1739f42187823e68d2ef6964b55040c6e9f3a29461f9929e2"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:80f50e0f5da4cd6b65e2d8ff38cb61b26559608a05dd3a3f9cfa6f19848e6f22"}, + {file = "ijson-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d9ca52f5650d820a2e7aa672dea1c560f609e165337e5b3ed7cf56d696bf309"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:940c8c5fd20fb89b56dde9194a4f1c7b779149f1ab26af6d8dc1da51a95d26dd"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:41dbb525666017ad856ac9b4f0f4b87d3e56b7dfde680d5f6d123556b22e2172"}, + {file = "ijson-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9f84f5e2eea5c2d271c97221c382db005534294d1175ddd046a12369617c41c"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0cd126c11835839bba8ac0baaba568f67d701fc4f717791cf37b10b74a2ebd7"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f9a9d3bbc6d91c24a2524a189d2aca703cb5f7e8eb34ad0aff3c91702404a983"}, + {file = "ijson-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:56679ee133470d0f1f598a8ad109d760fcfebeef4819531e29335aefb7e4cb1a"}, + {file = "ijson-3.4.0-cp39-cp39-win32.whl", hash = "sha256:583c15ded42ba80104fa1d0fa0dfdd89bb47922f3bb893a931bb843aeb55a3f3"}, + {file = "ijson-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:4563e603e56f4451572d96b47311dffef5b933d825f3417881d4d3630c6edac2"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:54e989c35dba9cf163d532c14bcf0c260897d5f465643f0cd1fba9c908bed7ef"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:494eeb8e87afef22fbb969a4cb81ac2c535f30406f334fb6136e9117b0bb5380"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81603de95de1688958af65cd2294881a4790edae7de540b70c65c8253c5dc44a"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8524be12c1773e1be466034cc49c1ecbe3d5b47bb86217bd2a57f73f970a6c19"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17994696ec895d05e0cfa21b11c68c920c82634b4a3d8b8a1455d6fe9fdee8f7"}, + {file = "ijson-3.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0b67727aaee55d43b2e82b6a866c3cbcb2b66a5e9894212190cbd8773d0d9857"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cdc8c5ca0eec789ed99db29c68012dda05027af0860bb360afd28d825238d69d"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:8e6b44b6ec45d5b1a0ee9d97e0e65ab7f62258727004cbbe202bf5f198bc21f7"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b51e239e4cb537929796e840d349fc731fdc0d58b1a0683ce5465ad725321e0f"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed05d43ec02be8ddb1ab59579761f6656b25d241a77fd74f4f0f7ec09074318a"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfeca1aaa59d93fd0a3718cbe5f7ef0effff85cf837e0bceb71831a47f39cc14"}, + {file = "ijson-3.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7ca72ca12e9a1dd4252c97d952be34282907f263f7e28fcdff3a01b83981e837"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f79b2cd52bd220fff83b3ee4ef89b54fd897f57cc8564a6d8ab7ac669de3930"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d16eed737610ad5ad8989b5864fbe09c64133129734e840c29085bb0d497fb03"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b3aac1d7a27e1e3bdec5bd0689afe55c34aa499baa06a80852eda31f1ffa6dc"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:784ae654aa9851851e87f323e9429b20b58a5399f83e6a7e348e080f2892081f"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d05bd8fa6a8adefb32bbf7b993d2a2f4507db08453dd1a444c281413a6d9685"}, + {file = "ijson-3.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b5a05fd935cc28786b88c16976313086cd96414c6a3eb0a3822c47ab48b1793e"}, + {file = "ijson-3.4.0.tar.gz", hash = "sha256:5f74dcbad9d592c428d3ca3957f7115a42689ee7ee941458860900236ae9bb13"}, ] [[package]] From 4ad93b296fae6062b6d17ff8d184a41636a6f84e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:20:07 +0000 Subject: [PATCH 444/691] Bump authlib from 1.5.2 to 1.6.0 (#18642) Bumps [authlib](https://github.com/authlib/authlib) from 1.5.2 to 1.6.0.
Release notes

Sourced from authlib's releases.

Version 1.6.0

Changelog

Sourced from authlib's changelog.

Version 1.6.0

Released on May 22, 2025

  • Fix issue when :rfc:RFC9207 <9207> is enabled and the authorization endpoint response is not a redirection. :pr:733
  • Fix missing state parameter in authorization error responses. :issue:525
  • Support for acr and amr claims in id_token. :issue:734
  • Support for the none JWS algorithm.
  • Fix response_types strict order during dynamic client registration. :issue:760
  • Implement :rfc:RFC9101 The OAuth 2.0 Authorization Framework: JWT-Secured Authorization Request (JAR) <9101>. :issue:723
  • OIDC :class:UserInfo endpoint <authlib.oidc.core.userinfo.UserInfoEndpoint> support. :issue:459
Commits
  • fe87a11 chore: release version 1.6.0
  • 036a0b7 Merge pull request #774 from azmeuk/459-userinfo
  • 449a1a2 feat: OIDC userinfo endpoint support
  • d429c36 Merge pull request #749 from azmeuk/724-jar
  • a524d23 chore: move 1.7 deprecations to 1.8
  • f37e60e feat: implement rfc9101 JWT authorization request
  • 8a6c714 refactor: OAuth2 hook mechanism overhaul
  • ff1b66b refactor: extract OAuth2Payload from OAuth2Request
  • 98eebd1 refactor: remove uncovered code in OAuth2Request
  • 1b848e2 refactor: create_authorization_response can take an optional 'grant' arg
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=authlib&package-manager=pip&previous-version=1.5.2&new-version=1.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 30a61a5538..ceb60fa98f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,15 +34,15 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a [[package]] name = "authlib" -version = "1.5.2" +version = "1.6.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "authlib-1.5.2-py2.py3-none-any.whl", hash = "sha256:8804dd4402ac5e4a0435ac49e0b6e19e395357cfa632a3f624dcb4f6df13b4b1"}, - {file = "authlib-1.5.2.tar.gz", hash = "sha256:fe85ec7e50c5f86f1e2603518bb3b4f632985eb4a355e52256530790e326c512"}, + {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, + {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, ] [package.dependencies] From fc710c4804a992ef77d942c7adc6f0cd42a61ebe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:28:02 +0000 Subject: [PATCH 445/691] Bump attrs from 24.2.0 to 25.3.0 (#18649) Bumps [attrs](https://github.com/sponsors/hynek) from 24.2.0 to 25.3.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=attrs&package-manager=pip&previous-version=24.2.0&new-version=25.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index ceb60fa98f..37e2731a5f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,23 +14,23 @@ files = [ [[package]] name = "attrs" -version = "24.2.0" +version = "25.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, + {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, ] [package.extras] -benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] -cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] -dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] -docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "authlib" From 6096cb4054acffe15785d550d6676a6103a8a308 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:01:59 +0000 Subject: [PATCH 446/691] Bump phonenumbers from 9.0.2 to 9.0.8 (#18652) Bumps [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) from 9.0.2 to 9.0.8.
Commits
  • 9959754 Prep for 9.0.8 release
  • 6ffa6ff Generated files for metadata
  • 4b028b4 Merge metadata changes from upstream 9.0.8
  • 6817dfb Prep for 9.0.7 release
  • e9a4843 Generated files for metadata
  • 8580645 Merge metadata changes from upstream 9.0.7
  • e6d5f62 Prep for 9.0.6 release
  • e4e7dbe Generated files for metadata
  • a8425e7 Merge metadata changes from upstream 9.0.6
  • e90d8ea Prep for 9.0.5 release
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=phonenumbers&package-manager=pip&previous-version=9.0.2&new-version=9.0.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 37e2731a5f..eed8012513 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1583,14 +1583,14 @@ dev = ["jinja2"] [[package]] name = "phonenumbers" -version = "9.0.2" +version = "9.0.8" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-9.0.2-py2.py3-none-any.whl", hash = "sha256:dbcec6bdfdf3973f60b81dc0fcac3f7b1638f877ac42da4d7b46724ed413e2b9"}, - {file = "phonenumbers-9.0.2.tar.gz", hash = "sha256:f590ee2b729bdd9873ca2d52989466add14c9953b48805c0aeb408348d4d6224"}, + {file = "phonenumbers-9.0.8-py2.py3-none-any.whl", hash = "sha256:53d357111c0ead0d6408ae443613b18d3a053431ca1ddf7e881457c0969afcf9"}, + {file = "phonenumbers-9.0.8.tar.gz", hash = "sha256:16f03f2cf65b5eee99ed25827d810febcab92b5d76f977e425fcd2e4ca6d4865"}, ] [[package]] @@ -1603,6 +1603,8 @@ groups = ["main"] files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, @@ -1612,6 +1614,8 @@ files = [ {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, @@ -1621,6 +1625,8 @@ files = [ {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, @@ -1633,6 +1639,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, @@ -1642,6 +1650,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, @@ -1651,6 +1661,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, @@ -1660,6 +1672,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, @@ -1669,6 +1683,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, @@ -1678,11 +1694,15 @@ files = [ {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, From cb0d5a3012387223b5d073e18472a990d90c5d26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:03:08 +0000 Subject: [PATCH 447/691] Bump typing-extensions from 4.12.2 to 4.14.0 (#18654) Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.12.2 to 4.14.0.
Release notes

Sourced from typing-extensions's releases.

4.14.0

This release adds several new features, including experimental support for inline typed dictionaries (PEP 764) and sentinels (PEP 661), and support for changes in Python 3.14. In addition, Python 3.8 is no longer supported.

Changes since 4.14.0rc1:

  • Remove __or__ and __ror__ methods from typing_extensions.Sentinel on Python versions <3.10. PEP 604 was introduced in Python 3.10, and typing_extensions does not generally attempt to backport PEP-604 methods to prior versions.
  • Further update typing_extensions.evaluate_forward_ref with changes in Python 3.14.

Changes included in 4.14.0rc1:

  • Drop support for Python 3.8 (including PyPy-3.8). Patch by Victorien Plot.
  • Do not attempt to re-export names that have been removed from typing, anticipating the removal of typing.no_type_check_decorator in Python 3.15. Patch by Jelle Zijlstra.
  • Update typing_extensions.Format, typing_extensions.evaluate_forward_ref, and typing_extensions.TypedDict to align with changes in Python 3.14. Patches by Jelle Zijlstra.
  • Fix tests for Python 3.14 and 3.15. Patches by Jelle Zijlstra.

New features:

  • Add support for inline typed dictionaries (PEP 764). Patch by Victorien Plot.
  • Add typing_extensions.Reader and typing_extensions.Writer. Patch by Sebastian Rittau.
  • Add support for sentinels (PEP 661). Patch by Victorien Plot.

4.14.0rc1

Major changes:

  • Drop support for Python 3.8 (including PyPy-3.8). Patch by Victorien Plot.
  • Do not attempt to re-export names that have been removed from typing, anticipating the removal of typing.no_type_check_decorator in Python 3.15. Patch by Jelle Zijlstra.
  • Update typing_extensions.Format, typing_extensions.evaluate_forward_ref, and typing_extensions.TypedDict to align with changes in Python 3.14. Patches by Jelle Zijlstra.
  • Fix tests for Python 3.14 and 3.15. Patches by Jelle Zijlstra.

New features:

  • Add support for inline typed dictionaries (PEP 764). Patch by Victorien Plot.
  • Add typing_extensions.Reader and typing_extensions.Writer. Patch by Sebastian Rittau.
  • Add support for sentinels (PEP 661). Patch by

... (truncated)

Changelog

Sourced from typing-extensions's changelog.

Release 4.14.0 (June 2, 2025)

Changes since 4.14.0rc1:

  • Remove __or__ and __ror__ methods from typing_extensions.Sentinel on Python versions <3.10. PEP 604 was introduced in Python 3.10, and typing_extensions does not generally attempt to backport PEP-604 methods to prior versions.
  • Further update typing_extensions.evaluate_forward_ref with changes in Python 3.14.

Release 4.14.0rc1 (May 24, 2025)

  • Drop support for Python 3.8 (including PyPy-3.8). Patch by Victorien Plot.
  • Do not attempt to re-export names that have been removed from typing, anticipating the removal of typing.no_type_check_decorator in Python 3.15. Patch by Jelle Zijlstra.
  • Update typing_extensions.Format, typing_extensions.evaluate_forward_ref, and typing_extensions.TypedDict to align with changes in Python 3.14. Patches by Jelle Zijlstra.
  • Fix tests for Python 3.14 and 3.15. Patches by Jelle Zijlstra.

New features:

  • Add support for inline typed dictionaries (PEP 764). Patch by Victorien Plot.
  • Add typing_extensions.Reader and typing_extensions.Writer. Patch by Sebastian Rittau.
  • Add support for sentinels (PEP 661). Patch by Victorien Plot.

Release 4.13.2 (April 10, 2025)

  • Fix TypeError when taking the union of typing_extensions.TypeAliasType and a typing.TypeAliasType on Python 3.12 and 3.13. Patch by Joren Hammudoglu.
  • Backport from CPython PR #132160 to avoid having user arguments shadowed in generated __new__ by @typing_extensions.deprecated. Patch by Victorien Plot.

Release 4.13.1 (April 3, 2025)

Bugfixes:

  • Fix regression in 4.13.0 on Python 3.10.2 causing a TypeError when using Concatenate. Patch by Daraan.
  • Fix TypeError when using evaluate_forward_ref on Python 3.10.1-2 and 3.9.8-10. Patch by Daraan.

Release 4.13.0 (March 25, 2025)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=typing-extensions&package-manager=pip&previous-version=4.12.2&new-version=4.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index eed8012513..5a2294f867 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3101,14 +3101,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.12.2" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.14.0" +description = "Backported and Experimental Type Hints for Python 3.9+" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, + {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, + {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, ] [[package]] From d61b919030372bcc7042c3689bd986284a20f5aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Jul 2025 09:05:25 +0000 Subject: [PATCH 448/691] Bump setuptools-rust from 1.10.2 to 1.11.1 (#18655) Bumps [setuptools-rust](https://github.com/PyO3/setuptools-rust) from 1.10.2 to 1.11.1.
Release notes

Sourced from setuptools-rust's releases.

v1.11.1

Fixed

  • Fix finding cargo artifacts when filenames are empty. #521

v1.11.0

Packaging

  • Drop support for Python 3.8. #479
  • Support free-threaded Python. #502
  • Support adding custom env vars. #504
Changelog

Sourced from setuptools-rust's changelog.

1.11.1 (2025-04-04)

Fixed

  • Fix finding cargo artifacts when filenames are empty. #521

1.11.0 (2025-03-14)

Packaging

  • Drop support for Python 3.8. #479
  • Support free-threaded Python. #502
  • Support adding custom env vars. #504
Commits
  • cc5c233 release: 1.11.1 (#529)
  • faa610d Fix finding cargo artifacts when filenames are empty (#521)
  • a766bd5 use SETUPTOOLS_RUST_CARGO_PROFILE=dev for main matrix, document it (#527)
  • 229c492 build(deps): bump the deps group across 5 directories with 1 update (#522)
  • 4141b0b build(deps): bump pypa/cibuildwheel from 2.22.0 to 2.23.2 (#523)
  • fc4d72c build(deps): bump pyo3 in /examples/hello-world-setuppy (#524)
  • f20ba9c replace quansight-labs/setup-python with actions/setup-python (#520)
  • 1054e1c release: 1.11.0 (#518)
  • d6817d7 Support adding custom env vars (#504)
  • 17980ef build(deps): bump the deps group across 3 directories with 1 update (#516)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=setuptools-rust&package-manager=pip&previous-version=1.10.2&new-version=1.11.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5a2294f867..ca0850353d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2613,18 +2613,18 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve [[package]] name = "setuptools-rust" -version = "1.10.2" +version = "1.11.1" description = "Setuptools Rust extension plugin" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools_rust-1.10.2-py3-none-any.whl", hash = "sha256:4b39c435ae9670315d522ed08fa0e8cb29f2a6048033966b6be2571a90ce4f1c"}, - {file = "setuptools_rust-1.10.2.tar.gz", hash = "sha256:5d73e7eee5f87a6417285b617c97088a7c20d1a70fcea60e3bdc94ff567c29dc"}, + {file = "setuptools_rust-1.11.1-py3-none-any.whl", hash = "sha256:5eaaddaed268dc24a527ffa659ce56b22d3cf17b781247b779efd611031fe8ea"}, + {file = "setuptools_rust-1.11.1.tar.gz", hash = "sha256:7dabc4392252ced314b8050d63276e05fdc5d32398fc7d3cce1f6a6ac35b76c0"}, ] [package.dependencies] -semantic-version = ">=2.8.2,<3" +semantic_version = ">=2.8.2,<3" setuptools = ">=62.4" [[package]] From 4a7b166d8069d3fc8a21df410efa34fb03235e29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:12:59 +0000 Subject: [PATCH 449/691] Bump ruff from 0.12.1 to 0.12.2 (#18657) Bumps [ruff](https://github.com/astral-sh/ruff) from 0.12.1 to 0.12.2.
Release notes

Sourced from ruff's releases.

0.12.2

Release Notes

Preview features

  • [flake8-pyi] Expand Optional[A] to A | None (PYI016) (#18572)
  • [pyupgrade] Mark UP008 fix safe if no comments are in range (#18683)

Bug fixes

  • [flake8-comprehensions] Fix C420 to prepend whitespace when needed (#18616)
  • [perflint] Fix PERF403 panic on attribute or subscription loop variable (#19042)
  • [pydocstyle] Fix D413 infinite loop for parenthesized docstring (#18930)
  • [pylint] Fix PLW0108 autofix introducing a syntax error when the lambda's body contains an assignment expression (#18678)
  • [refurb] Fix false positive on empty tuples (FURB168) (#19058)
  • [ruff] Allow more field calls from attrs (RUF009) (#19021)
  • [ruff] Fix syntax error introduced for an empty string followed by a u-prefixed string (UP025) (#18899)

Rule changes

  • [flake8-executable] Allow uvx in shebang line (EXE003) (#18967)
  • [pandas] Avoid flagging PD002 if pandas is not imported (#18963)
  • [pyupgrade] Avoid PEP-604 unions with typing.NamedTuple (UP007, UP045) (#18682)

Documentation

  • Document link between import-outside-top-level (PLC0415) and lint.flake8-tidy-imports.banned-module-level-imports (#18733)
  • Fix description of the format.skip-magic-trailing-comma example (#19095)
  • [airflow] Make AIR302 example error out-of-the-box (#18988)
  • [airflow] Make AIR312 example error out-of-the-box (#18989)
  • [flake8-annotations] Make ANN401 example error out-of-the-box (#18974)
  • [flake8-async] Make ASYNC100 example error out-of-the-box (#18993)
  • [flake8-async] Make ASYNC105 example error out-of-the-box (#19002)
  • [flake8-async] Make ASYNC110 example error out-of-the-box (#18975)
  • [flake8-async] Make ASYNC210 example error out-of-the-box (#18977)
  • [flake8-async] Make ASYNC220, ASYNC221, and ASYNC222 examples error out-of-the-box (#18978)
  • [flake8-async] Make ASYNC251 example error out-of-the-box (#18990)
  • [flake8-bandit] Make S201 example error out-of-the-box (#19017)
  • [flake8-bandit] Make S604 and S609 examples error out-of-the-box (#19049)
  • [flake8-bugbear] Make B028 example error out-of-the-box (#19054)
  • [flake8-bugbear] Make B911 example error out-of-the-box (#19051)
  • [flake8-datetimez] Make DTZ011 example error out-of-the-box (#19055)
  • [flake8-datetimez] Make DTZ901 example error out-of-the-box (#19056)
  • [flake8-pyi] Make PYI032 example error out-of-the-box (#19061)
  • [flake8-pyi] Make example error out-of-the-box (PYI014, PYI015) (#19097)
  • [flake8-pyi] Make example error out-of-the-box (PYI042) (#19101)
  • [flake8-pyi] Make example error out-of-the-box (PYI059) (#19080)
  • [flake8-pyi] Make example error out-of-the-box (PYI062) (#19079)
  • [flake8-pytest-style] Make example error out-of-the-box (PT023) (#19104)
  • [flake8-pytest-style] Make example error out-of-the-box (PT030) (#19105)

... (truncated)

Changelog

Sourced from ruff's changelog.

0.12.2

Preview features

  • [flake8-pyi] Expand Optional[A] to A | None (PYI016) (#18572)
  • [pyupgrade] Mark UP008 fix safe if no comments are in range (#18683)

Bug fixes

  • [flake8-comprehensions] Fix C420 to prepend whitespace when needed (#18616)
  • [perflint] Fix PERF403 panic on attribute or subscription loop variable (#19042)
  • [pydocstyle] Fix D413 infinite loop for parenthesized docstring (#18930)
  • [pylint] Fix PLW0108 autofix introducing a syntax error when the lambda's body contains an assignment expression (#18678)
  • [refurb] Fix false positive on empty tuples (FURB168) (#19058)
  • [ruff] Allow more field calls from attrs (RUF009) (#19021)
  • [ruff] Fix syntax error introduced for an empty string followed by a u-prefixed string (UP025) (#18899)

Rule changes

  • [flake8-executable] Allow uvx in shebang line (EXE003) (#18967)
  • [pandas] Avoid flagging PD002 if pandas is not imported (#18963)
  • [pyupgrade] Avoid PEP-604 unions with typing.NamedTuple (UP007, UP045) (#18682)

Documentation

  • Document link between import-outside-top-level (PLC0415) and lint.flake8-tidy-imports.banned-module-level-imports (#18733)
  • Fix description of the format.skip-magic-trailing-comma example (#19095)
  • [airflow] Make AIR302 example error out-of-the-box (#18988)
  • [airflow] Make AIR312 example error out-of-the-box (#18989)
  • [flake8-annotations] Make ANN401 example error out-of-the-box (#18974)
  • [flake8-async] Make ASYNC100 example error out-of-the-box (#18993)
  • [flake8-async] Make ASYNC105 example error out-of-the-box (#19002)
  • [flake8-async] Make ASYNC110 example error out-of-the-box (#18975)
  • [flake8-async] Make ASYNC210 example error out-of-the-box (#18977)
  • [flake8-async] Make ASYNC220, ASYNC221, and ASYNC222 examples error out-of-the-box (#18978)
  • [flake8-async] Make ASYNC251 example error out-of-the-box (#18990)
  • [flake8-bandit] Make S201 example error out-of-the-box (#19017)
  • [flake8-bandit] Make S604 and S609 examples error out-of-the-box (#19049)
  • [flake8-bugbear] Make B028 example error out-of-the-box (#19054)
  • [flake8-bugbear] Make B911 example error out-of-the-box (#19051)
  • [flake8-datetimez] Make DTZ011 example error out-of-the-box (#19055)
  • [flake8-datetimez] Make DTZ901 example error out-of-the-box (#19056)
  • [flake8-pyi] Make PYI032 example error out-of-the-box (#19061)
  • [flake8-pyi] Make example error out-of-the-box (PYI014, PYI015) (#19097)
  • [flake8-pyi] Make example error out-of-the-box (PYI042) (#19101)
  • [flake8-pyi] Make example error out-of-the-box (PYI059) (#19080)
  • [flake8-pyi] Make example error out-of-the-box (PYI062) (#19079)
  • [flake8-pytest-style] Make example error out-of-the-box (PT023) (#19104)
  • [flake8-pytest-style] Make example error out-of-the-box (PT030) (#19105)
  • [flake8-quotes] Make example error out-of-the-box (Q003) (#19106)

... (truncated)

Commits
  • 9bee837 Bump 0.12.2 (#19126)
  • 1c6717b Filter private symbols from stubs if they are internal types (#19121)
  • 1b813cd Fix description of the format.skip-magic-trailing-comma example (#19095)
  • b00f68a [ruff] Allow more field calls from attrs (RUF009) (#19021)
  • 710c60f [flake8-pytest-style] Make example error out-of-the-box (PT023) (#19104)
  • 811e25d [flake8-pytest-style] Make example error out-of-the-box (PT030) (#19105)
  • b78af2d [flake8-quotes] Make example error out-of-the-box (Q003) (#19106)
  • 4f36f06 Document link between import-outside-top-level (PLC0415) and `lint.flake8-t...
  • 2589a29 [flake8-simplify] Make example error out-of-the-box (SIM113) (#19109)
  • 26bb8f7 [flake8-simplify] Make example error out-of-the-box (SIM401) (#19110)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ruff&package-manager=pip&previous-version=0.12.1&new-version=0.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index ca0850353d..305934c624 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2448,30 +2448,30 @@ files = [ [[package]] name = "ruff" -version = "0.12.1" +version = "0.12.2" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.12.1-py3-none-linux_armv6l.whl", hash = "sha256:6013a46d865111e2edb71ad692fbb8262e6c172587a57c0669332a449384a36b"}, - {file = "ruff-0.12.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b3f75a19e03a4b0757d1412edb7f27cffb0c700365e9d6b60bc1b68d35bc89e0"}, - {file = "ruff-0.12.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9a256522893cb7e92bb1e1153283927f842dea2e48619c803243dccc8437b8be"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:069052605fe74c765a5b4272eb89880e0ff7a31e6c0dbf8767203c1fbd31c7ff"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a684f125a4fec2d5a6501a466be3841113ba6847827be4573fddf8308b83477d"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdecdef753bf1e95797593007569d8e1697a54fca843d78f6862f7dc279e23bd"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:70d52a058c0e7b88b602f575d23596e89bd7d8196437a4148381a3f73fcd5010"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84d0a69d1e8d716dfeab22d8d5e7c786b73f2106429a933cee51d7b09f861d4e"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cc32e863adcf9e71690248607ccdf25252eeeab5193768e6873b901fd441fed"}, - {file = "ruff-0.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fd49a4619f90d5afc65cf42e07b6ae98bb454fd5029d03b306bd9e2273d44cc"}, - {file = "ruff-0.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ed5af6aaaea20710e77698e2055b9ff9b3494891e1b24d26c07055459bb717e9"}, - {file = "ruff-0.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:801d626de15e6bf988fbe7ce59b303a914ff9c616d5866f8c79eb5012720ae13"}, - {file = "ruff-0.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2be9d32a147f98a1972c1e4df9a6956d612ca5f5578536814372113d09a27a6c"}, - {file = "ruff-0.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:49b7ce354eed2a322fbaea80168c902de9504e6e174fd501e9447cad0232f9e6"}, - {file = "ruff-0.12.1-py3-none-win32.whl", hash = "sha256:d973fa626d4c8267848755bd0414211a456e99e125dcab147f24daa9e991a245"}, - {file = "ruff-0.12.1-py3-none-win_amd64.whl", hash = "sha256:9e1123b1c033f77bd2590e4c1fe7e8ea72ef990a85d2484351d408224d603013"}, - {file = "ruff-0.12.1-py3-none-win_arm64.whl", hash = "sha256:78ad09a022c64c13cc6077707f036bab0fac8cd7088772dcd1e5be21c5002efc"}, - {file = "ruff-0.12.1.tar.gz", hash = "sha256:806bbc17f1104fd57451a98a58df35388ee3ab422e029e8f5cf30aa4af2c138c"}, + {file = "ruff-0.12.2-py3-none-linux_armv6l.whl", hash = "sha256:093ea2b221df1d2b8e7ad92fc6ffdca40a2cb10d8564477a987b44fd4008a7be"}, + {file = "ruff-0.12.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:09e4cf27cc10f96b1708100fa851e0daf21767e9709e1649175355280e0d950e"}, + {file = "ruff-0.12.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8ae64755b22f4ff85e9c52d1f82644abd0b6b6b6deedceb74bd71f35c24044cc"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eb3a6b2db4d6e2c77e682f0b988d4d61aff06860158fdb413118ca133d57922"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73448de992d05517170fc37169cbca857dfeaeaa8c2b9be494d7bcb0d36c8f4b"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8b94317cbc2ae4a2771af641739f933934b03555e51515e6e021c64441532d"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:45fc42c3bf1d30d2008023a0a9a0cfb06bf9835b147f11fe0679f21ae86d34b1"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce48f675c394c37e958bf229fb5c1e843e20945a6d962cf3ea20b7a107dcd9f4"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793d8859445ea47591272021a81391350205a4af65a9392401f418a95dfb75c9"}, + {file = "ruff-0.12.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6932323db80484dda89153da3d8e58164d01d6da86857c79f1961934354992da"}, + {file = "ruff-0.12.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6aa7e623a3a11538108f61e859ebf016c4f14a7e6e4eba1980190cacb57714ce"}, + {file = "ruff-0.12.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2a4a20aeed74671b2def096bdf2eac610c7d8ffcbf4fb0e627c06947a1d7078d"}, + {file = "ruff-0.12.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:71a4c550195612f486c9d1f2b045a600aeba851b298c667807ae933478fcef04"}, + {file = "ruff-0.12.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4987b8f4ceadf597c927beee65a5eaf994c6e2b631df963f86d8ad1bdea99342"}, + {file = "ruff-0.12.2-py3-none-win32.whl", hash = "sha256:369ffb69b70cd55b6c3fc453b9492d98aed98062db9fec828cdfd069555f5f1a"}, + {file = "ruff-0.12.2-py3-none-win_amd64.whl", hash = "sha256:dca8a3b6d6dc9810ed8f328d406516bf4d660c00caeaef36eb831cf4871b0639"}, + {file = "ruff-0.12.2-py3-none-win_arm64.whl", hash = "sha256:48d6c6bfb4761df68bc05ae630e24f506755e702d4fb08f08460be778c7ccb12"}, + {file = "ruff-0.12.2.tar.gz", hash = "sha256:d7b4f55cd6f325cb7621244f19c873c565a08aff5a4ba9c69aa7355f3f7afd3e"}, ] [[package]] @@ -3404,4 +3404,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "4f24cf959d33ebacbcbd791990d42e5fea2ab74bfad32cf2115d2aadd1c46d21" +content-hash = "6871453202e2ffc3c9ee5761481e0ef4ad99f08d9b8f24c6d6a64c2683683845" diff --git a/pyproject.toml b/pyproject.toml index be5bff31b6..5b2a306f8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -317,7 +317,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.12.1" +ruff = "0.12.2" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From 49e46b84de7b530914237622d0250cab2a6b3756 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:13:41 +0000 Subject: [PATCH 450/691] Bump types-psycopg2 from 2.9.21.20250318 to 2.9.21.20250516 (#18658) Bumps [types-psycopg2](https://github.com/typeshed-internal/stub_uploader) from 2.9.21.20250318 to 2.9.21.20250516.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-psycopg2&package-manager=pip&previous-version=2.9.21.20250318&new-version=2.9.21.20250516)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 305934c624..2bb9ab3586 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3034,14 +3034,14 @@ files = [ [[package]] name = "types-psycopg2" -version = "2.9.21.20250318" +version = "2.9.21.20250516" description = "Typing stubs for psycopg2" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_psycopg2-2.9.21.20250318-py3-none-any.whl", hash = "sha256:7296d111ad950bbd2fc979a1ab0572acae69047f922280e77db657c00d2c79c0"}, - {file = "types_psycopg2-2.9.21.20250318.tar.gz", hash = "sha256:eb6eac5bfb16adfd5f16b818918b9e26a40ede147e0f2bbffdf53a6ef7025a87"}, + {file = "types_psycopg2-2.9.21.20250516-py3-none-any.whl", hash = "sha256:2a9212d1e5e507017b31486ce8147634d06b85d652769d7a2d91d53cb4edbd41"}, + {file = "types_psycopg2-2.9.21.20250516.tar.gz", hash = "sha256:6721018279175cce10b9582202e2a2b4a0da667857ccf82a97691bdb5ecd610f"}, ] [[package]] From 616ada3dfe9688af59ebeef7a1a9dd63ebadc290 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 15:20:43 +0200 Subject: [PATCH 451/691] Bump tokio from 1.46.0 to 1.46.1 (#18667) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.46.0 to 1.46.1.
Release notes

Sourced from tokio's releases.

Tokio v1.46.1

1.46.1 (July 4th, 2025)

This release fixes incorrect spawn locations in runtime task hooks for tasks spawned using tokio::spawn rather than Runtime::spawn. This issue only effected the spawn location in TaskMeta::spawned_at, and did not effect task locations in Tracing events.

Unstable

  • runtime: add TaskMeta::spawn_location tracking where a task was spawned (#7440)

#7440: tokio-rs/tokio#7440

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tokio&package-manager=cargo&previous-version=1.46.0&new-version=1.46.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24994225e0..671eec0579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,9 +1571,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.0" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", From 57220706b26b9c1d7591c982320b1c4d6866f287 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:24:30 +0000 Subject: [PATCH 452/691] Bump typing-extensions from 4.14.0 to 4.14.1 (#18668) Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.14.0 to 4.14.1.
Release notes

Sourced from typing-extensions's releases.

4.14.1

Release 4.14.1 (July 4, 2025)

  • Fix usage of typing_extensions.TypedDict nested inside other types (e.g., typing.Type[typing_extensions.TypedDict]). This is not allowed by the type system but worked on older versions, so we maintain support.
Changelog

Sourced from typing-extensions's changelog.

Release 4.14.1 (July 4, 2025)

  • Fix usage of typing_extensions.TypedDict nested inside other types (e.g., typing.Type[typing_extensions.TypedDict]). This is not allowed by the type system but worked on older versions, so we maintain support.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=typing-extensions&package-manager=pip&previous-version=4.14.0&new-version=4.14.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2bb9ab3586..d72821432c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3101,14 +3101,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.14.0" +version = "4.14.1" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, - {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, + {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, + {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, ] [[package]] From 9b86458900be1628211896bcdee2451917862e7d Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 8 Jul 2025 12:53:12 +0200 Subject: [PATCH 453/691] Speed up the building of Docker images using ARM CI workers (#18620) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This splits the building of docker images in 2 jobs, one for each platform, using the native ARM runners for arm64. The tricky part here is to get back a nice multi-arch manifest. Previously, you'd do that by pushing each platform image in two distinct tags, then referencing them in a multi-arch manifest. Nowadays, it's possible to push images by their digest only, then creating the manifest for those pushed digests separately This is inspired by the Docker docs on how to distribute multi-platform image builds: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners `ghcr.io/element-hq/synapse:sha-c733dd6` is an example image that got built by this workflow (there is a temporary sha-* tag on workflow_dispatch runs to help trying out the workflow) I also had to make sure we sign the manifests correctly: ``` $ cosign verify --certificate-oidc-issuer https://token.actions.githubusercontent.com --certificate-identity-regexp 'https://github.com/element-hq/synapse/.github/workflows/docker.yml@.*' ghcr.io/element-hq/synapse:sha-c733dd6 Verification for ghcr.io/element-hq/synapse:sha-c733dd6 -- The following checks were performed on each of these signatures: - The cosign claims were validated - Existence of the claims in the transparency log was verified offline - The code-signing certificate was verified using trusted certificate authority certificates ``` And the numbers aaaaare 🥁 - [before](https://github.com/element-hq/synapse/actions/runs/16118229296/job/45477093703): 30 minutes - [after](https://github.com/element-hq/synapse/actions/runs/16021743575): 4 minutes --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- .github/workflows/docker.yml | 140 +++++++++++++++++------- .github/workflows/release-artifacts.yml | 7 +- changelog.d/18620.misc | 1 + 3 files changed, 102 insertions(+), 46 deletions(-) create mode 100644 changelog.d/18620.misc diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 67d7df0b23..9ed3aa4a5b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ name: Build docker images on: push: tags: ["v*"] - branches: [ master, main, develop ] + branches: [master, main, develop] workflow_dispatch: permissions: @@ -14,24 +14,22 @@ permissions: id-token: write # needed for signing the images with GitHub OIDC Token jobs: build: - runs-on: ubuntu-22.04 + name: Build and push image for ${{ matrix.platform }} + runs-on: ${{ matrix.runs_on }} + strategy: + matrix: + include: + - platform: linux/amd64 + runs_on: ubuntu-24.04 + suffix: linux-amd64 + - platform: linux/arm64 + runs_on: ubuntu-24.04-arm + suffix: linux-arm64 steps: - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - with: - platforms: arm64 - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Inspect builder - run: docker buildx inspect - - - name: Install Cosign - uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 - - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -55,13 +53,79 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Calculate docker image tag - id: set-tag - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + - name: Build and push by digest + id: build + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - images: | + push: true + labels: | + gitsha1=${{ github.sha }} + org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }} + tags: | docker.io/matrixdotorg/synapse ghcr.io/element-hq/synapse + file: "docker/Dockerfile" + platforms: ${{ matrix.platform }} + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.suffix }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Push merged images to ${{ matrix.repository }} + runs-on: ubuntu-latest + strategy: + matrix: + repository: + - docker.io/matrixdotorg/synapse + - ghcr.io/element-hq/synapse + + needs: + - build + steps: + - name: Download digests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - name: Log in to DockerHub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + if: ${{ startsWith(matrix.repository, 'docker.io') }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + if: ${{ startsWith(matrix.repository, 'ghcr.io') }} + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Install Cosign + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 + + - name: Calculate docker image tag + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: ${{ matrix.repository }} flavor: | latest=false tags: | @@ -69,31 +133,23 @@ jobs: type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }} type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=pep440,pattern={{raw}} + type=sha - - name: Build and push all platforms - id: build-and-push - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - push: true - labels: | - gitsha1=${{ github.sha }} - org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }} - tags: "${{ steps.set-tag.outputs.tags }}" - file: "docker/Dockerfile" - platforms: linux/amd64,linux/arm64 - - # arm64 builds OOM without the git fetch setting. c.f. - # https://github.com/rust-lang/cargo/issues/10583 - build-args: | - CARGO_NET_GIT_FETCH_WITH_CLI=true - - - name: Sign the images with GitHub OIDC Token + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests env: - DIGEST: ${{ steps.build-and-push.outputs.digest }} - TAGS: ${{ steps.set-tag.outputs.tags }} + REPOSITORY: ${{ matrix.repository }} run: | - images="" - for tag in ${TAGS}; do - images+="${tag}@${DIGEST} " + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "$REPOSITORY@sha256:%s " *) + + - name: Sign each manifest + env: + REPOSITORY: ${{ matrix.repository }} + run: | + DIGESTS="" + for TAG in $(echo "$DOCKER_METADATA_OUTPUT_JSON" | jq -r '.tags[]'); do + DIGEST="$(docker buildx imagetools inspect $TAG --format '{{json .Manifest}}' | jq -r '.digest')" + DIGESTS="$DIGESTS $REPOSITORY@$DIGEST" done - cosign sign --yes ${images} + cosign sign --yes $DIGESTS diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index e80f4f4d71..89474dd4da 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.x' + python-version: "3.x" - id: set-distros run: | # if we're running from a tag, get the full list of distros; otherwise just use debian:sid @@ -76,7 +76,7 @@ jobs: - name: Set up python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.x' + python-version: "3.x" - name: Build the packages # see https://github.com/docker/build-push-action/issues/252 @@ -179,7 +179,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.10' + python-version: "3.10" - run: pip install build @@ -191,7 +191,6 @@ jobs: name: Sdist path: dist/*.tar.gz - # if it's a tag, create a release and attach the artifacts to it attach-assets: name: "Attach assets to release" diff --git a/changelog.d/18620.misc b/changelog.d/18620.misc new file mode 100644 index 0000000000..fa23af2e11 --- /dev/null +++ b/changelog.d/18620.misc @@ -0,0 +1 @@ +Speed up the building of Docker images in CI. From b8ad9bf1b7271eaa8068faa8f801dfc836f3030a Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:57:23 +0100 Subject: [PATCH 454/691] Add `.zed/` to `.gitignore` (#18623) --- .gitignore | 1 + changelog.d/18623.misc | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/18623.misc diff --git a/.gitignore b/.gitignore index a89f149ec1..e333f2320b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ __pycache__/ /.idea/ /.ropeproject/ /.vscode/ +/.zed/ # build products !/.coveragerc diff --git a/changelog.d/18623.misc b/changelog.d/18623.misc new file mode 100644 index 0000000000..3e973b32ec --- /dev/null +++ b/changelog.d/18623.misc @@ -0,0 +1 @@ +Add `.zed/` directory to `.gitignore`. \ No newline at end of file From f8a7872ddbbb56763d3d59c5471da13f64801f5c Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 8 Jul 2025 13:30:36 +0200 Subject: [PATCH 455/691] Use native arm runners for building wheels (#18618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This takes down the CI time to build wheels from 50 minutes to <10 minutes. **It also fixes macOS ARM builds, and includes more ARM builds in general** (we were ignoring pypy and musl before). This doesn't cost much for us to do this, reasons for not doing this is 1. space on PyPI and 2. keeping them 'officially' supported? This is the list of wheels this built (`+` are the ones added): ```diff matrix_synapse-1.133.0-cp39-abi3-macosx_10_9_x86_64.whl + matrix_synapse-1.133.0-cp39-abi3-macosx_11_0_arm64.whl matrix_synapse-1.133.0-cp39-abi3-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-cp39-abi3-manylinux_2_28_x86_64.whl + matrix_synapse-1.133.0-cp39-abi3-musllinux_1_2_aarch64.whl matrix_synapse-1.133.0-cp39-abi3-musllinux_1_2_x86_64.whl matrix_synapse-1.133.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl + matrix_synapse-1.133.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl + matrix_synapse-1.133.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl matrix_synapse-1.133.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl + matrix_synapse-1.133.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl + matrix_synapse-1.133.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl matrix_synapse-1.133.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl ``` And the numbers aaaaare 🥁 - [before](https://github.com/element-hq/synapse/actions/runs/16072488018): 54 minutes - [after](https://github.com/element-hq/synapse/actions/runs/16004034949?pr=18618): 10 minutes **Revert [e43b0f9](https://github.com/element-hq/synapse/pull/18618/commits/e43b0f9bd18f6d4ec9f9e9d992ecdcceaaf10982) before merging** --- .github/workflows/release-artifacts.yml | 33 +++++++++---------------- changelog.d/18618.misc | 1 + pyproject.toml | 7 ++---- 3 files changed, 14 insertions(+), 27 deletions(-) create mode 100644 changelog.d/18618.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 89474dd4da..7bac012845 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -107,12 +107,15 @@ jobs: path: debs/* build-wheels: - name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04, macos-13] - arch: [x86_64, aarch64] + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + - macos-13 # This uses x86-64 + - macos-14 # This uses arm64 # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: @@ -122,12 +125,11 @@ jobs: # Don't build macos wheels on PR CI. - is_pr: true os: "macos-13" - # Don't build aarch64 wheels on mac. - - os: "macos-13" - arch: aarch64 + - is_pr: true + os: "macos-14" # Don't build aarch64 wheels on PR CI. - is_pr: true - arch: aarch64 + os: "ubuntu-24.04-arm" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -141,19 +143,9 @@ jobs: - name: Install cibuildwheel run: python -m pip install cibuildwheel==3.0.0 - - name: Set up QEMU to emulate aarch64 - if: matrix.arch == 'aarch64' - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - with: - platforms: arm64 - - - name: Build aarch64 wheels - if: matrix.arch == 'aarch64' - run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV - - name: Only build a single wheel on PR if: startsWith(github.ref, 'refs/pull/') - run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV + run: echo "CIBW_BUILD="cp39-manylinux_*"" >> $GITHUB_ENV - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse @@ -161,13 +153,10 @@ jobs: # Skip testing for platforms which various libraries don't have wheels # for, and so need extra build deps. CIBW_TEST_SKIP: pp3*-* *i686* *musl* - # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583 - CARGO_NET_GIT_FETCH_WITH_CLI: true - CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Wheel-${{ matrix.os }}-${{ matrix.arch }} + name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl build-sdist: diff --git a/changelog.d/18618.misc b/changelog.d/18618.misc new file mode 100644 index 0000000000..bd9aaf305d --- /dev/null +++ b/changelog.d/18618.misc @@ -0,0 +1 @@ +Speed up the building of arm-based wheels in CI. diff --git a/pyproject.toml b/pyproject.toml index 5b2a306f8d..a5977700ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -375,13 +375,10 @@ build-backend = "poetry.core.masonry.api" # Skip unsupported platforms (by us or by Rust). # See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets. # We skip: -# - CPython 3.6, 3.7 and 3.8: EOLed -# - PyPy 3.7 and 3.8: we only support Python 3.9+ +# - CPython and PyPy 3.8: EOLed # - musllinux i686: excluded to reduce number of wheels we build. # c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677 -# - PyPy on Aarch64 and musllinux on aarch64: too slow to build. -# c.f. https://github.com/matrix-org/synapse/pull/14259 -skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" +skip = "cp38* pp38* *-musllinux_i686" # Enable non-default builds. # "pypy" used to be included by default up until cibuildwheel 3. enable = "pypy" From 5c2765bd7aeb1a7cfcc2698a7f54db5db02d16ff Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jul 2025 11:31:01 +0100 Subject: [PATCH 456/691] 1.134.0rc1 --- CHANGES.md | 81 +++++++++++++++++++++++++++++++ changelog.d/18070.feature | 1 - changelog.d/18196.feature | 1 - changelog.d/18241.feature | 1 - changelog.d/18418.removal | 1 - changelog.d/18509.bugfix | 1 - changelog.d/18519.doc | 1 - changelog.d/18553.misc | 1 - changelog.d/18573.misc | 1 - changelog.d/18575.misc | 1 - changelog.d/18576.misc | 1 - changelog.d/18582.bugfix | 1 - changelog.d/18593.bugfix | 1 - changelog.d/18594.bugfix | 1 - changelog.d/18595.misc | 1 - changelog.d/18596.misc | 1 - changelog.d/18600.misc | 1 - changelog.d/18602.misc | 1 - changelog.d/18605.bugfix | 1 - changelog.d/18618.misc | 1 - changelog.d/18620.misc | 1 - changelog.d/18623.misc | 1 - changelog.d/18625.misc | 1 - changelog.d/18646.misc | 1 - debian/changelog | 6 +++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 27 files changed, 89 insertions(+), 25 deletions(-) delete mode 100644 changelog.d/18070.feature delete mode 100644 changelog.d/18196.feature delete mode 100644 changelog.d/18241.feature delete mode 100644 changelog.d/18418.removal delete mode 100644 changelog.d/18509.bugfix delete mode 100644 changelog.d/18519.doc delete mode 100644 changelog.d/18553.misc delete mode 100644 changelog.d/18573.misc delete mode 100644 changelog.d/18575.misc delete mode 100644 changelog.d/18576.misc delete mode 100644 changelog.d/18582.bugfix delete mode 100644 changelog.d/18593.bugfix delete mode 100644 changelog.d/18594.bugfix delete mode 100644 changelog.d/18595.misc delete mode 100644 changelog.d/18596.misc delete mode 100644 changelog.d/18600.misc delete mode 100644 changelog.d/18602.misc delete mode 100644 changelog.d/18605.bugfix delete mode 100644 changelog.d/18618.misc delete mode 100644 changelog.d/18620.misc delete mode 100644 changelog.d/18623.misc delete mode 100644 changelog.d/18625.misc delete mode 100644 changelog.d/18646.misc diff --git a/CHANGES.md b/CHANGES.md index f3e541563f..8948cf1e29 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,84 @@ +# Synapse 1.134.0rc1 (2025-07-09) + +### Features + +- Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): `via` query param for hierarchy endpoint. Contributed by Krishan (@kfiven). ([\#18070](https://github.com/element-hq/synapse/issues/18070)) +- Add `forget_forced_upon_leave` capability as per [MSC4267](https://github.com/matrix-org/matrix-spec-proposals/pull/4267). ([\#18196](https://github.com/element-hq/synapse/issues/18196)) +- Add `federated_user_may_invite` spam checker callback which receives the entire invite event. Contributed by @tulir @ Beeper. ([\#18241](https://github.com/element-hq/synapse/issues/18241)) + +### Bugfixes + +- Fix `KeyError` on background updates when using split main/state databases. ([\#18509](https://github.com/element-hq/synapse/issues/18509)) +- Improve performance of device deletion by adding missing index. ([\#18582](https://github.com/element-hq/synapse/issues/18582)) +- Fix `avatar_url` and `displayname` being sent on federation profile queries when they are not set. ([\#18593](https://github.com/element-hq/synapse/issues/18593)) +- Respond with 401 & `M_USER_LOCKED` when a locked user calls `POST /login`, as per the spec. ([\#18594](https://github.com/element-hq/synapse/issues/18594)) +- Ensure policy servers are not asked to scan policy server change events, allowing rooms to disable the use of a policy server while the policy server is down. ([\#18605](https://github.com/element-hq/synapse/issues/18605)) + +### Improved Documentation + +- Fix documentation of the Delete Room Admin API's status field. ([\#18519](https://github.com/element-hq/synapse/issues/18519)) + +### Deprecations and Removals + +- Stop adding the "origin" field to newly-created events (PDUs). ([\#18418](https://github.com/element-hq/synapse/issues/18418)) + +### Internal Changes + +- Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. ([\#18553](https://github.com/element-hq/synapse/issues/18553), [\#18646](https://github.com/element-hq/synapse/issues/18646)) +- Improve docstring on `simple_upsert_many`. ([\#18573](https://github.com/element-hq/synapse/issues/18573)) +- Raise poetry-core version cap to 2.1.3. ([\#18575](https://github.com/element-hq/synapse/issues/18575)) +- Raise setuptools_rust version cap to 1.11.1. ([\#18576](https://github.com/element-hq/synapse/issues/18576)) +- Better handling of ratelimited requests. ([\#18595](https://github.com/element-hq/synapse/issues/18595), [\#18600](https://github.com/element-hq/synapse/issues/18600)) +- Update to Rust 1.87.0 in CI, and bump the pinned commit of the `dtolnay/rust-toolchain` GitHub Action to `b3b07ba8b418998c39fb20f53e8b695cdcc8de1b`. ([\#18596](https://github.com/element-hq/synapse/issues/18596)) +- Speed up bulk device deletion. ([\#18602](https://github.com/element-hq/synapse/issues/18602)) +- Speed up the building of arm-based wheels in CI. ([\#18618](https://github.com/element-hq/synapse/issues/18618)) +- Speed up the building of Docker images in CI. ([\#18620](https://github.com/element-hq/synapse/issues/18620)) +- Add `.zed/` directory to `.gitignore`. ([\#18623](https://github.com/element-hq/synapse/issues/18623)) +- Log the room ID we're purging state for. ([\#18625](https://github.com/element-hq/synapse/issues/18625)) + + + +### Updates to locked dependencies + +* Bump Swatinem/rust-cache from 2.7.8 to 2.8.0. ([\#18612](https://github.com/element-hq/synapse/issues/18612)) +* Bump attrs from 24.2.0 to 25.3.0. ([\#18649](https://github.com/element-hq/synapse/issues/18649)) +* Bump authlib from 1.5.2 to 1.6.0. ([\#18642](https://github.com/element-hq/synapse/issues/18642)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18589](https://github.com/element-hq/synapse/issues/18589)) +* Bump base64 from 0.21.7 to 0.22.1. ([\#18629](https://github.com/element-hq/synapse/issues/18629)) +* Bump docker/build-push-action from 6.17.0 to 6.18.0. ([\#18497](https://github.com/element-hq/synapse/issues/18497)) +* Bump docker/setup-buildx-action from 3.10.0 to 3.11.1. ([\#18587](https://github.com/element-hq/synapse/issues/18587)) +* Bump hiredis from 3.1.0 to 3.2.1. ([\#18638](https://github.com/element-hq/synapse/issues/18638)) +* Bump ijson from 3.3.0 to 3.4.0. ([\#18650](https://github.com/element-hq/synapse/issues/18650)) +* Bump jsonschema from 4.23.0 to 4.24.0. ([\#18630](https://github.com/element-hq/synapse/issues/18630)) +* Bump msgpack from 1.1.0 to 1.1.1. ([\#18651](https://github.com/element-hq/synapse/issues/18651)) +* Bump mypy-zope from 1.0.11 to 1.0.12. ([\#18640](https://github.com/element-hq/synapse/issues/18640)) +* Bump phonenumbers from 9.0.2 to 9.0.8. ([\#18652](https://github.com/element-hq/synapse/issues/18652)) +* Bump pillow from 11.2.1 to 11.3.0. ([\#18624](https://github.com/element-hq/synapse/issues/18624)) +* Bump prometheus-client from 0.21.0 to 0.22.1. ([\#18609](https://github.com/element-hq/synapse/issues/18609)) +* Bump pyasn1-modules from 0.4.1 to 0.4.2. ([\#18495](https://github.com/element-hq/synapse/issues/18495)) +* Bump pydantic from 2.11.4 to 2.11.7. ([\#18639](https://github.com/element-hq/synapse/issues/18639)) +* Bump reqwest from 0.12.15 to 0.12.20. ([\#18590](https://github.com/element-hq/synapse/issues/18590)) +* Bump reqwest from 0.12.20 to 0.12.22. ([\#18627](https://github.com/element-hq/synapse/issues/18627)) +* Bump ruff from 0.11.11 to 0.12.1. ([\#18645](https://github.com/element-hq/synapse/issues/18645)) +* Bump ruff from 0.12.1 to 0.12.2. ([\#18657](https://github.com/element-hq/synapse/issues/18657)) +* Bump sentry-sdk from 2.22.0 to 2.32.0. ([\#18633](https://github.com/element-hq/synapse/issues/18633)) +* Bump setuptools-rust from 1.10.2 to 1.11.1. ([\#18655](https://github.com/element-hq/synapse/issues/18655)) +* Bump sigstore/cosign-installer from 3.8.2 to 3.9.0. ([\#18588](https://github.com/element-hq/synapse/issues/18588)) +* Bump sigstore/cosign-installer from 3.9.0 to 3.9.1. ([\#18608](https://github.com/element-hq/synapse/issues/18608)) +* Bump stefanzweifel/git-auto-commit-action from 5.2.0 to 6.0.1. ([\#18607](https://github.com/element-hq/synapse/issues/18607)) +* Bump tokio from 1.45.1 to 1.46.0. ([\#18628](https://github.com/element-hq/synapse/issues/18628)) +* Bump tokio from 1.46.0 to 1.46.1. ([\#18667](https://github.com/element-hq/synapse/issues/18667)) +* Bump treq from 24.9.1 to 25.5.0. ([\#18610](https://github.com/element-hq/synapse/issues/18610)) +* Bump types-bleach from 6.2.0.20241123 to 6.2.0.20250514. ([\#18634](https://github.com/element-hq/synapse/issues/18634)) +* Bump types-jsonschema from 4.23.0.20250516 to 4.24.0.20250528. ([\#18611](https://github.com/element-hq/synapse/issues/18611)) +* Bump types-opentracing from 2.4.10.6 to 2.4.10.20250622. ([\#18586](https://github.com/element-hq/synapse/issues/18586)) +* Bump types-psycopg2 from 2.9.21.20250318 to 2.9.21.20250516. ([\#18658](https://github.com/element-hq/synapse/issues/18658)) +* Bump types-pyyaml from 6.0.12.20241230 to 6.0.12.20250516. ([\#18643](https://github.com/element-hq/synapse/issues/18643)) +* Bump types-setuptools from 75.2.0.20241019 to 80.9.0.20250529. ([\#18644](https://github.com/element-hq/synapse/issues/18644)) +* Bump typing-extensions from 4.12.2 to 4.14.0. ([\#18654](https://github.com/element-hq/synapse/issues/18654)) +* Bump typing-extensions from 4.14.0 to 4.14.1. ([\#18668](https://github.com/element-hq/synapse/issues/18668)) +* Bump urllib3 from 2.2.2 to 2.5.0. ([\#18572](https://github.com/element-hq/synapse/issues/18572)) + # Synapse 1.133.0 (2025-07-01) Pre-built wheels are now built using the [manylinux_2_28](https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based) base, which is expected to be compatible with distros using glibc 2.28 or later, including: diff --git a/changelog.d/18070.feature b/changelog.d/18070.feature deleted file mode 100644 index ecf468f0b6..0000000000 --- a/changelog.d/18070.feature +++ /dev/null @@ -1 +0,0 @@ -Support for [MSC4235](https://github.com/matrix-org/matrix-spec-proposals/pull/4235): via query param for hierarchy endpoint. Contributed by Krishan (@kfiven). diff --git a/changelog.d/18196.feature b/changelog.d/18196.feature deleted file mode 100644 index 6f4a67b519..0000000000 --- a/changelog.d/18196.feature +++ /dev/null @@ -1 +0,0 @@ -Add `forget_forced_upon_leave` capability as per [MSC4267](https://github.com/matrix-org/matrix-spec-proposals/pull/4267). diff --git a/changelog.d/18241.feature b/changelog.d/18241.feature deleted file mode 100644 index 01760c9d67..0000000000 --- a/changelog.d/18241.feature +++ /dev/null @@ -1 +0,0 @@ -Add `federated_user_may_invite` spam checker callback which receives the entire invite event. Contributed by @tulir @ Beeper. diff --git a/changelog.d/18418.removal b/changelog.d/18418.removal deleted file mode 100644 index ff1d081d8f..0000000000 --- a/changelog.d/18418.removal +++ /dev/null @@ -1 +0,0 @@ -Stop adding the "origin" field to newly-created events (PDUs). \ No newline at end of file diff --git a/changelog.d/18509.bugfix b/changelog.d/18509.bugfix deleted file mode 100644 index 2c821d0395..0000000000 --- a/changelog.d/18509.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `KeyError` on background updates when using split main/state databases. diff --git a/changelog.d/18519.doc b/changelog.d/18519.doc deleted file mode 100644 index b6fde5f42b..0000000000 --- a/changelog.d/18519.doc +++ /dev/null @@ -1 +0,0 @@ -Fix documentation of the Delete Room Admin API's status field. \ No newline at end of file diff --git a/changelog.d/18553.misc b/changelog.d/18553.misc deleted file mode 100644 index bb143aacfc..0000000000 --- a/changelog.d/18553.misc +++ /dev/null @@ -1 +0,0 @@ -Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file diff --git a/changelog.d/18573.misc b/changelog.d/18573.misc deleted file mode 100644 index 373852638f..0000000000 --- a/changelog.d/18573.misc +++ /dev/null @@ -1 +0,0 @@ -Improve docstring on `simple_upsert_many`. \ No newline at end of file diff --git a/changelog.d/18575.misc b/changelog.d/18575.misc deleted file mode 100644 index 0033f27df8..0000000000 --- a/changelog.d/18575.misc +++ /dev/null @@ -1 +0,0 @@ -Raise poetry-core version cap to 2.1.3. diff --git a/changelog.d/18576.misc b/changelog.d/18576.misc deleted file mode 100644 index 46cc78ec2d..0000000000 --- a/changelog.d/18576.misc +++ /dev/null @@ -1 +0,0 @@ -Raise setuptools_rust version cap to 1.11.1. diff --git a/changelog.d/18582.bugfix b/changelog.d/18582.bugfix deleted file mode 100644 index 7091bcc997..0000000000 --- a/changelog.d/18582.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve performance of device deletion by adding missing index. \ No newline at end of file diff --git a/changelog.d/18593.bugfix b/changelog.d/18593.bugfix deleted file mode 100644 index 9079b9534a..0000000000 --- a/changelog.d/18593.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `avatar_url` and `displayname` being sent on federation profile queries when they are not set. \ No newline at end of file diff --git a/changelog.d/18594.bugfix b/changelog.d/18594.bugfix deleted file mode 100644 index 3293a731f2..0000000000 --- a/changelog.d/18594.bugfix +++ /dev/null @@ -1 +0,0 @@ -Respond with 401 & `M_USER_LOCKED` when a locked user calls `POST /login`, as per the spec. diff --git a/changelog.d/18595.misc b/changelog.d/18595.misc deleted file mode 100644 index b1a39b43f7..0000000000 --- a/changelog.d/18595.misc +++ /dev/null @@ -1 +0,0 @@ -Better handling of ratelimited requests. diff --git a/changelog.d/18596.misc b/changelog.d/18596.misc deleted file mode 100644 index ce7bfdd129..0000000000 --- a/changelog.d/18596.misc +++ /dev/null @@ -1 +0,0 @@ -Update to Rust 1.87.0 in CI, and bump the pinned commit of the `dtolnay/rust-toolchain` GitHub Action to `b3b07ba8b418998c39fb20f53e8b695cdcc8de1b`. \ No newline at end of file diff --git a/changelog.d/18600.misc b/changelog.d/18600.misc deleted file mode 100644 index b1a39b43f7..0000000000 --- a/changelog.d/18600.misc +++ /dev/null @@ -1 +0,0 @@ -Better handling of ratelimited requests. diff --git a/changelog.d/18602.misc b/changelog.d/18602.misc deleted file mode 100644 index 637d842682..0000000000 --- a/changelog.d/18602.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up bulk device deletion. diff --git a/changelog.d/18605.bugfix b/changelog.d/18605.bugfix deleted file mode 100644 index 516f09e502..0000000000 --- a/changelog.d/18605.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure policy servers are not asked to scan policy server change events, allowing rooms to disable the use of a policy server while the policy server is down. diff --git a/changelog.d/18618.misc b/changelog.d/18618.misc deleted file mode 100644 index bd9aaf305d..0000000000 --- a/changelog.d/18618.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up the building of arm-based wheels in CI. diff --git a/changelog.d/18620.misc b/changelog.d/18620.misc deleted file mode 100644 index fa23af2e11..0000000000 --- a/changelog.d/18620.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up the building of Docker images in CI. diff --git a/changelog.d/18623.misc b/changelog.d/18623.misc deleted file mode 100644 index 3e973b32ec..0000000000 --- a/changelog.d/18623.misc +++ /dev/null @@ -1 +0,0 @@ -Add `.zed/` directory to `.gitignore`. \ No newline at end of file diff --git a/changelog.d/18625.misc b/changelog.d/18625.misc deleted file mode 100644 index f2672864c2..0000000000 --- a/changelog.d/18625.misc +++ /dev/null @@ -1 +0,0 @@ -Log the room ID we're purging state for. diff --git a/changelog.d/18646.misc b/changelog.d/18646.misc deleted file mode 100644 index bb143aacfc..0000000000 --- a/changelog.d/18646.misc +++ /dev/null @@ -1 +0,0 @@ -Replace `PyICU` crate with equivalent `icu_segmenter` Rust crate. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 32070a32d0..5b1f1f339a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.134.0~rc1) stable; urgency=medium + + * New Synapse release 1.134.0rc1. + + -- Synapse Packaging team Wed, 09 Jul 2025 11:27:13 +0100 + matrix-synapse-py3 (1.133.0) stable; urgency=medium * New synapse release 1.133.0. diff --git a/pyproject.toml b/pyproject.toml index a5977700ab..79b2cf9690 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.133.0" +version = "1.134.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index e4dcd36c30..bcce9b6359 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.133/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.134/synapse-config.schema.json type: object properties: modules: From bf0370162f63136d3eebf8c3a9e6b71b83925bda Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Jul 2025 11:48:06 +0100 Subject: [PATCH 457/691] Speed up inserting into stream_positions (#18672) By ensuring we don't do a no-op `UPDATE`, as this causes new tuples to be written in postgres. --- changelog.d/18672.misc | 1 + synapse/storage/util/id_generators.py | 14 +++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) create mode 100644 changelog.d/18672.misc diff --git a/changelog.d/18672.misc b/changelog.d/18672.misc new file mode 100644 index 0000000000..8d7be17fbc --- /dev/null +++ b/changelog.d/18672.misc @@ -0,0 +1 @@ +Minor speed up of insertion into `stream_positions` table. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e8588f33cf..fe6b6579e6 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -794,20 +794,16 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # We upsert the value, ensuring on conflict that we always increase the # value (or decrease if stream goes backwards). - if isinstance(self._db.engine, PostgresEngine): - agg = "GREATEST" if self._positive else "LEAST" - else: - agg = "MAX" if self._positive else "MIN" + cmp = "<" if self._positive else ">" - sql = """ + sql = f""" INSERT INTO stream_positions (stream_name, instance_name, stream_id) VALUES (?, ?, ?) ON CONFLICT (stream_name, instance_name) DO UPDATE SET - stream_id = %(agg)s(stream_positions.stream_id, EXCLUDED.stream_id) - """ % { - "agg": agg, - } + stream_id = EXCLUDED.stream_id + WHERE stream_positions.stream_id {cmp} EXCLUDED.stream_id + """ pos = self.get_current_token_for_writer(self._instance_name) txn.execute(sql, (self._stream_name, self._instance_name, pos)) From e1b429d88e05312777ebed86cc8c5c4de944a5e8 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Wed, 9 Jul 2025 21:08:21 +0200 Subject: [PATCH 458/691] Add experimental support for MSC4277: Harmonizing the reporting endpoints (#18263) [MSC4277](https://github.com/matrix-org/matrix-spec-proposals/pull/4277): Harmonizing the reporting endpoints --- changelog.d/18263.feature | 1 + synapse/config/experimental.py | 6 ++++++ synapse/rest/client/reporting.py | 25 ++++++++++++++++------ tests/rest/client/test_reporting.py | 33 +++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18263.feature diff --git a/changelog.d/18263.feature b/changelog.d/18263.feature new file mode 100644 index 0000000000..da5830b654 --- /dev/null +++ b/changelog.d/18263.feature @@ -0,0 +1 @@ +Add experimental support for [MSC4277](https://github.com/matrix-org/matrix-spec-proposals/pull/4277). diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 1ae47d3c16..b14bc97ae7 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -561,6 +561,12 @@ class ExperimentalConfig(Config): # MSC4076: Add `disable_badge_count`` to pusher configuration self.msc4076_enabled: bool = experimental.get("msc4076_enabled", False) + # MSC4277: Harmonizing the reporting endpoints + # + # If enabled, ignore the score parameter and respond with HTTP 200 on + # reporting requests regardless of the subject's existence. + self.msc4277_enabled: bool = experimental.get("msc4277_enabled", False) + # MSC4235: Add `via` param to hierarchy endpoint self.msc4235_enabled: bool = experimental.get("msc4235_enabled", False) diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 484827d9f2..81faf38a7f 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -69,7 +69,10 @@ class ReportEventRestServlet(RestServlet): "Param 'reason' must be a string", Codes.BAD_JSON, ) - if type(body.get("score", 0)) is not int: # noqa: E721 + if ( + not self.hs.config.experimental.msc4277_enabled + and type(body.get("score", 0)) is not int + ): # noqa: E721 raise SynapseError( HTTPStatus.BAD_REQUEST, "Param 'score' must be an integer", @@ -85,10 +88,15 @@ class ReportEventRestServlet(RestServlet): event = None if event is None: - raise NotFoundError( - "Unable to report event: " - "it does not exist or you aren't able to see it." - ) + if self.hs.config.experimental.msc4277_enabled: + # Respond with 200 and no content regardless of whether the event + # exists to prevent enumeration attacks. + return 200, {} + else: + raise NotFoundError( + "Unable to report event: " + "it does not exist or you aren't able to see it." + ) await self.store.add_event_report( room_id=room_id, @@ -138,7 +146,12 @@ class ReportRoomRestServlet(RestServlet): room = await self.store.get_room(room_id) if room is None: - raise NotFoundError("Room does not exist") + if self.hs.config.experimental.msc4277_enabled: + # Respond with 200 and no content regardless of whether the room + # exists to prevent enumeration attacks. + return 200, {} + else: + raise NotFoundError("Room does not exist") await self.store.add_room_report( room_id=room_id, diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 9335d46eb5..37517e5c21 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -29,6 +29,7 @@ from synapse.types import JsonDict from synapse.util import Clock from tests import unittest +from tests.unittest import override_config class ReportEventTestCase(unittest.HomeserverTestCase): @@ -81,6 +82,11 @@ class ReportEventTestCase(unittest.HomeserverTestCase): data = {"reason": None, "score": None} self._assert_status(400, data) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_score_str(self) -> None: + data = {"score": "string"} + self._assert_status(200, data) + def test_cannot_report_nonexistent_event(self) -> None: """ Tests that we don't accept event reports for events which do not exist. @@ -98,6 +104,19 @@ class ReportEventTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_event_existence_hidden(self) -> None: + """ + Tests that the requester cannot infer the existence of an event. + """ + channel = self.make_request( + "POST", + f"rooms/{self.room_id}/report/$nonsenseeventid:test", + {"reason": "i am very sad"}, + access_token=self.other_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.result["body"]) + def test_cannot_report_event_if_not_in_room(self) -> None: """ Tests that we don't accept event reports for events that exist, but for which @@ -193,6 +212,20 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"experimental_features": {"msc4277_enabled": True}}) + def test_room_existence_hidden(self) -> None: + """ + Tests that the requester cannot infer the existence of a room. + """ + channel = self.make_request( + "POST", + "/_matrix/client/v3/rooms/!bloop:example.org/report", + {"reason": "i am very sad"}, + access_token=self.other_user_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.result["body"]) + def _assert_status(self, response_status: int, data: JsonDict) -> None: channel = self.make_request( "POST", From b9b8775db7fe4a46463bed1a2db3c33802db6419 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Wed, 9 Jul 2025 21:13:54 +0200 Subject: [PATCH 459/691] Add plain-text handling for rich-text topics as per MSC3765 (#18195) This implements https://github.com/matrix-org/matrix-spec-proposals/pull/3765 which is already merged and, therefore, can use stable identifiers. For `/publicRooms` and `/hierarchy`, the topic is read from the eponymous field of the `current_state_events` table. Rather than introduce further columns in this table, I changed the insertion / update logic to write the plain-text topic from the rich topic into the existing field. This will not take effect for existing rooms unless their topic is changed. However, existing rooms shouldn't have rich topics to begin with. Similarly, for server-side search, I changed the insertion logic of the `event_search` table to prefer the value from the rich topic. Again, existing events shouldn't have rich topics and, therefore, don't need to be migrated in the table. Spec doc: https://spec.matrix.org/v1.15/client-server-api/#mroomtopic Part of supporting Matrix v1.15: https://spec.matrix.org/v1.15/client-server-api/#mroomtopic Signed-off-by: Johannes Marbach Co-authored-by: Eric Eastwood --- changelog.d/18195.feature | 1 + synapse/api/constants.py | 12 +++ synapse/config/workers.py | 30 +----- synapse/handlers/room.py | 9 +- synapse/handlers/stats.py | 5 +- synapse/storage/databases/main/events.py | 6 +- synapse/storage/databases/main/search.py | 5 +- synapse/storage/databases/main/stats.py | 5 +- synapse/types/rest/__init__.py | 24 +---- synapse/util/events.py | 102 ++++++++++++++++++++ synapse/util/pydantic_models.py | 39 ++++++++ tests/rest/client/test_rooms.py | 53 ++++++++++ tests/util/test_events.py | 118 +++++++++++++++++++++++ 13 files changed, 356 insertions(+), 53 deletions(-) create mode 100644 changelog.d/18195.feature create mode 100644 synapse/util/pydantic_models.py create mode 100644 tests/util/test_events.py diff --git a/changelog.d/18195.feature b/changelog.d/18195.feature new file mode 100644 index 0000000000..7f7903bd71 --- /dev/null +++ b/changelog.d/18195.feature @@ -0,0 +1 @@ +Add plain-text handling for rich-text topics as per [MSC3765](https://github.com/matrix-org/matrix-spec-proposals/pull/3765). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index e36461486b..47b84b02eb 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -262,6 +262,11 @@ class EventContentFields: TOMBSTONE_SUCCESSOR_ROOM: Final = "replacement_room" + # Used in m.room.topic events. + TOPIC: Final = "topic" + M_TOPIC: Final = "m.topic" + M_TEXT: Final = "m.text" + class EventUnsignedContentFields: """Fields found inside the 'unsigned' data on events""" @@ -270,6 +275,13 @@ class EventUnsignedContentFields: MEMBERSHIP: Final = "membership" +class MTextFields: + """Fields found inside m.text content blocks.""" + + BODY: Final = "body" + MIMETYPE: Final = "mimetype" + + class RoomTypes: """Understood values of the room_type field of m.room.create events.""" diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 2dfeb47c2e..c486c81274 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -27,8 +27,6 @@ from typing import Any, Dict, List, Optional, Union import attr from synapse._pydantic_compat import ( - BaseModel, - Extra, StrictBool, StrictInt, StrictStr, @@ -47,6 +45,7 @@ from synapse.config.server import ( parse_listener_def, ) from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel _DEPRECATED_WORKER_DUTY_OPTION_USED = """ The '%s' configuration option is deprecated and will be removed in a future @@ -90,30 +89,7 @@ def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]: return obj -class ConfigModel(BaseModel): - """A custom version of Pydantic's BaseModel which - - - ignores unknown fields and - - does not allow fields to be overwritten after construction, - - but otherwise uses Pydantic's default behaviour. - - For now, ignore unknown fields. In the future, we could change this so that unknown - config values cause a ValidationError, provided the error messages are meaningful to - server operators. - - Subclassing in this way is recommended by - https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally - """ - - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False - - -class InstanceTcpLocationConfig(ConfigModel): +class InstanceTcpLocationConfig(ParseModel): """The host and port to talk to an instance via HTTP replication.""" host: StrictStr @@ -129,7 +105,7 @@ class InstanceTcpLocationConfig(ConfigModel): return f"{self.host}:{self.port}" -class InstanceUnixLocationConfig(ConfigModel): +class InstanceUnixLocationConfig(ParseModel): """The socket file to talk to an instance via HTTP replication.""" path: StrictStr diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0f004d02d3..b063e301e6 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -51,6 +51,7 @@ from synapse.api.constants import ( HistoryVisibility, JoinRules, Membership, + MTextFields, RoomCreationPreset, RoomEncryptionAlgorithms, RoomTypes, @@ -1303,7 +1304,13 @@ class RoomCreationHandler: topic = room_config["topic"] topic_event, topic_context = await create_event( EventTypes.Topic, - {"topic": topic}, + { + EventContentFields.TOPIC: topic, + EventContentFields.M_TOPIC: { + # The mimetype property defaults to `text/plain` if omitted. + EventContentFields.M_TEXT: [{MTextFields.BODY: topic}] + }, + }, True, ) events_to_send.append((topic_event, topic_context)) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 8f90c17060..aa33260809 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -36,6 +36,7 @@ from synapse.metrics import event_processing_positions from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import JsonDict +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -299,7 +300,9 @@ class StatsHandler: elif delta.event_type == EventTypes.Name: room_state["name"] = event_content.get("name") elif delta.event_type == EventTypes.Topic: - room_state["topic"] = event_content.get("topic") + room_state["topic"] = get_plain_text_topic_from_event_content( + event_content + ) elif delta.event_type == EventTypes.RoomAvatar: room_state["avatar"] = event_content.get("url") elif delta.event_type == EventTypes.CanonicalAlias: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index b7cc0433e7..b7fbfdc0ca 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -78,6 +78,7 @@ from synapse.types import ( from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter from synapse.util import json_encoder +from synapse.util.events import get_plain_text_topic_from_event_content from synapse.util.iterutils import batch_iter, sorted_topologically from synapse.util.stringutils import non_null_str_or_none @@ -3102,7 +3103,10 @@ class PersistEventsStore: def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("topic"), str): self.store_event_search_txn( - txn, event, "content.topic", event.content["topic"] + txn, + event, + "content.topic", + get_plain_text_topic_from_event_content(event.content) or "", ) def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase) -> None: diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 1d5c5e72ff..47dfdf64e5 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -49,6 +49,7 @@ from synapse.storage.database import ( from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import JsonDict +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -212,7 +213,9 @@ class SearchBackgroundUpdateStore(SearchWorkerStore): value = content["body"] elif etype == "m.room.topic": key = "content.topic" - value = content["topic"] + value = ( + get_plain_text_topic_from_event_content(content) or "", + ) elif etype == "m.room.name": key = "content.name" value = content["name"] diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index 79c49e7fd9..74830b7129 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -48,6 +48,7 @@ from synapse.storage.databases.main.events_worker import InvalidEventError from synapse.storage.databases.main.state_deltas import StateDeltasStore from synapse.types import JsonDict from synapse.util.caches.descriptors import cached +from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: from synapse.server import HomeServer @@ -611,7 +612,9 @@ class StatsStore(StateDeltasStore): elif event.type == EventTypes.Name: room_state["name"] = event.content.get("name") elif event.type == EventTypes.Topic: - room_state["topic"] = event.content.get("topic") + room_state["topic"] = get_plain_text_topic_from_event_content( + event.content + ) elif event.type == EventTypes.RoomAvatar: room_state["avatar"] = event.content.get("url") elif event.type == EventTypes.CanonicalAlias: diff --git a/synapse/types/rest/__init__.py b/synapse/types/rest/__init__.py index 183831e79a..a02836deee 100644 --- a/synapse/types/rest/__init__.py +++ b/synapse/types/rest/__init__.py @@ -18,26 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # -from synapse._pydantic_compat import BaseModel, Extra +from synapse.util.pydantic_models import ParseModel -class RequestBodyModel(BaseModel): - """A custom version of Pydantic's BaseModel which - - - ignores unknown fields and - - does not allow fields to be overwritten after construction, - - but otherwise uses Pydantic's default behaviour. - - Ignoring unknown fields is a useful default. It means that clients can provide - unstable field not known to the server without the request being refused outright. - - Subclassing in this way is recommended by - https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally - """ - - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False +class RequestBodyModel(ParseModel): + pass diff --git a/synapse/util/events.py b/synapse/util/events.py index ad9b946578..4808268702 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -13,6 +13,11 @@ # # +from typing import Any, List, Optional + +from synapse._pydantic_compat import Field, StrictStr, ValidationError, validator +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -27,3 +32,100 @@ def generate_fake_event_id() -> str: A string intended to look like an event ID, but with no actual meaning. """ return "$" + random_string(43) + + +class MTextRepresentation(ParseModel): + """ + See `TextualRepresentation` in the Matrix specification. + """ + + body: StrictStr + mimetype: Optional[StrictStr] + + +class MTopic(ParseModel): + """ + `m.room.topic` -> `content` -> `m.topic` + + Textual representation of the room topic in different mimetypes. Added in Matrix v1.15. + + See `TopicContentBlock` in the Matrix specification. + """ + + m_text: Optional[List[MTextRepresentation]] = Field(alias="m.text") + """ + An ordered array of textual representations in different mimetypes. + """ + + # Because "Receivers SHOULD use the first representation in the array that they + # understand.", we ignore invalid representations in the `m.text` field and use + # what we can. + @validator("m_text", pre=True) + def ignore_invalid_representations( + cls, m_text: Any + ) -> Optional[List[MTextRepresentation]]: + if not isinstance(m_text, list): + raise ValueError("m.text must be a list") + representations = [] + for element in m_text: + try: + representations.append(MTextRepresentation.parse_obj(element)) + except ValidationError: + continue + return representations + + +class TopicContent(ParseModel): + """ + Represents the `content` field of an `m.room.topic` event + """ + + topic: StrictStr + """ + The topic in plain text. + """ + + m_topic: Optional[MTopic] = Field(alias="m.topic") + """ + Textual representation of the room topic in different mimetypes. + """ + + # We ignore invalid `m.topic` fields as we can always fall back to the plain-text + # `topic` field. + @validator("m_topic", pre=True) + def ignore_invalid_m_topic(cls, m_topic: Any) -> Optional[MTopic]: + try: + return MTopic.parse_obj(m_topic) + except ValidationError: + return None + + +def get_plain_text_topic_from_event_content(content: JsonDict) -> Optional[str]: + """ + Given the `content` of an `m.room.topic` event, returns the plain-text topic + representation. Prefers pulling plain-text from the newer `m.topic` field if + available with a fallback to `topic`. + + Args: + content: The `content` field of an `m.room.topic` event. + + Returns: + A string representing the plain text topic. + """ + + try: + topic_content = TopicContent.parse_obj(content) + except ValidationError: + return None + + # Find the first `text/plain` topic ("Receivers SHOULD use the first + # representationin the array that they understand.") + if topic_content.m_topic and topic_content.m_topic.m_text: + for representation in topic_content.m_topic.m_text: + # The mimetype property defaults to `text/plain` if omitted. + if not representation.mimetype or representation.mimetype == "text/plain": + return representation.body + + # Fallback to the plain-old `topic` field if there isn't any `text/plain` topic + # representation available. + return topic_content.topic diff --git a/synapse/util/pydantic_models.py b/synapse/util/pydantic_models.py new file mode 100644 index 0000000000..ba9e7bb7d5 --- /dev/null +++ b/synapse/util/pydantic_models.py @@ -0,0 +1,39 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from synapse._pydantic_compat import BaseModel, Extra + + +class ParseModel(BaseModel): + """A custom version of Pydantic's BaseModel which + + - ignores unknown fields and + - does not allow fields to be overwritten after construction, + + but otherwise uses Pydantic's default behaviour. + + For now, ignore unknown fields. In the future, we could change this so that unknown + config values cause a ValidationError, provided the error messages are meaningful to + server operators. + + Subclassing in this way is recommended by + https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally + """ + + class Config: + # By default, ignore fields that we don't recognise. + extra = Extra.ignore + # By default, don't allow fields to be reassigned after parsing. + allow_mutation = False diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 6c93ead3b8..3ba7584c65 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -757,6 +757,59 @@ class RoomsCreateTestCase(RoomBase): assert channel.resource_usage is not None self.assertEqual(37, channel.resource_usage.db_txn_count) + def test_post_room_topic(self) -> None: + # POST with topic key, expect new room id + channel = self.make_request("POST", "/createRoom", b'{"topic":"shenanigans"}') + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from topic key + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": "shenanigans"}]}}, + channel.json_body, + ) + + def test_post_room_topic_initial_state(self) -> None: + # POST with m.room.topic in initial state, expect new room id + channel = self.make_request( + "POST", + "/createRoom", + b'{"initial_state":[{"type": "m.room.topic", "content": {"topic": "foobar"}}]}', + ) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from initial state + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "foobar"}, + channel.json_body, + ) + + def test_post_room_topic_overriding_initial_state(self) -> None: + # POST with m.room.topic in initial state and topic key, expect new room id + channel = self.make_request( + "POST", + "/createRoom", + b'{"initial_state":[{"type": "m.room.topic", "content": {"topic": "foobar"}}], "topic":"shenanigans"}', + ) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertTrue("room_id" in channel.json_body) + room_id = channel.json_body["room_id"] + + # GET topic event, expect content from topic key + channel = self.make_request("GET", "/rooms/%s/state/m.room.topic" % (room_id,)) + self.assertEqual(HTTPStatus.OK, channel.code) + self.assertEqual( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": "shenanigans"}]}}, + channel.json_body, + ) + def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id channel = self.make_request("POST", "/createRoom", b'{"visibility":"private"}') diff --git a/tests/util/test_events.py b/tests/util/test_events.py new file mode 100644 index 0000000000..d61ca6b3ab --- /dev/null +++ b/tests/util/test_events.py @@ -0,0 +1,118 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + + +from synapse.util.events import get_plain_text_topic_from_event_content + +from tests import unittest + + +class EventsTestCase(unittest.TestCase): + def test_get_plain_text_topic_no_topic(self) -> None: + # No legacy or rich topic, expect None + topic = get_plain_text_topic_from_event_content({}) + self.assertEqual(None, topic) + + def test_get_plain_text_topic_no_rich_topic(self) -> None: + # Only legacy topic, expect legacy topic + topic = get_plain_text_topic_from_event_content({"topic": "shenanigans"}) + self.assertEqual("shenanigans", topic) + + def test_get_plain_text_topic_rich_topic_without_representations(self) -> None: + # Legacy topic and rich topic without representations, expect legacy topic + topic = get_plain_text_topic_from_event_content( + {"topic": "shenanigans", "m.topic": {"m.text": []}} + ) + self.assertEqual("shenanigans", topic) + + def test_get_plain_text_topic_rich_topic_without_plain_text_representation( + self, + ) -> None: + # Legacy topic and rich topic without plain text representation, expect legacy topic + topic = get_plain_text_topic_from_event_content( + { + "topic": "shenanigans", + "m.topic": { + "m.text": [ + {"mimetype": "text/html", "body": "foobar"} + ] + }, + } + ) + self.assertEqual("shenanigans", topic) + + def test_get_plain_text_topic_rich_topic_with_plain_text_representation( + self, + ) -> None: + # Legacy topic and rich topic with plain text representation, expect plain text representation + topic = get_plain_text_topic_from_event_content( + { + "topic": "shenanigans", + "m.topic": {"m.text": [{"mimetype": "text/plain", "body": "foobar"}]}, + } + ) + self.assertEqual("foobar", topic) + + def test_get_plain_text_topic_rich_topic_with_implicit_plain_text_representation( + self, + ) -> None: + # Legacy topic and rich topic with implicit plain text representation, expect plain text representation + topic = get_plain_text_topic_from_event_content( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": "foobar"}]}} + ) + self.assertEqual("foobar", topic) + + def test_get_plain_text_topic_rich_topic_with_invalid_plain_text_representation( + self, + ) -> None: + # Legacy topic and rich topic with invalid plain text representation, expect legacy topic + topic = get_plain_text_topic_from_event_content( + {"topic": "shenanigans", "m.topic": {"m.text": [{"body": 1337}]}} + ) + self.assertEqual("shenanigans", topic) + + def test_get_plain_text_topic_rich_topic_with_invalid_and_second_valid_plain_text_representation( + self, + ) -> None: + # Legacy topic and rich topic with invalid and second valid plain text representation, expect second plain text representation + topic = get_plain_text_topic_from_event_content( + { + "topic": "shenanigans", + "m.topic": {"m.text": [{"body": 1337}, {"body": "foobar"}]}, + } + ) + self.assertEqual("foobar", topic) + + def test_get_plain_text_topic_rich_topic_with_plain_text_and_other_representation( + self, + ) -> None: + # Legacy topic and rich topic with plain text representation, expect plain text representation + topic = get_plain_text_topic_from_event_content( + { + "topic": "shenanigans", + "m.topic": { + "m.text": [ + {"mimetype": "text/html", "body": "foobar"}, + {"mimetype": "text/plain", "body": "foobar"}, + ] + }, + } + ) + self.assertEqual("foobar", topic) From 66daf0bfaeae34b42a175652a2f3cda93994c2f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Jul 2025 13:39:09 +0100 Subject: [PATCH 460/691] Add ability to limit amount uploaded by a user (#18527) You can now configure how much media can be uploaded by a user in a given time period. Note the first commit here is a refactor of create/upload content function --- changelog.d/18527.feature | 1 + .../configuration/config_documentation.md | 17 +++ schema/synapse-config.schema.yaml | 24 ++++ synapse/config/repository.py | 16 +++ synapse/handlers/sso.py | 2 +- synapse/media/media_repository.py | 128 +++++++++--------- synapse/rest/media/upload_resource.py | 6 +- .../databases/main/media_repository.py | 36 +++++ tests/federation/test_federation_media.py | 10 +- tests/media/test_media_retention.py | 2 +- tests/rest/client/test_media.py | 123 ++++++++++++++++- 11 files changed, 292 insertions(+), 73 deletions(-) create mode 100644 changelog.d/18527.feature diff --git a/changelog.d/18527.feature b/changelog.d/18527.feature new file mode 100644 index 0000000000..3394f6786f --- /dev/null +++ b/changelog.d/18527.feature @@ -0,0 +1 @@ +Add ability to limit amount uploaded by a user in a given time period. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 257ea4a1a2..1fff68ed0a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2086,6 +2086,23 @@ Example configuration: max_upload_size: 60M ``` --- +### `media_upload_limits` + +*(array)* A list of media upload limits defining how much data a given user can upload in a given time period. + +An empty list means no limits are applied. + +Defaults to `[]`. + +Example configuration: +```yaml +media_upload_limits: +- time_period: 1h + max_size: 100M +- time_period: 1w + max_size: 500M +``` +--- ### `max_image_pixels` *(byte size)* Maximum number of pixels that will be thumbnailed. Defaults to `"32M"`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index bcce9b6359..26f0e7f0bc 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2335,6 +2335,30 @@ properties: default: 50M examples: - 60M + media_upload_limits: + type: array + description: >- + A list of media upload limits defining how much data a given user can + upload in a given time period. + + + An empty list means no limits are applied. + default: [] + items: + time_period: + type: "#/$defs/duration" + description: >- + The time period over which the limit applies. Required. + max_size: + type: "#/$defs/bytes" + description: >- + Amount of data that can be uploaded in the time period by the user. + Required. + examples: + - - time_period: 1h + max_size: 100M + - time_period: 1w + max_size: 500M max_image_pixels: $ref: "#/$defs/bytes" description: Maximum number of pixels that will be thumbnailed. diff --git a/synapse/config/repository.py b/synapse/config/repository.py index fc5a90c85a..e6a5064c16 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -119,6 +119,15 @@ def parse_thumbnail_requirements( } +@attr.s(auto_attribs=True, slots=True, frozen=True) +class MediaUploadLimit: + """A limit on the amount of data a user can upload in a given time + period.""" + + max_bytes: int + time_period_ms: int + + class ContentRepositoryConfig(Config): section = "media" @@ -274,6 +283,13 @@ class ContentRepositoryConfig(Config): self.enable_authenticated_media = config.get("enable_authenticated_media", True) + self.media_upload_limits: List[MediaUploadLimit] = [] + for limit_config in config.get("media_upload_limits", []): + time_period_ms = self.parse_duration(limit_config["time_period"]) + max_bytes = self.parse_size(limit_config["max_size"]) + + self.media_upload_limits.append(MediaUploadLimit(max_bytes, time_period_ms)) + def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str: assert data_dir_path is not None media_store = os.path.join(data_dir_path, "media_store") diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 7c5cf91aba..1cdf632649 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -824,7 +824,7 @@ class SsoHandler: return True # store it in media repository - avatar_mxc_url = await self._media_repo.create_content( + avatar_mxc_url = await self._media_repo.create_or_update_content( media_type=headers[b"Content-Type"][0].decode("utf-8"), upload_name=upload_name, content=picture, diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 1d05c536af..d7259176e7 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -177,6 +177,13 @@ class MediaRepository: else: self.url_previewer = None + # We get the media upload limits and sort them in descending order of + # time period, so that we can apply some optimizations. + self.media_upload_limits = hs.config.media.media_upload_limits + self.media_upload_limits.sort( + key=lambda limit: limit.time_period_ms, reverse=True + ) + def _start_update_recently_accessed(self) -> Deferred: return run_as_background_process( "update_recently_accessed_media", self._update_recently_accessed @@ -285,63 +292,16 @@ class MediaRepository: raise NotFoundError("Media ID has expired") @trace - async def update_content( - self, - media_id: str, - media_type: str, - upload_name: Optional[str], - content: IO, - content_length: int, - auth_user: UserID, - ) -> None: - """Update the content of the given media ID. - - Args: - media_id: The media ID to replace. - media_type: The content type of the file. - upload_name: The name of the file, if provided. - content: A file like object that is the content to store - content_length: The length of the content - auth_user: The user_id of the uploader - """ - file_info = FileInfo(server_name=None, file_id=media_id) - sha256reader = SHA256TransparentIOReader(content) - # This implements all of IO as it has a passthrough - fname = await self.media_storage.store_file(sha256reader.wrap(), file_info) - sha256 = sha256reader.hexdigest() - should_quarantine = await self.store.get_is_hash_quarantined(sha256) - logger.info("Stored local media in file %r", fname) - - if should_quarantine: - logger.warning( - "Media has been automatically quarantined as it matched existing quarantined media" - ) - - await self.store.update_local_media( - media_id=media_id, - media_type=media_type, - upload_name=upload_name, - media_length=content_length, - user_id=auth_user, - sha256=sha256, - quarantined_by="system" if should_quarantine else None, - ) - - try: - await self._generate_thumbnails(None, media_id, media_id, media_type) - except Exception as e: - logger.info("Failed to generate thumbnails: %s", e) - - @trace - async def create_content( + async def create_or_update_content( self, media_type: str, upload_name: Optional[str], content: IO, content_length: int, auth_user: UserID, + media_id: Optional[str] = None, ) -> MXCUri: - """Store uploaded content for a local user and return the mxc URL + """Create or update the content of the given media ID. Args: media_type: The content type of the file. @@ -349,16 +309,20 @@ class MediaRepository: content: A file like object that is the content to store content_length: The length of the content auth_user: The user_id of the uploader + media_id: The media ID to update if provided, otherwise creates + new media ID. Returns: The mxc url of the stored content """ - media_id = random_string(24) + is_new_media = media_id is None + if media_id is None: + media_id = random_string(24) file_info = FileInfo(server_name=None, file_id=media_id) - # This implements all of IO as it has a passthrough sha256reader = SHA256TransparentIOReader(content) + # This implements all of IO as it has a passthrough fname = await self.media_storage.store_file(sha256reader.wrap(), file_info) sha256 = sha256reader.hexdigest() should_quarantine = await self.store.get_is_hash_quarantined(sha256) @@ -370,16 +334,56 @@ class MediaRepository: "Media has been automatically quarantined as it matched existing quarantined media" ) - await self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=upload_name, - media_length=content_length, - user_id=auth_user, - sha256=sha256, - quarantined_by="system" if should_quarantine else None, - ) + # Check that the user has not exceeded any of the media upload limits. + + # This is the total size of media uploaded by the user in the last + # `time_period_ms` milliseconds, or None if we haven't checked yet. + uploaded_media_size: Optional[int] = None + + # Note: the media upload limits are sorted so larger time periods are + # first. + for limit in self.media_upload_limits: + # We only need to check the amount of media uploaded by the user in + # this latest (smaller) time period if the amount of media uploaded + # in a previous (larger) time period is above the limit. + # + # This optimization means that in the common case where the user + # hasn't uploaded much media, we only need to query the database + # once. + if ( + uploaded_media_size is None + or uploaded_media_size + content_length > limit.max_bytes + ): + uploaded_media_size = await self.store.get_media_uploaded_size_for_user( + user_id=auth_user.to_string(), time_period_ms=limit.time_period_ms + ) + + if uploaded_media_size + content_length > limit.max_bytes: + raise SynapseError( + 400, "Media upload limit exceeded", Codes.RESOURCE_LIMIT_EXCEEDED + ) + + if is_new_media: + await self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, + ) + else: + await self.store.update_local_media( + media_id=media_id, + media_type=media_type, + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + sha256=sha256, + quarantined_by="system" if should_quarantine else None, + ) try: await self._generate_thumbnails(None, media_id, media_id, media_type) diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index 572f7897fd..74d8280582 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -120,7 +120,7 @@ class UploadServlet(BaseUploadServlet): try: content: IO = request.content # type: ignore - content_uri = await self.media_repo.create_content( + content_uri = await self.media_repo.create_or_update_content( media_type, upload_name, content, content_length, requester.user ) except SpamMediaException: @@ -170,13 +170,13 @@ class AsyncUploadServlet(BaseUploadServlet): try: content: IO = request.content # type: ignore - await self.media_repo.update_content( - media_id, + await self.media_repo.create_or_update_content( media_type, upload_name, content, content_length, requester.user, + media_id=media_id, ) except SpamMediaException: # For uploading of media we want to respond with a 400, instead of diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 04866524e3..f726846e57 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -1034,3 +1034,39 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): "local_media_repository", sha256, ) + + async def get_media_uploaded_size_for_user( + self, user_id: str, time_period_ms: int + ) -> int: + """Get the total size of media uploaded by a user in the last + time_period_ms milliseconds. + + Args: + user_id: The user ID to check. + time_period_ms: The time period in milliseconds to consider. + + Returns: + The total size of media uploaded by the user in bytes. + """ + + sql = """ + SELECT COALESCE(SUM(media_length), 0) + FROM local_media_repository + WHERE user_id = ? AND created_ts > ? + """ + + def _get_media_uploaded_size_for_user_txn( + txn: LoggingTransaction, + ) -> int: + # Calculate the timestamp for the start of the time period + start_ts = self._clock.time_msec() - time_period_ms + txn.execute(sql, (user_id, start_ts)) + row = txn.fetchone() + if row is None: + return 0 + return row[0] + + return await self.db_pool.runInteraction( + "get_media_uploaded_size_for_user", + _get_media_uploaded_size_for_user_txn, + ) diff --git a/tests/federation/test_federation_media.py b/tests/federation/test_federation_media.py index 9c92003ce5..cd4905239f 100644 --- a/tests/federation/test_federation_media.py +++ b/tests/federation/test_federation_media.py @@ -67,7 +67,7 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): def test_file_download(self) -> None: content = io.BytesIO(b"file_to_stream") content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "text/plain", "test_upload", content, @@ -110,7 +110,7 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): content = io.BytesIO(SMALL_PNG) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_upload", content, @@ -152,7 +152,7 @@ class FederationMediaDownloadsTest(unittest.FederatingHomeserverTestCase): content = io.BytesIO(b"file_to_stream") content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "text/plain", "test_upload", content, @@ -215,7 +215,7 @@ class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def test_thumbnail_download_scaled(self) -> None: content = io.BytesIO(small_png.data) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_thumbnail", content, @@ -255,7 +255,7 @@ class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def test_thumbnail_download_cropped(self) -> None: content = io.BytesIO(small_png.data) content_uri = self.get_success( - self.media_repo.create_content( + self.media_repo.create_or_update_content( "image/png", "test_png_thumbnail", content, diff --git a/tests/media/test_media_retention.py b/tests/media/test_media_retention.py index d8f4f57c8c..89cf61430a 100644 --- a/tests/media/test_media_retention.py +++ b/tests/media/test_media_retention.py @@ -78,7 +78,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): # If the meda random_content = bytes(random_string(24), "utf-8") mxc_uri: MXCUri = self.get_success( - media_repository.create_content( + media_repository.create_or_update_content( media_type="text/plain", upload_name=None, content=io.BytesIO(random_content), diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 6ee761e44b..7aa1f2406c 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1952,7 +1952,7 @@ class RemoteDownloadLimiterTestCase(unittest.HomeserverTestCase): def test_file_download(self) -> None: content = io.BytesIO(b"file_to_stream") content_uri = self.get_success( - self.repo.create_content( + self.repo.create_or_update_content( "text/plain", "test_upload", content, @@ -2846,3 +2846,124 @@ class AuthenticatedMediaTestCase(unittest.HomeserverTestCase): custom_headers=[("If-None-Match", etag)], ) self.assertEqual(channel3.code, 404) + + +class MediaUploadLimits(unittest.HomeserverTestCase): + """ + This test case simulates a homeserver with media upload limits configured. + """ + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + config = self.default_config() + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + config["media_store_path"] = self.media_store_path + + provider_config = { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + config["media_storage_providers"] = [provider_config] + + # These are the limits that we are testing + config["media_upload_limits"] = [ + {"time_period": "1d", "max_size": "1K"}, + {"time_period": "1w", "max_size": "3K"}, + ] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.repo = hs.get_media_repository() + self.client = hs.get_federation_http_client() + self.store = hs.get_datastores().main + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + def create_resource_dict(self) -> Dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def upload_media(self, size: int) -> FakeChannel: + """Helper to upload media of a given size.""" + return self.make_request( + "POST", + "/_matrix/media/v3/upload", + content=b"0" * size, + access_token=self.tok, + shorthand=False, + content_type=b"text/plain", + custom_headers=[("Content-Length", str(size))], + ) + + def test_upload_under_limit(self) -> None: + """Test that uploading media under the limit works.""" + channel = self.upload_media(67) + self.assertEqual(channel.code, 200) + + def test_over_day_limit(self) -> None: + """Test that uploading media over the daily limit fails.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + channel = self.upload_media(800) + self.assertEqual(channel.code, 400) + + def test_under_daily_limit(self) -> None: + """Test that uploading media under the daily limit fails.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + # This will succeed as the daily limit has reset + channel = self.upload_media(800) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + # ... and again + channel = self.upload_media(800) + self.assertEqual(channel.code, 200) + + def test_over_weekly_limit(self) -> None: + """Test that uploading media over the weekly limit fails.""" + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(60 * 60 * 24) # Advance by one day + + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(2 * 60 * 60 * 24) # Advance by one day + + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) + + self.reactor.advance(2 * 60 * 60 * 24) # Advance by one day + + # This will fail as the weekly limit has been exceeded + channel = self.upload_media(900) + self.assertEqual(channel.code, 400) + + # Reset the weekly limit by advancing a week + self.reactor.advance(7 * 60 * 60 * 24) # Advance by 7 days + + # This will succeed as the weekly limit has reset + channel = self.upload_media(900) + self.assertEqual(channel.code, 200) From 1dc29563c1504a2523e467aa7bef6a7ac05cc60c Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Thu, 10 Jul 2025 15:13:27 +0200 Subject: [PATCH 461/691] Move registrations off the main worker (#18552) This is mainly moving a few store methods around. Note that this doesn't yet remove the replication servlet to avoid breaking during rollout. --- changelog.d/18552.misc | 1 + docs/upgrade.md | 8 + synapse/app/generic_worker.py | 2 +- synapse/handlers/register.py | 45 +-- synapse/replication/http/register.py | 2 + .../storage/databases/main/registration.py | 313 +++++++++--------- 6 files changed, 182 insertions(+), 189 deletions(-) create mode 100644 changelog.d/18552.misc diff --git a/changelog.d/18552.misc b/changelog.d/18552.misc new file mode 100644 index 0000000000..3882c8fed2 --- /dev/null +++ b/changelog.d/18552.misc @@ -0,0 +1 @@ +Allow user registrations to be done on workers. diff --git a/docs/upgrade.md b/docs/upgrade.md index ca9ca121f2..e79ca93c04 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.135.0 + +## `on_user_registration` module API callback may now run on any worker + +Previously, the `on_user_registration` callback would only run on the main +process. Modules relying on this callback must assume that they may now be +called from any worker, not just the main process. + # Upgrading to v1.134.0 ## ICU bundled with Synapse diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 75c65ccc0d..d0924c413b 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -118,7 +118,6 @@ class GenericWorkerStore( # FIXME(https://github.com/matrix-org/synapse/issues/3714): We need to add # UserDirectoryStore as we write directly rather than going via the correct worker. UserDirectoryStore, - StatsStore, UIAuthWorkerStore, EndToEndRoomKeyStore, PresenceStore, @@ -154,6 +153,7 @@ class GenericWorkerStore( StreamWorkerStore, EventsWorkerStore, RegistrationWorkerStore, + StatsStore, SearchStore, TransactionWorkerStore, LockStore, diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 970013ef20..b90126f9c7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -49,7 +49,6 @@ from synapse.http.servlet import assert_params_in_dict from synapse.replication.http.login import RegisterDeviceReplicationServlet from synapse.replication.http.register import ( ReplicationPostRegisterActionsServlet, - ReplicationRegisterServlet, ) from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import GUEST_USER_ID_PATTERN, RoomAlias, UserID, create_requester @@ -120,7 +119,6 @@ class RegistrationHandler: self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker if hs.config.worker.worker_app: - self._register_client = ReplicationRegisterServlet.make_client(hs) self._register_device_client = RegisterDeviceReplicationServlet.make_client( hs ) @@ -738,37 +736,20 @@ class RegistrationHandler: shadow_banned: Whether to shadow-ban the user approved: Whether to mark the user as approved by an administrator """ - if self.hs.config.worker.worker_app: - await self._register_client( - user_id=user_id, - password_hash=password_hash, - was_guest=was_guest, - make_guest=make_guest, - appservice_id=appservice_id, - create_profile_with_displayname=create_profile_with_displayname, - admin=admin, - user_type=user_type, - address=address, - shadow_banned=shadow_banned, - approved=approved, - ) - else: - await self.store.register_user( - user_id=user_id, - password_hash=password_hash, - was_guest=was_guest, - make_guest=make_guest, - appservice_id=appservice_id, - create_profile_with_displayname=create_profile_with_displayname, - admin=admin, - user_type=user_type, - shadow_banned=shadow_banned, - approved=approved, - ) + await self.store.register_user( + user_id=user_id, + password_hash=password_hash, + was_guest=was_guest, + make_guest=make_guest, + appservice_id=appservice_id, + create_profile_with_displayname=create_profile_with_displayname, + admin=admin, + user_type=user_type, + shadow_banned=shadow_banned, + approved=approved, + ) - # Only call the account validity module(s) on the main process, to avoid - # repeating e.g. database writes on all of the workers. - await self._account_validity_handler.on_user_registration(user_id) + await self._account_validity_handler.on_user_registration(user_id) async def register_device( self, diff --git a/synapse/replication/http/register.py b/synapse/replication/http/register.py index 42a58b2858..27d3504c3c 100644 --- a/synapse/replication/http/register.py +++ b/synapse/replication/http/register.py @@ -33,6 +33,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# FIXME(2025-07-22): Remove this on the next release, this may only be used +# during rollout to Synapse 1.134 and can be removed after that release. class ReplicationRegisterServlet(ReplicationEndpoint): """Register a new user""" diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 1e21996b12..38d1e650ad 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -175,7 +175,7 @@ class ThreepidValidationSession: """timestamp of when this session was validated if so""" -class RegistrationWorkerStore(CacheInvalidationWorkerStore): +class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): def __init__( self, database: DatabasePool, @@ -217,12 +217,167 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): self._set_expiration_date_when_missing, ) + # If support for MSC3866 is enabled and configured to require approval for new + # account, we will create new users with an 'approved' flag set to false. + self._require_approval = ( + hs.config.experimental.msc3866.enabled + and hs.config.experimental.msc3866.require_approval_for_new_accounts + ) + # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: self._clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS ) + async def register_user( + self, + user_id: str, + password_hash: Optional[str] = None, + was_guest: bool = False, + make_guest: bool = False, + appservice_id: Optional[str] = None, + create_profile_with_displayname: Optional[str] = None, + admin: bool = False, + user_type: Optional[str] = None, + shadow_banned: bool = False, + approved: bool = False, + ) -> None: + """Attempts to register an account. + + Args: + user_id: The desired user ID to register. + password_hash: Optional. The password hash for this user. + was_guest: Whether this is a guest account being upgraded to a + non-guest account. + make_guest: True if the the new user should be guest, false to add a + regular user account. + appservice_id: The ID of the appservice registering the user. + create_profile_with_displayname: Optionally create a profile for + the user, setting their displayname to the given value + admin: is an admin user? + user_type: type of user. One of the values from api.constants.UserTypes, + a custom value set in the configuration file, or None for a normal + user. + shadow_banned: Whether the user is shadow-banned, i.e. they may be + told their requests succeeded but we ignore them. + approved: Whether to consider the user has already been approved by an + administrator. + + Raises: + StoreError if the user_id could not be registered. + """ + await self.db_pool.runInteraction( + "register_user", + self._register_user, + user_id, + password_hash, + was_guest, + make_guest, + appservice_id, + create_profile_with_displayname, + admin, + user_type, + shadow_banned, + approved, + ) + + def _register_user( + self, + txn: LoggingTransaction, + user_id: str, + password_hash: Optional[str], + was_guest: bool, + make_guest: bool, + appservice_id: Optional[str], + create_profile_with_displayname: Optional[str], + admin: bool, + user_type: Optional[str], + shadow_banned: bool, + approved: bool, + ) -> None: + user_id_obj = UserID.from_string(user_id) + + now = int(self._clock.time()) + + user_approved = approved or not self._require_approval + + try: + if was_guest: + # Ensure that the guest user actually exists + # ``allow_none=False`` makes this raise an exception + # if the row isn't in the database. + self.db_pool.simple_select_one_txn( + txn, + "users", + keyvalues={"name": user_id, "is_guest": 1}, + retcols=("name",), + allow_none=False, + ) + + self.db_pool.simple_update_one_txn( + txn, + "users", + keyvalues={"name": user_id, "is_guest": 1}, + updatevalues={ + "password_hash": password_hash, + "upgrade_ts": now, + "is_guest": 1 if make_guest else 0, + "appservice_id": appservice_id, + "admin": 1 if admin else 0, + "user_type": user_type, + "shadow_banned": shadow_banned, + "approved": user_approved, + }, + ) + else: + self.db_pool.simple_insert_txn( + txn, + "users", + values={ + "name": user_id, + "password_hash": password_hash, + "creation_ts": now, + "is_guest": 1 if make_guest else 0, + "appservice_id": appservice_id, + "admin": 1 if admin else 0, + "user_type": user_type, + "shadow_banned": shadow_banned, + "approved": user_approved, + }, + ) + + except self.database_engine.module.IntegrityError: + raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE) + + if self._account_validity_enabled: + self.set_expiration_date_for_user_txn(txn, user_id) + + if create_profile_with_displayname: + # set a default displayname serverside to avoid ugly race + # between auto-joins and clients trying to set displaynames + # + # *obviously* the 'profiles' table uses localpart for user_id + # while everything else uses the full mxid. + txn.execute( + "INSERT INTO profiles(full_user_id, user_id, displayname) VALUES (?,?,?)", + (user_id, user_id_obj.localpart, create_profile_with_displayname), + ) + + if self.hs.config.stats.stats_enabled: + # we create a new completed user statistics row + + # we don't strictly need current_token since this user really can't + # have any state deltas before now (as it is a new user), but still, + # we include it for completeness. + current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn) + + self._update_stats_delta_txn( + txn, now, "user", user_id, {}, complete_with_stream_id=current_token + ) + + self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) + @cached() async def get_user_by_id(self, user_id: str) -> Optional[UserInfo]: """Returns info about the user account, if it exists.""" @@ -2354,7 +2509,7 @@ class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): return nb_processed -class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): +class RegistrationStore(RegistrationBackgroundUpdateStore): def __init__( self, database: DatabasePool, @@ -2370,13 +2525,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") - # If support for MSC3866 is enabled and configured to require approval for new - # account, we will create new users with an 'approved' flag set to false. - self._require_approval = ( - hs.config.experimental.msc3866.enabled - and hs.config.experimental.msc3866.require_approval_for_new_accounts - ) - # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: self._clock.looping_call( @@ -2524,153 +2672,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): device_id, ) - async def register_user( - self, - user_id: str, - password_hash: Optional[str] = None, - was_guest: bool = False, - make_guest: bool = False, - appservice_id: Optional[str] = None, - create_profile_with_displayname: Optional[str] = None, - admin: bool = False, - user_type: Optional[str] = None, - shadow_banned: bool = False, - approved: bool = False, - ) -> None: - """Attempts to register an account. - - Args: - user_id: The desired user ID to register. - password_hash: Optional. The password hash for this user. - was_guest: Whether this is a guest account being upgraded to a - non-guest account. - make_guest: True if the the new user should be guest, false to add a - regular user account. - appservice_id: The ID of the appservice registering the user. - create_profile_with_displayname: Optionally create a profile for - the user, setting their displayname to the given value - admin: is an admin user? - user_type: type of user. One of the values from api.constants.UserTypes, - a custom value set in the configuration file, or None for a normal - user. - shadow_banned: Whether the user is shadow-banned, i.e. they may be - told their requests succeeded but we ignore them. - approved: Whether to consider the user has already been approved by an - administrator. - - Raises: - StoreError if the user_id could not be registered. - """ - await self.db_pool.runInteraction( - "register_user", - self._register_user, - user_id, - password_hash, - was_guest, - make_guest, - appservice_id, - create_profile_with_displayname, - admin, - user_type, - shadow_banned, - approved, - ) - - def _register_user( - self, - txn: LoggingTransaction, - user_id: str, - password_hash: Optional[str], - was_guest: bool, - make_guest: bool, - appservice_id: Optional[str], - create_profile_with_displayname: Optional[str], - admin: bool, - user_type: Optional[str], - shadow_banned: bool, - approved: bool, - ) -> None: - user_id_obj = UserID.from_string(user_id) - - now = int(self._clock.time()) - - user_approved = approved or not self._require_approval - - try: - if was_guest: - # Ensure that the guest user actually exists - # ``allow_none=False`` makes this raise an exception - # if the row isn't in the database. - self.db_pool.simple_select_one_txn( - txn, - "users", - keyvalues={"name": user_id, "is_guest": 1}, - retcols=("name",), - allow_none=False, - ) - - self.db_pool.simple_update_one_txn( - txn, - "users", - keyvalues={"name": user_id, "is_guest": 1}, - updatevalues={ - "password_hash": password_hash, - "upgrade_ts": now, - "is_guest": 1 if make_guest else 0, - "appservice_id": appservice_id, - "admin": 1 if admin else 0, - "user_type": user_type, - "shadow_banned": shadow_banned, - "approved": user_approved, - }, - ) - else: - self.db_pool.simple_insert_txn( - txn, - "users", - values={ - "name": user_id, - "password_hash": password_hash, - "creation_ts": now, - "is_guest": 1 if make_guest else 0, - "appservice_id": appservice_id, - "admin": 1 if admin else 0, - "user_type": user_type, - "shadow_banned": shadow_banned, - "approved": user_approved, - }, - ) - - except self.database_engine.module.IntegrityError: - raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE) - - if self._account_validity_enabled: - self.set_expiration_date_for_user_txn(txn, user_id) - - if create_profile_with_displayname: - # set a default displayname serverside to avoid ugly race - # between auto-joins and clients trying to set displaynames - # - # *obviously* the 'profiles' table uses localpart for user_id - # while everything else uses the full mxid. - txn.execute( - "INSERT INTO profiles(full_user_id, user_id, displayname) VALUES (?,?,?)", - (user_id, user_id_obj.localpart, create_profile_with_displayname), - ) - - if self.hs.config.stats.stats_enabled: - # we create a new completed user statistics row - - # we don't strictly need current_token since this user really can't - # have any state deltas before now (as it is a new user), but still, - # we include it for completeness. - current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn) - self._update_stats_delta_txn( - txn, now, "user", user_id, {}, complete_with_stream_id=current_token - ) - - self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) - async def user_set_password_hash( self, user_id: str, password_hash: Optional[str] ) -> None: From 28c9ed3ccbd95a4a8d06866d4088320e5acfbbd0 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 11 Jul 2025 10:47:54 +0200 Subject: [PATCH 462/691] Remove unnecessary replication calls (#18564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be reviewed commit by commit. Nowadays it's trivial to propagate cache invalidations, which means we can move some things off the main process, and not go through HTTP replication. `ReplicationGetQueryRestServlet` appeared to be unused, and was very weird, as it was being called if the current instance is the main one… to RPC to the main one (if no instance is set on a replication client, it makes it to the main process) The other two handlers could be relatively trivially moved to any workers, moving some methods to the worker store. **I've intentionally not removed the replication servlets yet** so that it's safe to rollout, and will do another PR that clean those up to remove on the N+1 version --- changelog.d/18564.misc | 1 + synapse/federation/federation_server.py | 6 - synapse/handlers/federation.py | 35 +----- synapse/replication/http/federation.py | 6 + synapse/storage/databases/main/cache.py | 11 +- .../databases/main/event_federation.py | 31 +++-- synapse/storage/databases/main/room.py | 118 +++++++++--------- 7 files changed, 97 insertions(+), 111 deletions(-) create mode 100644 changelog.d/18564.misc diff --git a/changelog.d/18564.misc b/changelog.d/18564.misc new file mode 100644 index 0000000000..a31c98370f --- /dev/null +++ b/changelog.d/18564.misc @@ -0,0 +1 @@ +Remove unnecessary HTTP replication calls. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8718b70401..2a7f5b2c4d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -85,7 +85,6 @@ from synapse.logging.opentracing import ( from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.replication.http.federation import ( ReplicationFederationSendEduRestServlet, - ReplicationGetQueryRestServlet, ) from synapse.storage.databases.main.lock import Lock from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary @@ -1380,7 +1379,6 @@ class FederationHandlerRegistry: # and use them. However we have guards before we use them to ensure that # we don't route to ourselves, and in monolith mode that will always be # the case. - self._get_query_client = ReplicationGetQueryRestServlet.make_client(hs) self._send_edu = ReplicationFederationSendEduRestServlet.make_client(hs) self.edu_handlers: Dict[str, Callable[[str, dict], Awaitable[None]]] = {} @@ -1469,10 +1467,6 @@ class FederationHandlerRegistry: if handler: return await handler(args) - # Check if we can route it somewhere else that isn't us - if self._instance_name == "master": - return await self._get_query_client(query_type=query_type, args=args) - # Uh oh, no handler! Let's raise an exception so the request returns an # error. logger.warning("No handler registered for query type %s", query_type) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 015fb3edca..f7806e67a9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -73,10 +73,6 @@ from synapse.logging.context import nested_logging_context from synapse.logging.opentracing import SynapseTags, set_tag, tag_args, trace from synapse.metrics.background_process_metrics import run_as_background_process from synapse.module_api import NOT_SPAM -from synapse.replication.http.federation import ( - ReplicationCleanRoomRestServlet, - ReplicationStoreRoomOnOutlierMembershipRestServlet, -) from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.invite_rule import InviteRule from synapse.types import JsonDict, StrCollection, get_domain_from_id @@ -163,19 +159,6 @@ class FederationHandler: self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._clean_room_for_join_client = ReplicationCleanRoomRestServlet.make_client( - hs - ) - - if hs.config.worker.worker_app: - self._maybe_store_room_on_outlier_membership = ( - ReplicationStoreRoomOnOutlierMembershipRestServlet.make_client(hs) - ) - else: - self._maybe_store_room_on_outlier_membership = ( - self.store.maybe_store_room_on_outlier_membership - ) - self._room_backfill = Linearizer("room_backfill") self._third_party_event_rules = ( @@ -647,7 +630,7 @@ class FederationHandler: # room. # In short, the races either have an acceptable outcome or should be # impossible. - await self._clean_room_for_join(room_id) + await self.store.clean_room_for_join(room_id) try: # Try the host we successfully got a response to /make_join/ @@ -857,7 +840,7 @@ class FederationHandler: event.internal_metadata.out_of_band_membership = True # Record the room ID and its version so that we have a record of the room - await self._maybe_store_room_on_outlier_membership( + await self.store.maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=event_format_version ) @@ -1115,7 +1098,7 @@ class FederationHandler: # keep a record of the room version, if we don't yet know it. # (this may get overwritten if we later get a different room version in a # join dance). - await self._maybe_store_room_on_outlier_membership( + await self.store.maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=room_version ) @@ -1761,18 +1744,6 @@ class FederationHandler: if "valid" not in response or not response["valid"]: raise AuthError(403, "Third party certificate was invalid") - async def _clean_room_for_join(self, room_id: str) -> None: - """Called to clean up any data in DB for a given room, ready for the - server to join the room. - - Args: - room_id - """ - if self.config.worker.worker_app: - await self._clean_room_for_join_client(room_id) - else: - await self.store.clean_room_for_join(room_id) - async def get_room_complexity( self, remote_room_hosts: List[str], room_id: str ) -> Optional[dict]: diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index 940f418396..f3f8ddfd65 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -202,6 +202,8 @@ class ReplicationFederationSendEduRestServlet(ReplicationEndpoint): return 200, {} +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationGetQueryRestServlet(ReplicationEndpoint): """Handle responding to queries from federation. @@ -249,6 +251,8 @@ class ReplicationGetQueryRestServlet(ReplicationEndpoint): return 200, result +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationCleanRoomRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. @@ -284,6 +288,8 @@ class ReplicationCleanRoomRestServlet(ReplicationEndpoint): return 200, {} +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 9418fb6dd7..dc37f67110 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -41,7 +41,6 @@ from synapse.storage.database import ( LoggingDatabaseConnection, LoggingTransaction, ) -from synapse.storage.databases.main.events import SLIDING_SYNC_RELEVANT_STATE_SET from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.util.caches.descriptors import CachedFunction @@ -284,6 +283,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): super().process_replication_position(stream_name, instance_name, token) def _process_event_stream_row(self, token: int, row: EventsStreamRow) -> None: + # This is needed to avoid a circular import. + from synapse.storage.databases.main.events import ( + SLIDING_SYNC_RELEVANT_STATE_SET, + ) + data = row.data if row.type == EventsStreamEventRow.TypeId: @@ -347,6 +351,11 @@ class CacheInvalidationWorkerStore(SQLBaseStore): relates_to: Optional[str], backfilled: bool, ) -> None: + # This is needed to avoid a circular import. + from synapse.storage.databases.main.events import ( + SLIDING_SYNC_RELEVANT_STATE_SET, + ) + # XXX: If you add something to this function make sure you add it to # `_invalidate_caches_for_room_events` as well. diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 46aa5902d8..dfc25d8935 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -46,13 +46,14 @@ from synapse.api.room_versions import EventFormatVersions, RoomVersion from synapse.events import EventBase, make_event_from_dict from synapse.logging.opentracing import tag_args, trace from synapse.metrics.background_process_metrics import wrap_as_background_process -from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause +from synapse.storage._base import db_to_json, make_in_list_sql_clause from synapse.storage.background_updates import ForeignKeyConstraint from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, ) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.signatures import SignatureWorkerStore from synapse.storage.engines import PostgresEngine, Sqlite3Engine @@ -123,7 +124,9 @@ class _NoChainCoverIndex(Exception): super().__init__("Unexpectedly no chain cover for events in %s" % (room_id,)) -class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBaseStore): +class EventFederationWorkerStore( + SignatureWorkerStore, EventsWorkerStore, CacheInvalidationWorkerStore +): # TODO: this attribute comes from EventPushActionWorkerStore. Should we inherit from # that store so that mypy can deduce this for itself? stream_ordering_month_ago: Optional[int] @@ -2053,6 +2056,19 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas number_pdus_in_federation_queue.set(count) oldest_pdu_in_federation_staging.set(age) + async def clean_room_for_join(self, room_id: str) -> None: + await self.db_pool.runInteraction( + "clean_room_for_join", self._clean_room_for_join_txn, room_id + ) + + def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None: + query = "DELETE FROM event_forward_extremities WHERE room_id = ?" + + txn.execute(query, (room_id,)) + self._invalidate_cache_and_stream( + txn, self.get_latest_event_ids_in_room, (room_id,) + ) + class EventFederationStore(EventFederationWorkerStore): """Responsible for storing and serving up the various graphs associated @@ -2078,17 +2094,6 @@ class EventFederationStore(EventFederationWorkerStore): self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth ) - async def clean_room_for_join(self, room_id: str) -> None: - await self.db_pool.runInteraction( - "clean_room_for_join", self._clean_room_for_join_txn, room_id - ) - - def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None: - query = "DELETE FROM event_forward_extremities WHERE room_id = ?" - - txn.execute(query, (room_id,)) - txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,)) - async def _background_delete_non_state_event_auth( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 58451d3ff1..153bdb3510 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1935,6 +1935,65 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): desc="set_room_is_public_appservice_false", ) + async def has_auth_chain_index(self, room_id: str) -> bool: + """Check if the room has (or can have) a chain cover index. + + Defaults to True if we don't have an entry in `rooms` table nor any + events for the room. + """ + + has_auth_chain_index = await self.db_pool.simple_select_one_onecol( + table="rooms", + keyvalues={"room_id": room_id}, + retcol="has_auth_chain_index", + desc="has_auth_chain_index", + allow_none=True, + ) + + if has_auth_chain_index: + return True + + # It's possible that we already have events for the room in our DB + # without a corresponding room entry. If we do then we don't want to + # mark the room as having an auth chain cover index. + max_ordering = await self.db_pool.simple_select_one_onecol( + table="events", + keyvalues={"room_id": room_id}, + retcol="MAX(stream_ordering)", + allow_none=True, + desc="has_auth_chain_index_fallback", + ) + + return max_ordering is None + + async def maybe_store_room_on_outlier_membership( + self, room_id: str, room_version: RoomVersion + ) -> None: + """ + When we receive an invite or any other event over federation that may relate to a room + we are not in, store the version of the room if we don't already know the room version. + """ + # It's possible that we already have events for the room in our DB + # without a corresponding room entry. If we do then we don't want to + # mark the room as having an auth chain cover index. + has_auth_chain_index = await self.has_auth_chain_index(room_id) + + await self.db_pool.simple_upsert( + desc="maybe_store_room_on_outlier_membership", + table="rooms", + keyvalues={"room_id": room_id}, + values={}, + insertion_values={ + "room_version": room_version.identifier, + "is_public": False, + # We don't worry about setting the `creator` here because + # we don't process any messages in a room while a user is + # invited (only after the join). + "creator": "", + "has_auth_chain_index": has_auth_chain_index, + }, + ) + class _BackgroundUpdates: REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" @@ -2186,37 +2245,6 @@ class RoomBackgroundUpdateStore(RoomWorkerStore): return len(rooms) - async def has_auth_chain_index(self, room_id: str) -> bool: - """Check if the room has (or can have) a chain cover index. - - Defaults to True if we don't have an entry in `rooms` table nor any - events for the room. - """ - - has_auth_chain_index = await self.db_pool.simple_select_one_onecol( - table="rooms", - keyvalues={"room_id": room_id}, - retcol="has_auth_chain_index", - desc="has_auth_chain_index", - allow_none=True, - ) - - if has_auth_chain_index: - return True - - # It's possible that we already have events for the room in our DB - # without a corresponding room entry. If we do then we don't want to - # mark the room as having an auth chain cover index. - max_ordering = await self.db_pool.simple_select_one_onecol( - table="events", - keyvalues={"room_id": room_id}, - retcol="MAX(stream_ordering)", - allow_none=True, - desc="has_auth_chain_index_fallback", - ) - - return max_ordering is None - async def _background_populate_room_depth_min_depth2( self, progress: JsonDict, batch_size: int ) -> int: @@ -2567,34 +2595,6 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): updatevalues={"join_event_id": join_event_id}, ) - async def maybe_store_room_on_outlier_membership( - self, room_id: str, room_version: RoomVersion - ) -> None: - """ - When we receive an invite or any other event over federation that may relate to a room - we are not in, store the version of the room if we don't already know the room version. - """ - # It's possible that we already have events for the room in our DB - # without a corresponding room entry. If we do then we don't want to - # mark the room as having an auth chain cover index. - has_auth_chain_index = await self.has_auth_chain_index(room_id) - - await self.db_pool.simple_upsert( - desc="maybe_store_room_on_outlier_membership", - table="rooms", - keyvalues={"room_id": room_id}, - values={}, - insertion_values={ - "room_version": room_version.identifier, - "is_public": False, - # We don't worry about setting the `creator` here because - # we don't process any messages in a room while a user is - # invited (only after the join). - "creator": "", - "has_auth_chain_index": has_auth_chain_index, - }, - ) - async def add_event_report( self, room_id: str, From c5999cf452e706d76e270aa9702d0c5dab6c7abc Mon Sep 17 00:00:00 2001 From: V02460 Date: Fri, 11 Jul 2025 12:15:12 +0200 Subject: [PATCH 463/691] Document config file merge behavior (#18664) Explains in the doc comment of `synapse.config._base.read_config_file` how config files are merged. --- changelog.d/18664.misc | 1 + synapse/config/_base.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18664.misc diff --git a/changelog.d/18664.misc b/changelog.d/18664.misc new file mode 100644 index 0000000000..eed76c95bf --- /dev/null +++ b/changelog.d/18664.misc @@ -0,0 +1 @@ +Add doc comment explaining that config files are shallowly merged. \ No newline at end of file diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d367d45fea..0a01fb5582 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -909,7 +909,10 @@ class RootConfig: def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: - """Read the config files into a dict + """Read the config files and shallowly merge them into a dict. + + Successive configurations are shallowly merged into ones provided earlier, + i.e., entirely replacing top-level sections of the configuration. Args: config_files: A list of the config files to read From 947216abc010212c8621031433fb79ff9aafe0fd Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:34:37 +0100 Subject: [PATCH 464/691] Update `latest_deps` workflow to migrate `poetry --no-dev` -> `--without dev` (#18617) --- .github/workflows/latest_deps.yml | 2 +- changelog.d/18617.misc | 1 + pyproject.toml | 2 +- synapse/util/check_dependencies.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18617.misc diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 702f1a5381..0cd654ac7e 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -60,7 +60,7 @@ jobs: - run: poetry run pip list > before.txt # Upgrade all runtime dependencies only. This is intended to mimic a fresh # `pip install matrix-synapse[all]` as closely as possible. - - run: poetry update --no-dev + - run: poetry update --without dev - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) - name: Remove unhelpful options from mypy config run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini diff --git a/changelog.d/18617.misc b/changelog.d/18617.misc new file mode 100644 index 0000000000..549ffb2cca --- /dev/null +++ b/changelog.d/18617.misc @@ -0,0 +1 @@ +Unbreak "Latest dependencies" workflow by using the `--without dev` poetry option instead of removed `--no-dev`. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 79b2cf9690..3ab936f1c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,7 +224,7 @@ pydantic = ">=1.7.4, <3" # https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and # `poetry build` do the right thing without this explicit dependency. # -# This isn't really a dev-dependency, as `poetry install --no-dev` will fail, +# This isn't really a dev-dependency, as `poetry install --without dev` will fail, # but the alternative is to add it to the main list of deps where it isn't # needed. setuptools_rust = ">=1.3" diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py index c790c9667f..1c79c0be48 100644 --- a/synapse/util/check_dependencies.py +++ b/synapse/util/check_dependencies.py @@ -74,7 +74,7 @@ def _is_dev_dependency(req: Requirement) -> bool: def _should_ignore_runtime_requirement(req: Requirement) -> bool: # This is a build-time dependency. Irritatingly, `poetry build` ignores the # requirements listed in the [build-system] section of pyproject.toml, so in order - # to support `poetry install --no-dev` we have to mark it as a runtime dependency. + # to support `poetry install --without dev` we have to mark it as a runtime dependency. # See discussion on https://github.com/python-poetry/poetry/issues/6154 (it sounds # like the poetry authors don't consider this a bug?) # From 1e5e6a48be08cb901e4c120f56a322e71e8247af Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 11 Jul 2025 14:54:46 +0200 Subject: [PATCH 465/691] Use markdown-it-py instead of commonmark in the release script. (#18637) `commonmark` has been deprecated in favor of `markdown-it-py`, and its type hints have been [removed from typeshed](https://github.com/python/typeshed/issues/13725). This switches the release script to use `markdown-it-py` instead of `commonmark` for parsing the `CHANGES.md` --- changelog.d/18637.misc | 1 + poetry.lock | 114 ++++++++++++----------------------------- pyproject.toml | 3 +- scripts-dev/release.py | 25 +++++---- 4 files changed, 50 insertions(+), 93 deletions(-) create mode 100644 changelog.d/18637.misc diff --git a/changelog.d/18637.misc b/changelog.d/18637.misc new file mode 100644 index 0000000000..ba4029cfdf --- /dev/null +++ b/changelog.d/18637.misc @@ -0,0 +1 @@ +Use `markdown-it-py` instead of `commonmark` in the release script. diff --git a/poetry.lock b/poetry.lock index d72821432c..2218b36e71 100644 --- a/poetry.lock +++ b/poetry.lock @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, @@ -366,21 +366,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - [[package]] name = "constantly" version = "15.1.0" @@ -450,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -493,7 +478,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -543,7 +528,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -880,7 +865,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1018,7 +1003,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1034,7 +1019,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, @@ -1194,14 +1179,14 @@ test = ["coverage[toml] (>=7.2.5)", "mypy (>=1.2.0)", "pytest (>=7.3.0)", "pytes [[package]] name = "markdown-it-py" -version = "2.2.0" +version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, - {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, ] [package.dependencies] @@ -1214,7 +1199,7 @@ compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0 linkify = ["linkify-it-py (>=1,<3)"] plugins = ["mdit-py-plugins"] profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] @@ -1314,7 +1299,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1546,7 +1531,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1603,8 +1588,6 @@ groups = ["main"] files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, - {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, - {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, @@ -1614,8 +1597,6 @@ files = [ {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, - {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, - {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, @@ -1625,8 +1606,6 @@ files = [ {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, - {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, - {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, @@ -1639,8 +1618,6 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, - {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, - {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, @@ -1650,8 +1627,6 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, - {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, - {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, @@ -1661,8 +1636,6 @@ files = [ {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, - {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, - {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, @@ -1672,8 +1645,6 @@ files = [ {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, - {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, - {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, @@ -1683,8 +1654,6 @@ files = [ {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, - {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, - {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, @@ -1694,15 +1663,11 @@ files = [ {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, - {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, - {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, - {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, - {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, @@ -1740,7 +1705,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1761,7 +1726,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1777,7 +1742,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2037,7 +2002,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2097,7 +2062,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2122,7 +2087,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2150,7 +2115,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2322,19 +2287,20 @@ idna2008 = ["idna"] [[package]] name = "rich" -version = "13.3.2" +version = "14.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" groups = ["dev"] files = [ - {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, - {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, + {file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"}, + {file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"}, ] [package.dependencies] -markdown-it-py = ">=2.2.0,<3.0.0" +markdown-it-py = ">=2.2.0" pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] @@ -2514,7 +2480,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, @@ -2702,7 +2668,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2718,7 +2684,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2780,7 +2746,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2917,7 +2883,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -2957,18 +2923,6 @@ files = [ [package.dependencies] types-setuptools = "*" -[[package]] -name = "types-commonmark" -version = "0.9.2.20240106" -description = "Typing stubs for commonmark" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "types-commonmark-0.9.2.20240106.tar.gz", hash = "sha256:52a062b71766d6ab258fca2d8e19fb0853796e25ca9afa9d0f67a1e42c93479f"}, - {file = "types_commonmark-0.9.2.20240106-py3-none-any.whl", hash = "sha256:606d9de1e3a96cab0b1c0b6cccf4df099116148d1d864d115fde2e27ad6877c3"}, -] - [[package]] name = "types-html5lib" version = "1.1.11.20240228" @@ -3260,7 +3214,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3404,4 +3358,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "6871453202e2ffc3c9ee5761481e0ef4ad99f08d9b8f24c6d6a64c2683683845" +content-hash = "1629969968eaaf20a8f49ee3cc0075b25e5dee7e81503fff66042ece72793103" diff --git a/pyproject.toml b/pyproject.toml index 3ab936f1c3..d27c91b193 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -326,7 +326,6 @@ lxml-stubs = ">=0.4.0" mypy = "*" mypy-zope = "*" types-bleach = ">=4.1.0" -types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" types-netaddr = ">=0.8.0.6" types-opentracing = ">=2.4.2" @@ -349,7 +348,7 @@ idna = ">=2.5" click = ">=8.1.3" # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. GitPython = ">=3.1.20" -commonmark = ">=0.9.1" +markdown-it-py = ">=3.0.0" pygithub = ">=1.55" # The following are executed as commands by the release script. twine = "*" diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 5de5814b17..73a4e7b7a9 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -36,11 +36,11 @@ from typing import Any, List, Match, Optional, Union import attr import click -import commonmark import git from click.exceptions import ClickException from git import GitCommandError, Repo from github import BadCredentialsException, Github +from markdown_it import MarkdownIt from packaging import version @@ -851,7 +851,7 @@ def get_changes_for_version(wanted_version: version.Version) -> str: # First we parse the changelog so that we can split it into sections based # on the release headings. - ast = commonmark.Parser().parse(changes) + tokens = MarkdownIt().parse(changes) @attr.s(auto_attribs=True) class VersionSection: @@ -862,19 +862,22 @@ def get_changes_for_version(wanted_version: version.Version) -> str: end_line: Optional[int] = None # Is none if its the last entry headings: List[VersionSection] = [] - for node, _ in ast.walker(): - # We look for all text nodes that are in a level 1 heading. - if node.t != "text": + for i, token in enumerate(tokens): + # We look for level 1 headings (h1 tags). + if token.type != "heading_open" or token.tag != "h1": continue - if node.parent.t != "heading" or node.parent.level != 1: - continue + # The next token should be an inline token containing the heading text + if i + 1 < len(tokens) and tokens[i + 1].type == "inline": + heading_text = tokens[i + 1].content + # The map property contains [line_begin, line_end] (0-based) + start_line = token.map[0] if token.map else 0 - # If we have a previous heading then we update its `end_line`. - if headings: - headings[-1].end_line = node.parent.sourcepos[0][0] - 1 + # If we have a previous heading then we update its `end_line`. + if headings: + headings[-1].end_line = start_line - headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1)) + headings.append(VersionSection(heading_text, start_line)) changes_by_line = changes.split("\n") From 3c13c3bebfa3a9fc8473582db2ef552794a78633 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:26:59 +0000 Subject: [PATCH 466/691] Bump base64 from 0.21.7 to 0.22.1 (#18666) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.21.7 to 0.22.1.
Changelog

Sourced from base64's changelog.

0.22.1

  • Correct the symbols used for the predefined alphabet::BIN_HEX.

0.22.0

  • DecodeSliceError::OutputSliceTooSmall is now conservative rather than precise. That is, the error will only occur if the decoded output cannot fit, meaning that Engine::decode_slice can now be used with exactly-sized output slices. As part of this, Engine::internal_decode now returns DecodeSliceError instead of DecodeError, but that is not expected to affect any external callers.
  • DecodeError::InvalidLength now refers specifically to the number of valid symbols being invalid (i.e. len % 4 == 1), rather than just the number of input bytes. This avoids confusing scenarios when based on interpretation you could make a case for either InvalidLength or InvalidByte being appropriate.
  • Decoding is somewhat faster (5-10%)
Commits
  • e144006 v0.22.1
  • 64cca59 Merge pull request #271 from JobanSD/patch-1
  • 838355e Correct BinHex 4.0 alphabet according to specifications
  • bf15ccf Merge pull request #270 from marshallpierce/mp/clippy
  • fc6aabe Appease clippy
  • 9a518a2 Merge pull request #267 from bdura/patch-1
  • d96c80f Merge branch 'marshallpierce:master' into patch-1
  • 5d70ba7 Merge pull request #269 from marshallpierce/mp/decode-precisely
  • efb6c00 Release notes
  • 2b91084 Add some tests to boost coverage
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=base64&package-manager=cargo&previous-version=0.21.7&new-version=0.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 671eec0579..abb3c6bed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,12 +65,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -380,7 +374,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "headers-core", "http", @@ -500,7 +494,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -1190,7 +1184,7 @@ version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-util", @@ -1464,7 +1458,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.7", + "base64", "blake2", "bytes", "futures", From 51296684498012217d6104deb5509f15dad93864 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 14 Jul 2025 09:55:19 -0600 Subject: [PATCH 467/691] Allow admins to see soft failed events (if they want to) (#18238) --- changelog.d/18238.feature | 1 + docs/SUMMARY.md | 1 + .../admin_api/client_server_api_extensions.md | 25 +++++ synapse/api/constants.py | 3 + synapse/appservice/api.py | 3 + synapse/events/utils.py | 23 +++++ synapse/storage/admin_client_config.py | 22 +++++ .../storage/databases/main/account_data.py | 16 +++ synapse/visibility.py | 23 ++++- tests/events/test_utils.py | 97 ++++++++++++++++++- tests/rest/client/test_relations.py | 10 +- 11 files changed, 211 insertions(+), 13 deletions(-) create mode 100644 changelog.d/18238.feature create mode 100644 docs/admin_api/client_server_api_extensions.md create mode 100644 synapse/storage/admin_client_config.py diff --git a/changelog.d/18238.feature b/changelog.d/18238.feature new file mode 100644 index 0000000000..01935a1286 --- /dev/null +++ b/changelog.d/18238.feature @@ -0,0 +1 @@ +If enabled by the user, server admins will see [soft failed](https://spec.matrix.org/v1.13/server-server-api/#soft-failure) events over the Client-Server API. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 33289ccaad..52f827c8df 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -74,6 +74,7 @@ - [Users](admin_api/user_admin_api.md) - [Server Version](admin_api/version_api.md) - [Federation](usage/administration/admin_api/federation.md) + - [Client-Server API Extensions](admin_api/client_server_api_extensions.md) - [Manhole](manhole.md) - [Monitoring](metrics-howto.md) - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md) diff --git a/docs/admin_api/client_server_api_extensions.md b/docs/admin_api/client_server_api_extensions.md new file mode 100644 index 0000000000..9cf74b23eb --- /dev/null +++ b/docs/admin_api/client_server_api_extensions.md @@ -0,0 +1,25 @@ +# Client-Server API Extensions + +Server administrators can set special account data to change how the Client-Server API behaves for +their clients. Setting the account data, or having it already set, as a non-admin has no effect. + +All configuration options can be set through the `io.element.synapse.admin_client_config` global +account data on the admin's user account. + +Example: +``` +PUT /_matrix/client/v3/user/{adminUserId}/account_data/io.element.synapse.admin_client_config +{ + "return_soft_failed_events": true +} +``` + +## See soft failed events + +Learn more about soft failure from [the spec](https://spec.matrix.org/v1.14/server-server-api/#soft-failure). + +To receive soft failed events in APIs like `/sync` and `/messages`, set `return_soft_failed_events` +to `true` in the admin client config. When `false`, the normal behaviour of these endpoints is to +exclude soft failed events. + +Default: `false` diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 47b84b02eb..f60a94ffc3 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -302,6 +302,9 @@ class AccountDataTypes: MSC4155_INVITE_PERMISSION_CONFIG: Final = ( "org.matrix.msc4155.invite_permission_config" ) + # Synapse-specific behaviour. See "Client-Server API Extensions" documentation + # in Admin API for more information. + SYNAPSE_ADMIN_CLIENT_CONFIG: Final = "io.element.synapse.admin_client_config" class HistoryVisibility: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 19322471dc..45371d6f3b 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -555,6 +555,9 @@ class ApplicationServiceApi(SimpleHttpClient): ) and service.is_interested_in_user(e.state_key) ), + # Appservices are considered 'trusted' by the admin and should have + # applicable metadata on their events. + include_admin_metadata=True, ), ) for e in events diff --git a/synapse/events/utils.py b/synapse/events/utils.py index eb18ba2db7..02ffe2c95e 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -421,11 +421,21 @@ class SerializeEventConfig: # False, that state will be removed from the event before it is returned. # Otherwise, it will be kept. include_stripped_room_state: bool = False + # When True, sets unsigned fields to help clients identify events which + # only server admins can see through other configuration. For example, + # whether an event was soft failed by the server. + include_admin_metadata: bool = False _DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig() +def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfig: + # Set the options which are only available to server admins, + # and copy the rest. + return attr.evolve(existing, include_admin_metadata=True) + + def serialize_event( e: Union[JsonDict, EventBase], time_now_ms: int, @@ -528,6 +538,9 @@ def serialize_event( d["content"] = dict(d["content"]) d["content"]["redacts"] = e.redacts + if config.include_admin_metadata and e.internal_metadata.is_soft_failed(): + d["unsigned"]["io.element.synapse.soft_failed"] = True + only_event_fields = config.only_event_fields if only_event_fields: if not isinstance(only_event_fields, list) or not all( @@ -548,6 +561,7 @@ class EventClientSerializer: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main + self._auth = hs.get_auth() self._add_extra_fields_to_unsigned_client_event_callbacks: List[ ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] @@ -576,6 +590,15 @@ class EventClientSerializer: if not isinstance(event, EventBase): return event + # Force-enable server admin metadata because the only time an event with + # relevant metadata will be when the admin requested it via their admin + # client config account data. Also, it's "just" some `unsigned` fields, so + # shouldn't cause much in terms of problems to downstream consumers. + if config.requester is not None and await self._auth.is_server_admin( + config.requester + ): + config = make_config_for_admin(config) + serialized_event = serialize_event(event, time_now, config=config) new_unsigned = {} diff --git a/synapse/storage/admin_client_config.py b/synapse/storage/admin_client_config.py new file mode 100644 index 0000000000..4359721a6d --- /dev/null +++ b/synapse/storage/admin_client_config.py @@ -0,0 +1,22 @@ +import logging +from typing import Optional + +from synapse.types import JsonMapping + +logger = logging.getLogger(__name__) + + +class AdminClientConfig: + """Class to track various Synapse-specific admin-only client-impacting config options.""" + + def __init__(self, account_data: Optional[JsonMapping]): + # Allow soft-failed events to be returned down `/sync` and other + # client APIs. `io.element.synapse.soft_failed: true` is added to the + # `unsigned` portion of the event to inform clients that the event + # is soft-failed. + self.return_soft_failed_events: bool = False + + if account_data: + self.return_soft_failed_events = account_data.get( + "return_soft_failed_events", False + ) diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index 715815cc09..be5c494ed6 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -37,6 +37,7 @@ from synapse.api.constants import AccountDataTypes from synapse.api.errors import Codes, SynapseError from synapse.replication.tcp.streams import AccountDataStream from synapse.storage._base import db_to_json +from synapse.storage.admin_client_config import AdminClientConfig from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -578,6 +579,21 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) ) return InviteRulesConfig(data) + async def get_admin_client_config_for_user(self, user_id: str) -> AdminClientConfig: + """ + Get the admin client configuration for the specified user. + + The admin client config contains Synapse-specific settings that clients running + server admin accounts can use. They have no effect on non-admin users. + + Args: + user_id: The user ID to get config for. + """ + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.SYNAPSE_ADMIN_CLIENT_CONFIG + ) + return AdminClientConfig(data) + def process_replication_rows( self, stream_name: str, diff --git a/synapse/visibility.py b/synapse/visibility.py index dc7b6e4065..280bcaa0ff 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -48,7 +48,13 @@ from synapse.logging.opentracing import trace from synapse.storage.controllers import StorageControllers from synapse.storage.databases.main import DataStore from synapse.synapse_rust.events import event_visible_to_server -from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id +from synapse.types import ( + RetentionPolicy, + StateMap, + StrCollection, + UserID, + get_domain_from_id, +) from synapse.types.state import StateFilter from synapse.util import Clock @@ -106,9 +112,18 @@ async def filter_events_for_client( of `user_id` at each event. """ # Filter out events that have been soft failed so that we don't relay them - # to clients. - events_before_filtering = events - events = [e for e in events if not e.internal_metadata.is_soft_failed()] + # to clients, unless they're a server admin and want that to happen. + # + # We copy the events list to guarantee any modifications we make will only + # happen within the function. + events_before_filtering = events.copy() + client_config = await storage.main.get_admin_client_config_for_user(user_id) + if not ( + filter_send_to_client + and client_config.return_soft_failed_events + and await storage.main.is_server_admin(UserID.from_string(user_id)) + ): + events = [e for e in events if not e.internal_metadata.is_soft_failed()] if len(events_before_filtering) != len(events): if filtered_event_logger.isEnabledFor(logging.DEBUG): filtered_event_logger.debug( diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 521251a97e..1dc6004b35 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -34,11 +34,13 @@ from synapse.events.utils import ( _split_field, clone_event, copy_and_fixup_power_levels_contents, + format_event_raw, + make_config_for_admin, maybe_upsert_event_field, prune_event, serialize_event, ) -from synapse.types import JsonDict +from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze @@ -49,7 +51,13 @@ def MockEvent(**kwargs: Any) -> EventBase: kwargs["type"] = "fake_type" if "content" not in kwargs: kwargs["content"] = {} - return make_event_from_dict(kwargs) + + # Move internal metadata out so we can call make_event properly + internal_metadata = kwargs.get("internal_metadata") + if internal_metadata is not None: + kwargs.pop("internal_metadata") + + return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata) class TestMaybeUpsertEventField(stdlib_unittest.TestCase): @@ -637,9 +645,18 @@ class CloneEventTestCase(stdlib_unittest.TestCase): class SerializeEventTestCase(stdlib_unittest.TestCase): - def serialize(self, ev: EventBase, fields: Optional[List[str]]) -> JsonDict: + def serialize( + self, + ev: EventBase, + fields: Optional[List[str]], + include_admin_metadata: bool = False, + ) -> JsonDict: return serialize_event( - ev, 1479807801915, config=SerializeEventConfig(only_event_fields=fields) + ev, + 1479807801915, + config=SerializeEventConfig( + only_event_fields=fields, include_admin_metadata=include_admin_metadata + ), ) def test_event_fields_works_with_keys(self) -> None: @@ -758,6 +775,78 @@ class SerializeEventTestCase(stdlib_unittest.TestCase): ["room_id", 4], # type: ignore[list-item] ) + def test_default_serialize_config_excludes_admin_metadata(self) -> None: + # We just really don't want this to be set to True accidentally + self.assertFalse(SerializeEventConfig().include_admin_metadata) + + def test_event_flagged_for_admins(self) -> None: + # Default behaviour should be *not* to include it + self.assertEqual( + self.serialize( + MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + internal_metadata={"soft_failed": True}, + ), + [], + ), + { + "type": "foo", + "event_id": "test", + "room_id": "!foo:bar", + "content": {"foo": "bar"}, + "unsigned": {}, + }, + ) + + # When asked though, we should set it + self.assertEqual( + self.serialize( + MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + internal_metadata={"soft_failed": True}, + ), + [], + True, + ), + { + "type": "foo", + "event_id": "test", + "room_id": "!foo:bar", + "content": {"foo": "bar"}, + "unsigned": {"io.element.synapse.soft_failed": True}, + }, + ) + + def test_make_serialize_config_for_admin_retains_other_fields(self) -> None: + non_default_config = SerializeEventConfig( + include_admin_metadata=False, # should be True in a moment + as_client_event=False, # default True + event_format=format_event_raw, # default format_event_for_client_v1 + requester=create_requester("@example:example.org"), # default None + only_event_fields=["foo"], # default None + include_stripped_room_state=True, # default False + ) + admin_config = make_config_for_admin(non_default_config) + self.assertEqual( + admin_config.as_client_event, non_default_config.as_client_event + ) + self.assertEqual(admin_config.event_format, non_default_config.event_format) + self.assertEqual(admin_config.requester, non_default_config.requester) + self.assertEqual( + admin_config.only_event_fields, non_default_config.only_event_fields + ) + self.assertEqual( + admin_config.include_stripped_room_state, + admin_config.include_stripped_room_state, + ) + self.assertTrue(admin_config.include_admin_metadata) + class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): def setUp(self) -> None: diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index f5a7602d0a..548aa15443 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -1181,7 +1181,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): bundled_aggregations, ) - self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 6) + self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 8) def test_thread(self) -> None: """ @@ -1226,21 +1226,21 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): # The "user" sent the root event and is making queries for the bundled # aggregations: they have participated. - self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 6) + self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 9) # The "user2" sent replies in the thread and is making queries for the # bundled aggregations: they have participated. # # Note that this re-uses some cached values, so the total number of # queries is much smaller. self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(True), 3, access_token=self.user2_token + RelationTypes.THREAD, _gen_assert(True), 6, access_token=self.user2_token ) # A user with no interactions with the thread: they have not participated. user3_id, user3_token = self._create_user("charlie") self.helper.join(self.room, user=user3_id, tok=user3_token) self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(False), 3, access_token=user3_token + RelationTypes.THREAD, _gen_assert(False), 6, access_token=user3_token ) def test_thread_with_bundled_aggregations_for_latest(self) -> None: @@ -1287,7 +1287,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): bundled_aggregations["latest_event"].get("unsigned"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 6) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) def test_nested_thread(self) -> None: """ From 481c4e2b5561cf1c4718ae6672e06d7a8fd154c0 Mon Sep 17 00:00:00 2001 From: V02460 Date: Mon, 14 Jul 2025 18:37:36 +0200 Subject: [PATCH 468/691] Add recaptcha_{private,public}_key_path config option (#17984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Another config option on my quest to a `*_path` variant for every secret. Adds the config options `recaptcha_private_key_path` and `recaptcha_public_key_path`. Tests and docs are included. A public key is of course no secret, but it is closely related to the private key, so it’s still useful to have a `*_path` variant for it. --- changelog.d/17984.feature | 1 + .../configuration/config_documentation.md | 30 +++++++++++++++++++ schema/synapse-config.schema.yaml | 30 +++++++++++++++++++ synapse/config/captcha.py | 26 +++++++++++++++- tests/config/test_load.py | 10 +++++++ 5 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 changelog.d/17984.feature diff --git a/changelog.d/17984.feature b/changelog.d/17984.feature new file mode 100644 index 0000000000..609f3a7cd8 --- /dev/null +++ b/changelog.d/17984.feature @@ -0,0 +1 @@ +Add `recaptcha_private_key_path` and `recaptcha_public_key_path` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1fff68ed0a..1d48f8110e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2357,6 +2357,21 @@ Example configuration: recaptcha_public_key: YOUR_PUBLIC_KEY ``` --- +### `recaptcha_public_key_path` + +*(string|null)* An alternative to [`recaptcha_public_key`](#recaptcha_public_key): allows the public key to be specified in an external file. + +The file should be a plain text file, containing only the public key. Synapse reads the public key from the given file once at startup. + +_Added in Synapse 1.134.0._ + +Defaults to `null`. + +Example configuration: +```yaml +recaptcha_public_key_path: /path/to/key/file +``` +--- ### `recaptcha_private_key` *(string|null)* This homeserver's ReCAPTCHA private key. Must be specified if [`enable_registration_captcha`](#enable_registration_captcha) is enabled. Defaults to `null`. @@ -2366,6 +2381,21 @@ Example configuration: recaptcha_private_key: YOUR_PRIVATE_KEY ``` --- +### `recaptcha_private_key_path` + +*(string|null)* An alternative to [`recaptcha_private_key`](#recaptcha_private_key): allows the private key to be specified in an external file. + +The file should be a plain text file, containing only the private key. Synapse reads the private key from the given file once at startup. + +_Added in Synapse 1.134.0._ + +Defaults to `null`. + +Example configuration: +```yaml +recaptcha_private_key_path: /path/to/key/file +``` +--- ### `enable_registration_captcha` *(boolean)* Set to `true` to require users to complete a CAPTCHA test when registering an account. Requires a valid ReCaptcha public/private key. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 26f0e7f0bc..24fbf2419a 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2692,6 +2692,21 @@ properties: default: null examples: - YOUR_PUBLIC_KEY + recaptcha_public_key_path: + type: ["string", "null"] + description: >- + An alternative to [`recaptcha_public_key`](#recaptcha_public_key): allows + the public key to be specified in an external file. + + + The file should be a plain text file, containing only the public key. + Synapse reads the public key from the given file once at startup. + + + _Added in Synapse 1.134.0._ + default: null + examples: + - /path/to/key/file recaptcha_private_key: type: ["string", "null"] description: >- @@ -2700,6 +2715,21 @@ properties: default: null examples: - YOUR_PRIVATE_KEY + recaptcha_private_key_path: + type: ["string", "null"] + description: >- + An alternative to [`recaptcha_private_key`](#recaptcha_private_key): + allows the private key to be specified in an external file. + + + The file should be a plain text file, containing only the private key. + Synapse reads the private key from the given file once at startup. + + + _Added in Synapse 1.134.0._ + default: null + examples: + - /path/to/key/file enable_registration_captcha: type: boolean description: >- diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 57d67abbc3..ea1a5e0b32 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -23,7 +23,17 @@ from typing import Any from synapse.types import JsonDict -from ._base import Config, ConfigError +from ._base import Config, ConfigError, read_file + +CONFLICTING_RECAPTCHA_PRIVATE_KEY_OPTS_ERROR = """\ +You have configured both `recaptcha_private_key` and +`recaptcha_private_key_path`. These are mutually incompatible. +""" + +CONFLICTING_RECAPTCHA_PUBLIC_KEY_OPTS_ERROR = """\ +You have configured both `recaptcha_public_key` and `recaptcha_public_key_path`. +These are mutually incompatible. +""" class CaptchaConfig(Config): @@ -38,6 +48,13 @@ class CaptchaConfig(Config): "Config options that expect an in-line secret as value are disabled", ("recaptcha_private_key",), ) + recaptcha_private_key_path = config.get("recaptcha_private_key_path") + if recaptcha_private_key_path: + if recaptcha_private_key: + raise ConfigError(CONFLICTING_RECAPTCHA_PRIVATE_KEY_OPTS_ERROR) + recaptcha_private_key = read_file( + recaptcha_private_key_path, ("recaptcha_private_key_path",) + ).strip() if recaptcha_private_key is not None and not isinstance( recaptcha_private_key, str ): @@ -50,6 +67,13 @@ class CaptchaConfig(Config): "Config options that expect an in-line secret as value are disabled", ("recaptcha_public_key",), ) + recaptcha_public_key_path = config.get("recaptcha_public_key_path") + if recaptcha_public_key_path: + if recaptcha_public_key: + raise ConfigError(CONFLICTING_RECAPTCHA_PUBLIC_KEY_OPTS_ERROR) + recaptcha_public_key = read_file( + recaptcha_public_key_path, ("recaptcha_public_key_path",) + ).strip() if recaptcha_public_key is not None and not isinstance( recaptcha_public_key, str ): diff --git a/tests/config/test_load.py b/tests/config/test_load.py index a5456ac6f8..b72365b6e3 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -138,6 +138,8 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "turn_shared_secret_path: /does/not/exist", "registration_shared_secret_path: /does/not/exist", "macaroon_secret_key_path: /does/not/exist", + "recaptcha_private_key_path: /does/not/exist", + "recaptcha_public_key_path: /does/not/exist", "form_secret_path: /does/not/exist", "worker_replication_secret_path: /does/not/exist", "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", @@ -167,6 +169,14 @@ class ConfigLoadingFileTestCase(ConfigFileTestCase): "macaroon_secret_key_path: {}", lambda c: c.key.macaroon_secret_key, ), + ( + "recaptcha_private_key_path: {}", + lambda c: c.captcha.recaptcha_private_key.encode("utf-8"), + ), + ( + "recaptcha_public_key_path: {}", + lambda c: c.captcha.recaptcha_public_key.encode("utf-8"), + ), ( "form_secret_path: {}", lambda c: c.key.form_secret.encode("utf-8"), From 945e22303c3dd5ca89b84292ec0a86d887ebaa1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 11:47:59 +0100 Subject: [PATCH 469/691] Bump phonenumbers from 9.0.8 to 9.0.9 (#18681) --- poetry.lock | 70 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2218b36e71..711dd2f016 100644 --- a/poetry.lock +++ b/poetry.lock @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, @@ -435,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -478,7 +478,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -528,7 +528,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -865,7 +865,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1003,7 +1003,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1019,7 +1019,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, @@ -1299,7 +1299,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1531,7 +1531,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1568,14 +1568,14 @@ dev = ["jinja2"] [[package]] name = "phonenumbers" -version = "9.0.8" +version = "9.0.9" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-9.0.8-py2.py3-none-any.whl", hash = "sha256:53d357111c0ead0d6408ae443613b18d3a053431ca1ddf7e881457c0969afcf9"}, - {file = "phonenumbers-9.0.8.tar.gz", hash = "sha256:16f03f2cf65b5eee99ed25827d810febcab92b5d76f977e425fcd2e4ca6d4865"}, + {file = "phonenumbers-9.0.9-py2.py3-none-any.whl", hash = "sha256:13b91aa153f87675902829b38a556bad54824f9c121b89588bbb5fa8550d97ef"}, + {file = "phonenumbers-9.0.9.tar.gz", hash = "sha256:c640545019a07e68b0bea57a5fede6eef45c7391165d28935f45615f9a567a5b"}, ] [[package]] @@ -1588,6 +1588,8 @@ groups = ["main"] files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, @@ -1597,6 +1599,8 @@ files = [ {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, @@ -1606,6 +1610,8 @@ files = [ {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, @@ -1618,6 +1624,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, @@ -1627,6 +1635,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, @@ -1636,6 +1646,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, @@ -1645,6 +1657,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, @@ -1654,6 +1668,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, @@ -1663,11 +1679,15 @@ files = [ {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, @@ -1705,7 +1725,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1726,7 +1746,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1742,7 +1762,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2002,7 +2022,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2062,7 +2082,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2087,7 +2107,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2115,7 +2135,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2480,7 +2500,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, @@ -2668,7 +2688,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2684,7 +2704,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2746,7 +2766,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2883,7 +2903,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3214,7 +3234,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From 97d2738eef1c966c841457e9628d3fadb4bf1a9d Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 15 Jul 2025 11:01:41 +0000 Subject: [PATCH 470/691] Fix CPU and database spinning when retrying sending events to servers whilst at the same time purging those events. (#18499) Fixes: #18491 Fix hotlooping due to skipped PDUs if there is still no progress to be made. This could bite if the event was purged since being skipped during catch-up. Signed-off-by: Olivier 'reivilibre --- changelog.d/18499.bugfix | 1 + synapse/federation/sender/per_destination_queue.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18499.bugfix diff --git a/changelog.d/18499.bugfix b/changelog.d/18499.bugfix new file mode 100644 index 0000000000..b07601a35d --- /dev/null +++ b/changelog.d/18499.bugfix @@ -0,0 +1 @@ +Fix CPU and database spinning when retrying sending events to servers whilst at the same time purging those events. \ No newline at end of file diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index b3f65e8237..8d6c77faee 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -129,6 +129,8 @@ class PerDestinationQueue: # The stream_ordering of the most recent PDU that was discarded due to # being in catch-up mode. + # Can be set to zero if no PDU has been discarded since the last time + # we queried for new PDUs during catch-up. self._catchup_last_skipped: int = 0 # Cache of the last successfully-transmitted stream ordering for this @@ -462,8 +464,18 @@ class PerDestinationQueue: # of a race condition, so we check that no new events have been # skipped due to us being in catch-up mode - if self._catchup_last_skipped > last_successful_stream_ordering: + if ( + self._catchup_last_skipped != 0 + and self._catchup_last_skipped > last_successful_stream_ordering + ): # another event has been skipped because we were in catch-up mode + # As an exception to this case: we can hit this branch if the + # room has been purged whilst we have been looping. + # In that case we avoid hot-looping by resetting the 'catch-up skipped + # PDU' flag. + # Then if there is still no progress to be made at the next iteration, + # we can exit catch-up mode. + self._catchup_last_skipped = 0 continue # we are done catching up! From 60be549c0ceb826946ea06a1ca6fd4a0ebcfc106 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Jul 2025 14:22:54 +0100 Subject: [PATCH 471/691] 1.134.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8948cf1e29..220e7b5613 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.134.0 (2025-07-15) + +No significant changes since 1.134.0rc1. + + + + # Synapse 1.134.0rc1 (2025-07-09) ### Features diff --git a/debian/changelog b/debian/changelog index 5b1f1f339a..49e27aed01 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.134.0) stable; urgency=medium + + * New Synapse release 1.134.0. + + -- Synapse Packaging team Tue, 15 Jul 2025 14:22:50 +0100 + matrix-synapse-py3 (1.134.0~rc1) stable; urgency=medium * New Synapse release 1.134.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 79b2cf9690..28de934602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.134.0rc1" +version = "1.134.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 78ce4dc26fc09001338a2d95817cdfe686cf19db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:42:54 +0100 Subject: [PATCH 472/691] Bump mypy from 1.13.0 to 1.16.1 (#18653) --- changelog.d/18653.misc | 1 + poetry.lock | 85 +++++++++++++--------- synapse/config/cas.py | 3 + synapse/config/key.py | 5 +- synapse/config/workers.py | 8 +- synapse/handlers/auth.py | 1 + synapse/handlers/cas.py | 3 +- synapse/handlers/presence.py | 2 +- synapse/handlers/register.py | 2 +- synapse/handlers/send_email.py | 2 +- synapse/handlers/sso.py | 2 +- synapse/http/connectproxyclient.py | 33 +++++---- synapse/metrics/_reactor_metrics.py | 2 +- synapse/module_api/__init__.py | 2 +- synapse/storage/databases/main/state.py | 7 +- synapse/storage/engines/_base.py | 6 +- synapse/util/gai_resolver.py | 2 +- synapse/util/patch_inline_callbacks.py | 7 ++ tests/federation/test_federation_sender.py | 2 +- 19 files changed, 109 insertions(+), 66 deletions(-) create mode 100644 changelog.d/18653.misc diff --git a/changelog.d/18653.misc b/changelog.d/18653.misc new file mode 100644 index 0000000000..d3a4e7c438 --- /dev/null +++ b/changelog.d/18653.misc @@ -0,0 +1 @@ +Fix typing errors with upgraded mypy version. diff --git a/poetry.lock b/poetry.lock index 711dd2f016..ed5028c017 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1425,50 +1425,51 @@ docs = ["sphinx (>=8,<9)", "sphinx-autobuild"] [[package]] name = "mypy" -version = "1.13.0" +version = "1.16.1" description = "Optional static typing for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, - {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, - {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, - {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, - {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, - {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, - {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, - {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, - {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, - {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, - {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, - {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, - {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, - {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, - {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, - {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, - {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, - {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, - {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, - {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, - {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, - {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, + {file = "mypy-1.16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b4f0fed1022a63c6fec38f28b7fc77fca47fd490445c69d0a66266c59dd0b88a"}, + {file = "mypy-1.16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86042bbf9f5a05ea000d3203cf87aa9d0ccf9a01f73f71c58979eb9249f46d72"}, + {file = "mypy-1.16.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea7469ee5902c95542bea7ee545f7006508c65c8c54b06dc2c92676ce526f3ea"}, + {file = "mypy-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:352025753ef6a83cb9e7f2427319bb7875d1fdda8439d1e23de12ab164179574"}, + {file = "mypy-1.16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ff9fa5b16e4c1364eb89a4d16bcda9987f05d39604e1e6c35378a2987c1aac2d"}, + {file = "mypy-1.16.1-cp310-cp310-win_amd64.whl", hash = "sha256:1256688e284632382f8f3b9e2123df7d279f603c561f099758e66dd6ed4e8bd6"}, + {file = "mypy-1.16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:472e4e4c100062488ec643f6162dd0d5208e33e2f34544e1fc931372e806c0cc"}, + {file = "mypy-1.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea16e2a7d2714277e349e24d19a782a663a34ed60864006e8585db08f8ad1782"}, + {file = "mypy-1.16.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08e850ea22adc4d8a4014651575567b0318ede51e8e9fe7a68f25391af699507"}, + {file = "mypy-1.16.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22d76a63a42619bfb90122889b903519149879ddbf2ba4251834727944c8baca"}, + {file = "mypy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2c7ce0662b6b9dc8f4ed86eb7a5d505ee3298c04b40ec13b30e572c0e5ae17c4"}, + {file = "mypy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:211287e98e05352a2e1d4e8759c5490925a7c784ddc84207f4714822f8cf99b6"}, + {file = "mypy-1.16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:af4792433f09575d9eeca5c63d7d90ca4aeceda9d8355e136f80f8967639183d"}, + {file = "mypy-1.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66df38405fd8466ce3517eda1f6640611a0b8e70895e2a9462d1d4323c5eb4b9"}, + {file = "mypy-1.16.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44e7acddb3c48bd2713994d098729494117803616e116032af192871aed80b79"}, + {file = "mypy-1.16.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ab5eca37b50188163fa7c1b73c685ac66c4e9bdee4a85c9adac0e91d8895e15"}, + {file = "mypy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb6229b2c9086247e21a83c309754b9058b438704ad2f6807f0d8227f6ebdd"}, + {file = "mypy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:1f0435cf920e287ff68af3d10a118a73f212deb2ce087619eb4e648116d1fe9b"}, + {file = "mypy-1.16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ddc91eb318c8751c69ddb200a5937f1232ee8efb4e64e9f4bc475a33719de438"}, + {file = "mypy-1.16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:87ff2c13d58bdc4bbe7dc0dedfe622c0f04e2cb2a492269f3b418df2de05c536"}, + {file = "mypy-1.16.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a7cfb0fe29fe5a9841b7c8ee6dffb52382c45acdf68f032145b75620acfbd6f"}, + {file = "mypy-1.16.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:051e1677689c9d9578b9c7f4d206d763f9bbd95723cd1416fad50db49d52f359"}, + {file = "mypy-1.16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d5d2309511cc56c021b4b4e462907c2b12f669b2dbeb68300110ec27723971be"}, + {file = "mypy-1.16.1-cp313-cp313-win_amd64.whl", hash = "sha256:4f58ac32771341e38a853c5d0ec0dfe27e18e27da9cdb8bbc882d2249c71a3ee"}, + {file = "mypy-1.16.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7fc688329af6a287567f45cc1cefb9db662defeb14625213a5b7da6e692e2069"}, + {file = "mypy-1.16.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e198ab3f55924c03ead626ff424cad1732d0d391478dfbf7bb97b34602395da"}, + {file = "mypy-1.16.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09aa4f91ada245f0a45dbc47e548fd94e0dd5a8433e0114917dc3b526912a30c"}, + {file = "mypy-1.16.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13c7cd5b1cb2909aa318a90fd1b7e31f17c50b242953e7dd58345b2a814f6383"}, + {file = "mypy-1.16.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:58e07fb958bc5d752a280da0e890c538f1515b79a65757bbdc54252ba82e0b40"}, + {file = "mypy-1.16.1-cp39-cp39-win_amd64.whl", hash = "sha256:f895078594d918f93337a505f8add9bd654d1a24962b4c6ed9390e12531eb31b"}, + {file = "mypy-1.16.1-py3-none-any.whl", hash = "sha256:5fc2ac4027d0ef28d6ba69a0343737a23c4d1b83672bf38d1fe237bdc0643b37"}, + {file = "mypy-1.16.1.tar.gz", hash = "sha256:6bd00a0a2094841c5e47e7374bb42b83d64c527a502e3334e1173a0c24437bab"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" +mypy_extensions = ">=1.0.0" +pathspec = ">=0.9.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.6.0" +typing_extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -1566,6 +1567,18 @@ files = [ [package.extras] dev = ["jinja2"] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "phonenumbers" version = "9.0.9" diff --git a/synapse/config/cas.py b/synapse/config/cas.py index c32bf36951..60d66d7019 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -42,6 +42,9 @@ class CasConfig(Config): self.cas_enabled = cas_config and cas_config.get("enabled", True) if self.cas_enabled: + if not isinstance(cas_config, dict): + raise ConfigError("Must be a dictionary", ("cas_config",)) + self.cas_server_url = cas_config["server_url"] # TODO Update this to a _synapse URL. diff --git a/synapse/config/key.py b/synapse/config/key.py index 29c558448b..f78ff5114f 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -212,11 +212,14 @@ class KeyConfig(Config): "Config options that expect an in-line secret as value are disabled", ("form_secret",), ) + if form_secret is not None and not isinstance(form_secret, str): + raise ConfigError("Config option must be a string", ("form_secret",)) + form_secret_path = config.get("form_secret_path", None) if form_secret_path: if form_secret: raise ConfigError(CONFLICTING_FORM_SECRET_OPTS_ERROR) - self.form_secret = read_file( + self.form_secret: Optional[str] = read_file( form_secret_path, ("form_secret_path",) ).strip() else: diff --git a/synapse/config/workers.py b/synapse/config/workers.py index c486c81274..69036f9b52 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -238,10 +238,16 @@ class WorkerConfig(Config): if worker_replication_secret_path: if worker_replication_secret: raise ConfigError(CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR) - self.worker_replication_secret = read_file( + self.worker_replication_secret: Optional[str] = read_file( worker_replication_secret_path, ("worker_replication_secret_path",) ).strip() else: + if worker_replication_secret is not None and not isinstance( + worker_replication_secret, str + ): + raise ConfigError( + "Config option must be a string", ("worker_replication_secret",) + ) self.worker_replication_secret = worker_replication_secret self.worker_name = config.get("worker_name", self.worker_app) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 9e3e70ec1c..257453674c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -174,6 +174,7 @@ def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]: # Accept both "phone" and "number" as valid keys in m.id.phone phone_number = identifier.get("phone", identifier["number"]) + assert isinstance(phone_number, str) # Convert user-provided phone number to a consistent representation msisdn = phone_number_to_msisdn(identifier["country"], phone_number) diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py index cc3d641b7d..fbe79c2e4c 100644 --- a/synapse/handlers/cas.py +++ b/synapse/handlers/cas.py @@ -378,7 +378,8 @@ class CasHandler: # Arbitrarily use the first attribute found. display_name = cas_response.attributes.get( - self._cas_displayname_attribute, [None] + self._cas_displayname_attribute, # type: ignore[arg-type] + [None], )[0] return UserAttributes(localpart=localpart, display_name=display_name) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 390cafa8f6..fb5d691d65 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1405,7 +1405,7 @@ class PresenceHandler(BasePresenceHandler): # Based on the state of each user's device calculate the new presence state. presence = _combine_device_states(devices.values()) - new_fields = {"state": presence} + new_fields: JsonDict = {"state": presence} if presence == PresenceState.ONLINE or presence == PresenceState.BUSY: new_fields["last_active_ts"] = now diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b90126f9c7..1e1f0c79c8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -557,7 +557,7 @@ class RegistrationHandler: if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) requires_invite = ( - join_rule and join_rule != JoinRules.PUBLIC + join_rule is not None and join_rule != JoinRules.PUBLIC ) # Send the invite, if necessary. diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 353991573f..92fed980e6 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -197,7 +197,7 @@ class SendEmailHandler: additional_headers: A map of additional headers to include. """ try: - from_string = self._from % {"app": app_name} + from_string = self._from % {"app": app_name} # type: ignore[operator] except (KeyError, TypeError): from_string = self._from diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 1cdf632649..0dd64618e0 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -818,7 +818,7 @@ class SsoHandler: server_name = avatar_url_parts[-2] media_id = avatar_url_parts[-1] if self._is_mine_server_name(server_name): - media = await self._media_repo.store.get_local_media(media_id) # type: ignore[has-type] + media = await self._media_repo.store.get_local_media(media_id) if media is not None and upload_name == media.upload_name: logger.info("skipping saving the user avatar") return True diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 4e4d78cb88..db803bc75a 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -33,10 +33,11 @@ from twisted.internet.interfaces import ( IAddress, IConnector, IProtocol, + IProtocolFactory, IReactorCore, IStreamClientEndpoint, ) -from twisted.internet.protocol import ClientFactory, Protocol, connectionDone +from twisted.internet.protocol import ClientFactory, connectionDone from twisted.python.failure import Failure from twisted.web import http @@ -116,11 +117,7 @@ class HTTPConnectProxyEndpoint: def __repr__(self) -> str: return "" % (self._proxy_endpoint,) - # Mypy encounters a false positive here: it complains that ClientFactory - # is incompatible with IProtocolFactory. But ClientFactory inherits from - # Factory, which implements IProtocolFactory. So I think this is a bug - # in mypy-zope. - def connect(self, protocolFactory: ClientFactory) -> "defer.Deferred[IProtocol]": # type: ignore[override] + def connect(self, protocolFactory: IProtocolFactory) -> "defer.Deferred[IProtocol]": f = HTTPProxiedClientFactory( self._host, self._port, protocolFactory, self._proxy_creds ) @@ -148,7 +145,7 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self, dst_host: bytes, dst_port: int, - wrapped_factory: ClientFactory, + wrapped_factory: IProtocolFactory, proxy_creds: Optional[ProxyCredentials], ): self.dst_host = dst_host @@ -158,7 +155,10 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self.on_connection: "defer.Deferred[None]" = defer.Deferred() def startedConnecting(self, connector: IConnector) -> None: - return self.wrapped_factory.startedConnecting(connector) + # We expect the wrapped factory to be a ClientFactory, but the generic + # interfaces only guarantee that it implements IProtocolFactory. + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.startedConnecting(connector) def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": wrapped_protocol = self.wrapped_factory.buildProtocol(addr) @@ -177,13 +177,15 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): logger.debug("Connection to proxy failed: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionFailed(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionFailed(connector, reason) def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy lost: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionLost(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionLost(connector, reason) class HTTPConnectProtocol(protocol.Protocol): @@ -208,7 +210,7 @@ class HTTPConnectProtocol(protocol.Protocol): self, host: bytes, port: int, - wrapped_protocol: Protocol, + wrapped_protocol: IProtocol, connected_deferred: defer.Deferred, proxy_creds: Optional[ProxyCredentials], ): @@ -223,11 +225,14 @@ class HTTPConnectProtocol(protocol.Protocol): ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) + # Set once we start connecting to the wrapped protocol + self.wrapped_connection_started = False + def connectionMade(self) -> None: self.http_setup_client.makeConnection(self.transport) def connectionLost(self, reason: Failure = connectionDone) -> None: - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: self.wrapped_protocol.connectionLost(reason) self.http_setup_client.connectionLost(reason) @@ -236,6 +241,8 @@ class HTTPConnectProtocol(protocol.Protocol): self.connected_deferred.errback(reason) def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: + self.wrapped_connection_started = True + assert self.transport is not None self.wrapped_protocol.makeConnection(self.transport) self.connected_deferred.callback(self.wrapped_protocol) @@ -247,7 +254,7 @@ class HTTPConnectProtocol(protocol.Protocol): def dataReceived(self, data: bytes) -> None: # if we've set up the HTTP protocol, we can send the data there - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: return self.wrapped_protocol.dataReceived(data) # otherwise, we must still be setting up the connection: send the data to the diff --git a/synapse/metrics/_reactor_metrics.py b/synapse/metrics/_reactor_metrics.py index c0a4ee16ee..fda0cd018b 100644 --- a/synapse/metrics/_reactor_metrics.py +++ b/synapse/metrics/_reactor_metrics.py @@ -33,7 +33,7 @@ from twisted.internet.asyncioreactor import AsyncioSelectorReactor from synapse.metrics._types import Collector try: - from selectors import KqueueSelector + from selectors import KqueueSelector # type: ignore[attr-defined] except ImportError: class KqueueSelector: # type: ignore[no-redef] diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index b1a2476da8..dd58d3aedc 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -284,7 +284,7 @@ class ModuleApi: try: app_name = self._hs.config.email.email_app_name - self._from_string = self._hs.config.email.email_notif_from % { + self._from_string = self._hs.config.email.email_notif_from % { # type: ignore[operator] "app": app_name } except (KeyError, TypeError): diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 788f7d1e32..cfcc731f86 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -990,11 +990,12 @@ class StateMapWrapper(Dict[StateKey, str]): raise Exception("State map was filtered and doesn't include: %s", key) return super().__getitem__(key) + @overload # type: ignore[override] + def get(self, key: StateKey, default: None = None, /) -> Optional[str]: ... @overload - def get(self, key: Tuple[str, str]) -> Optional[str]: ... - + def get(self, key: StateKey, default: str, /) -> str: ... @overload - def get(self, key: Tuple[str, str], default: Union[str, _T]) -> Union[str, _T]: ... + def get(self, key: StateKey, default: _T, /) -> Union[str, _T]: ... def get( self, key: StateKey, default: Union[str, _T, None] = None diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 9d82c59384..9fec42c2e0 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -34,9 +34,9 @@ AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$" class IsolationLevel(IntEnum): - READ_COMMITTED: int = 1 - REPEATABLE_READ: int = 2 - SERIALIZABLE: int = 3 + READ_COMMITTED = 1 + REPEATABLE_READ = 2 + SERIALIZABLE = 3 class IncorrectDatabaseSetup(RuntimeError): diff --git a/synapse/util/gai_resolver.py b/synapse/util/gai_resolver.py index fecf829ade..3c7a966e87 100644 --- a/synapse/util/gai_resolver.py +++ b/synapse/util/gai_resolver.py @@ -97,7 +97,7 @@ _GETADDRINFO_RESULT = List[ SocketKind, int, str, - Union[Tuple[str, int], Tuple[str, int, int, int]], + Union[Tuple[str, int], Tuple[str, int, int, int], Tuple[int, bytes]], ] ] diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py index beea4d2888..c776ad65b3 100644 --- a/synapse/util/patch_inline_callbacks.py +++ b/synapse/util/patch_inline_callbacks.py @@ -20,6 +20,7 @@ import functools import sys +from types import GeneratorType from typing import Any, Callable, Generator, List, TypeVar, cast from typing_extensions import ParamSpec @@ -151,6 +152,12 @@ def _check_yield_points( ) -> Generator["Deferred[object]", object, T]: gen = f(*args, **kwargs) + # We only patch if we have a native generator function, as we rely on + # `gen.gi_frame`. + if not isinstance(gen, GeneratorType): + ret = yield from gen + return ret + last_yield_line_no = gen.gi_frame.f_lineno result: Any = None while True: diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index cd906bbbc7..64e8c12817 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -672,7 +672,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): self.assertEqual(edu["edu_type"], EduTypes.DEVICE_LIST_UPDATE) c = edu["content"] if stream_id is not None: - self.assertEqual(c["prev_id"], [stream_id]) # type: ignore[unreachable] + self.assertEqual(c["prev_id"], [stream_id]) self.assertGreaterEqual(c["stream_id"], stream_id) stream_id = c["stream_id"] devices = {edu["content"]["device_id"] for edu in self.edus} From 5f027adb33bc131aaa9618602497d01c9dcd59e9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Jul 2025 15:04:29 +0100 Subject: [PATCH 473/691] Update URL Preview code to work with `lxml` 6.0.0 (#18622) --- changelog.d/18622.misc | 1 + synapse/media/preview_html.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18622.misc diff --git a/changelog.d/18622.misc b/changelog.d/18622.misc new file mode 100644 index 0000000000..9a8f36ae2c --- /dev/null +++ b/changelog.d/18622.misc @@ -0,0 +1 @@ +Update URL Preview code to work with `lxml` 6.0.0+. \ No newline at end of file diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 62ce7789be..38ae126a23 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -133,7 +133,7 @@ def decode_body( content_type: The Content-Type header. Returns: - The parsed HTML body, or None if an error occurred during processed. + The parsed HTML body, or None if an error occurred during processing. """ # If there's no body, nothing useful is going to be found. if not body: @@ -158,9 +158,31 @@ def decode_body( # Create an HTML parser. parser = etree.HTMLParser(recover=True, encoding=encoding) - # Attempt to parse the body. Returns None if the body was successfully - # parsed, but no tree was found. - return etree.fromstring(body, parser) + # Attempt to parse the body. With `lxml` 6.0.0+, this will be an empty HTML + # tree if the body was successfully parsed, but no tree was found. In + # previous `lxml` versions, `etree.fromstring` would return `None` in that + # case. + html_tree = etree.fromstring(body, parser) + + # Account for the above referenced case where `html_tree` is an HTML tree + # with an empty body. If so, return None. + if html_tree is not None and html_tree.tag == "html": + # If the tree has only a single element and it's empty, then + # return None. + body_el = html_tree.find("body") + if body_el is not None and len(html_tree) == 1: + # Extract the content of the body tag as text. + body_text = "".join(cast(Iterable[str], body_el.itertext())) + + # Strip any undecodable Unicode characters and whitespace. + body_text = body_text.strip("\ufffd").strip() + + # If there's no text left, and there were no child tags, + # then we consider the tag empty. + if not body_text and len(body_el) == 0: + return None + + return html_tree def _get_meta_tags( From 88f38ea1499665027e6b2bed6b6179e9c43a3e44 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 09:05:45 -0500 Subject: [PATCH 474/691] Correct version that `recaptcha_{private,public}_key_path` config options were introduced (#18684) Introduced in https://github.com/element-hq/synapse/pull/17984 I already see a [`v1.134.0rc1`](https://github.com/element-hq/synapse/releases/tag/v1.134.0rc1) tag from 5 days ago so I assume https://github.com/element-hq/synapse/pull/17984 will actually ship in the next release (which will be `v1.135.0`) --- changelog.d/18684.feature | 1 + docs/usage/configuration/config_documentation.md | 4 ++-- schema/synapse-config.schema.yaml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18684.feature diff --git a/changelog.d/18684.feature b/changelog.d/18684.feature new file mode 100644 index 0000000000..5099881d3a --- /dev/null +++ b/changelog.d/18684.feature @@ -0,0 +1 @@ +Add `recaptcha_private_key_path` and `recaptcha_public_key_path` config option. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1d48f8110e..0a3adf021a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2363,7 +2363,7 @@ recaptcha_public_key: YOUR_PUBLIC_KEY The file should be a plain text file, containing only the public key. Synapse reads the public key from the given file once at startup. -_Added in Synapse 1.134.0._ +_Added in Synapse 1.135.0._ Defaults to `null`. @@ -2387,7 +2387,7 @@ recaptcha_private_key: YOUR_PRIVATE_KEY The file should be a plain text file, containing only the private key. Synapse reads the private key from the given file once at startup. -_Added in Synapse 1.134.0._ +_Added in Synapse 1.135.0._ Defaults to `null`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 24fbf2419a..a2ba557fe0 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2703,7 +2703,7 @@ properties: Synapse reads the public key from the given file once at startup. - _Added in Synapse 1.134.0._ + _Added in Synapse 1.135.0._ default: null examples: - /path/to/key/file @@ -2726,7 +2726,7 @@ properties: Synapse reads the private key from the given file once at startup. - _Added in Synapse 1.134.0._ + _Added in Synapse 1.135.0._ default: null examples: - /path/to/key/file From 49cb78376ec28943f4ed8bbf4aadcb34c3091413 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Jul 2025 15:07:20 +0100 Subject: [PATCH 475/691] Advertise support for Matrix v1.12 (#18647) --- changelog.d/18647.feature | 1 + synapse/rest/client/versions.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/18647.feature diff --git a/changelog.d/18647.feature b/changelog.d/18647.feature new file mode 100644 index 0000000000..a9c273bd29 --- /dev/null +++ b/changelog.d/18647.feature @@ -0,0 +1 @@ +Advertise support for Matrix v1.12. \ No newline at end of file diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f58f11e5cc..fa39eb9e6d 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -112,6 +112,7 @@ class VersionsRestServlet(RestServlet): "v1.9", "v1.10", "v1.11", + "v1.12", ], # as per MSC1497: "unstable_features": { From b274d6561c1ebcc8a464caa9e899a849d5d0d813 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Tue, 15 Jul 2025 20:25:25 +0200 Subject: [PATCH 476/691] Document that some config options for the user directory are in violation of the Matrix spec (#18548) Fix #17534 Signed-off-by: Johannes Marbach --- changelog.d/18548.doc | 1 + docs/usage/configuration/config_documentation.md | 6 +++++- schema/synapse-config.schema.yaml | 9 ++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18548.doc diff --git a/changelog.d/18548.doc b/changelog.d/18548.doc new file mode 100644 index 0000000000..69b073221b --- /dev/null +++ b/changelog.d/18548.doc @@ -0,0 +1 @@ +Document that some config options for the user directory are in violation of the Matrix spec. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 0a3adf021a..6918559dea 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3808,7 +3808,11 @@ encryption_enabled_by_default_for_room_type: invite This setting has the following sub-options: -* `enabled` (boolean): Defines whether users can search the user directory. If false then empty responses are returned to all queries. Defaults to `true`. +* `enabled` (boolean): Defines whether users can search the user directory. If `false` then empty responses are returned to all queries. + + *Warning: While the homeserver may determine which subset of users are searched, the Matrix specification requires homeservers to include (at minimum) users visible in public rooms and users sharing a room with the requester. Using `false` improves performance but violates this requirement.* + + Defaults to `true`. * `search_all_users` (boolean): Defines whether to search all users visible to your homeserver at the time the search is performed. If set to true, will return all users known to the homeserver matching the search query. If false, search results will only contain users visible in public rooms and users sharing a room with the requester. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index a2ba557fe0..d9a5a98496 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -4719,8 +4719,15 @@ properties: enabled: type: boolean description: >- - Defines whether users can search the user directory. If false then + Defines whether users can search the user directory. If `false` then empty responses are returned to all queries. + + + *Warning: While the homeserver may determine which subset of users are + searched, the Matrix specification requires homeservers to include (at + minimum) users visible in public rooms and users sharing a room with + the requester. Using `false` improves performance but violates this + requirement.* default: true search_all_users: type: boolean From d72c278a07ac172959dd8d121ce02f7e52446395 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 15:53:56 -0500 Subject: [PATCH 477/691] Remove `allow_no_prev_events` option (MSC2716 cleanup) (#18676) This option is no longer used since we backed out the MSC2716 changes in https://github.com/matrix-org/synapse/pull/15748 and is even mentioned as a follow-up task in the PR description there. The `allow_no_prev_events` option was first introduced in https://github.com/matrix-org/synapse/pull/11243 to support MSC2716 back in the day. --- changelog.d/18676.misc | 1 + synapse/handlers/message.py | 40 +++----------------- synapse/handlers/room_member.py | 21 +---------- tests/handlers/test_message.py | 66 +++------------------------------ 4 files changed, 12 insertions(+), 116 deletions(-) create mode 100644 changelog.d/18676.misc diff --git a/changelog.d/18676.misc b/changelog.d/18676.misc new file mode 100644 index 0000000000..81306954b6 --- /dev/null +++ b/changelog.d/18676.misc @@ -0,0 +1 @@ +Remove unused `allow_no_prev_events` option when creating an event. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5d6ee6996f..6f68b8f603 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -568,7 +568,6 @@ class EventCreationHandler: requester: Requester, event_dict: dict, txn_id: Optional[str] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -594,10 +593,6 @@ class EventCreationHandler: requester event_dict: An entire event txn_id - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -717,7 +712,6 @@ class EventCreationHandler: event, unpersisted_context = await self.create_new_client_event( builder=builder, requester=requester, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, state_event_ids=state_event_ids, @@ -945,7 +939,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -962,10 +955,6 @@ class EventCreationHandler: Args: requester: The requester sending the event. event_dict: An entire event. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events. Should normally be left as None to automatically request them @@ -1051,7 +1040,6 @@ class EventCreationHandler: return await self._create_and_send_nonmember_event_locked( requester=requester, event_dict=event_dict, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, ratelimit=ratelimit, @@ -1065,7 +1053,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -1097,7 +1084,6 @@ class EventCreationHandler: requester, event_dict, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, outlier=outlier, @@ -1180,7 +1166,6 @@ class EventCreationHandler: self, builder: EventBuilder, requester: Optional[Requester] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -1200,10 +1185,6 @@ class EventCreationHandler: Args: builder: requester: - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -1241,7 +1222,6 @@ class EventCreationHandler: if state_event_ids is not None: # Do a quick check to make sure that prev_event_ids is present to # make the type-checking around `builder.build` happy. - # prev_event_ids could be an empty array though. assert prev_event_ids is not None temp_event = await builder.build( @@ -1269,24 +1249,14 @@ class EventCreationHandler: else: prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + # We now ought to have some `prev_events` (unless it's a create event). + # # Do a quick sanity check here, rather than waiting until we've created the # event and then try to auth it (which fails with a somewhat confusing "No # create event in auth events") - if allow_no_prev_events: - # We allow events with no `prev_events` but it better have some `auth_events` - assert ( - builder.type == EventTypes.Create - # Allow an event to have empty list of prev_event_ids - # only if it has auth_event_ids. - or auth_event_ids - ), ( - "Attempting to create a non-m.room.create event with no prev_events or auth_event_ids" - ) - else: - # we now ought to have some prev_events (unless it's a create event). - assert builder.type == EventTypes.Create or prev_event_ids, ( - "Attempting to create a non-m.room.create event with no prev_events" - ) + assert builder.type == EventTypes.Create or len(prev_event_ids) > 0, ( + "Attempting to create an event with no prev_events" + ) if for_batch: assert prev_event_ids is not None diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index cf9db7b018..b6800a9f63 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -388,11 +388,11 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): async def _local_membership_update( self, + *, requester: Requester, target: UserID, room_id: str, membership: str, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -414,11 +414,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): desired membership event. room_id: membership: - - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -486,7 +481,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): "origin_server_ts": origin_server_ts, }, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -583,7 +577,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -607,10 +600,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -680,7 +669,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room=new_room, require_consent=require_consent, outlier=outlier, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -703,7 +691,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -729,10 +716,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -933,7 +916,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): ) # InviteRule.IGNORE is handled at the sync layer. - # An empty prev_events list is allowed as long as the auth_event_ids are present if prev_event_ids is not None: return await self._local_membership_update( requester=requester, @@ -942,7 +924,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): membership=effective_membership_state, txn_id=txn_id, ratelimit=ratelimit, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 76ab83d1f7..990c906d2c 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -204,46 +204,19 @@ class EventCreationTestCase(unittest.HomeserverTestCase): self.assertEqual(len(events), 2) self.assertEqual(events[0].event_id, events[1].event_id) - def test_when_empty_prev_events_allowed_create_event_with_empty_prev_events( + def test_reject_event_with_empty_prev_events( self, ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events (only auth_events). """ - # Create a member event we can use as an auth_event - memberEvent, _ = self._create_and_persist_member_event() - - # Try to create the event with empty prev_events bit with some auth_events - event, _ = self.get_success( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - # Empty prev_events is the key thing we're testing here - prev_event_ids=[], - # But with some auth_events - auth_event_ids=[memberEvent.event_id], - # Allow no prev_events! - allow_no_prev_events=True, - ) - ) - self.assertIsNotNone(event) - - def test_when_empty_prev_events_not_allowed_reject_event_with_empty_prev_events( - self, - ) -> None: - """When we set allow_no_prev_events=False, shouldn't be able to create a - event without any prev_events even if it has auth_events. Expect an - exception to be raised. + Shouldn't be able to create an event without any `prev_events` even if it has + `auth_events`. Expect an exception to be raised. """ # Create a member event we can use as an auth_event memberEvent, _ = self._create_and_persist_member_event() # Try to create the event with empty prev_events but with some auth_events + # + # We expect the test to fail because empty prev_events are not allowed self.get_failure( self.handler.create_event( self.requester, @@ -257,35 +230,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): prev_event_ids=[], # But with some auth_events auth_event_ids=[memberEvent.event_id], - # We expect the test to fail because empty prev_events are not - # allowed here! - allow_no_prev_events=False, - ), - AssertionError, - ) - - def test_when_empty_prev_events_allowed_reject_event_with_empty_prev_events_and_auth_events( - self, - ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events or auth_events. Expect an exception to be - raised. - """ - # Try to create the event with empty prev_events and empty auth_events - self.get_failure( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - prev_event_ids=[], - # The event should be rejected when there are no auth_events - auth_event_ids=[], - # Allow no prev_events! - allow_no_prev_events=True, ), AssertionError, ) From fc10a5ee29825e2eea76bc073669c5494f336d33 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 15:55:23 -0500 Subject: [PATCH 478/691] Refactor `Measure` block metrics to be homeserver-scoped (v2) (#18601) Refactor `Measure` block metrics to be homeserver-scoped (add `server_name` label to block metrics). Part of https://github.com/element-hq/synapse/issues/18592 ### Testing strategy #### See behavior of previous `metrics` listener 1. Add the `metrics` listener in your `homeserver.yaml` ```yaml listeners: - port: 9323 type: metrics bind_addresses: ['127.0.0.1'] ``` 1. Start the homeserver: `poetry run synapse_homeserver --config-path homeserver.yaml` 1. Fetch `http://localhost:9323/metrics` 1. Observe response includes the block metrics (`synapse_util_metrics_block_count`, `synapse_util_metrics_block_in_flight`, etc) #### See behavior of the `http` `metrics` resource 1. Add the `metrics` resource to a new or existing `http` listeners in your `homeserver.yaml` ```yaml listeners: - port: 9322 type: http bind_addresses: ['127.0.0.1'] resources: - names: [metrics] compress: false ``` 1. Start the homeserver: `poetry run synapse_homeserver --config-path homeserver.yaml` 1. Fetch `http://localhost:9322/_synapse/metrics` (it's just a `GET` request so you can even do in the browser) 1. Observe response includes the block metrics (`synapse_util_metrics_block_count`, `synapse_util_metrics_block_in_flight`, etc) --- changelog.d/18601.misc | 1 + synapse/federation/send_queue.py | 4 +- synapse/federation/sender/__init__.py | 6 +- .../federation/sender/transaction_manager.py | 4 +- synapse/handlers/appservice.py | 11 +- synapse/handlers/delayed_events.py | 5 +- synapse/handlers/device.py | 5 +- synapse/handlers/message.py | 8 +- synapse/handlers/presence.py | 14 +- synapse/handlers/sync.py | 17 ++- synapse/handlers/typing.py | 9 +- synapse/handlers/user_directory.py | 4 +- .../federation/matrix_federation_agent.py | 7 + .../http/federation/well_known_resolver.py | 15 ++- synapse/http/matrixfederationclient.py | 7 +- synapse/metrics/__init__.py | 40 +++++- .../callbacks/media_repository_callbacks.py | 13 +- .../callbacks/ratelimit_callbacks.py | 7 +- .../callbacks/spamchecker_callbacks.py | 85 ++++++++++-- synapse/push/bulk_push_rule_evaluator.py | 3 +- synapse/replication/http/federation.py | 5 +- synapse/replication/http/send_event.py | 5 +- synapse/replication/http/send_events.py | 5 +- synapse/replication/tcp/client.py | 7 +- synapse/replication/tcp/resource.py | 7 +- synapse/state/__init__.py | 12 +- synapse/storage/_base.py | 1 + synapse/storage/controllers/persist_events.py | 25 +++- synapse/storage/controllers/state.py | 5 +- .../storage/databases/main/events_worker.py | 4 +- synapse/storage/databases/main/roommember.py | 6 +- synapse/util/metrics.py | 121 +++++++++++++----- tests/handlers/test_typing.py | 1 + .../test_matrix_federation_agent.py | 4 + .../test_federation_sender_shard.py | 1 + 35 files changed, 380 insertions(+), 94 deletions(-) create mode 100644 changelog.d/18601.misc diff --git a/changelog.d/18601.misc b/changelog.d/18601.misc new file mode 100644 index 0000000000..c8893c2806 --- /dev/null +++ b/changelog.d/18601.misc @@ -0,0 +1 @@ +Refactor `Measure` block metrics to be homeserver-scoped. diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index b5c9fcff7c..e309836a52 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -156,7 +156,9 @@ class FederationRemoteSendQueue(AbstractFederationSender): def _clear_queue_before_pos(self, position_to_delete: int) -> None: """Clear all the queues from before a given position""" - with Measure(self.clock, "send_queue._clear"): + with Measure( + self.clock, name="send_queue._clear", server_name=self.server_name + ): # Delete things out of presence maps keys = self.presence_destinations.keys() i = self.presence_destinations.bisect_left(position_to_delete) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 2eef7b707d..8010cc62f3 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -657,7 +657,11 @@ class FederationSender(AbstractFederationSender): logger.debug( "Handling %i events in room %s", len(events), events[0].room_id ) - with Measure(self.clock, "handle_room_events"): + with Measure( + self.clock, + name="handle_room_events", + server_name=self.server_name, + ): for event in events: await handle_event(event) diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index d8a3eaa525..21e2fed085 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -58,7 +58,7 @@ class TransactionManager: """ def __init__(self, hs: "synapse.server.HomeServer"): - self._server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self._store = hs.get_datastores().main self._transaction_actions = TransactionActions(self._store) @@ -116,7 +116,7 @@ class TransactionManager: transaction = Transaction( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, - origin=self._server_name, + origin=self.server_name, destination=destination, pdus=[p.get_pdu_json() for p in pdus], edus=[edu.get_dict() for edu in edus], diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f3bbdb5a05..5aefc73aba 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -73,6 +73,7 @@ events_processed_counter = Counter("synapse_handlers_appservice_events_processed class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() @@ -120,7 +121,9 @@ class ApplicationServicesHandler: @wrap_as_background_process("notify_interested_services") async def _notify_interested_services(self, max_token: RoomStreamToken) -> None: - with Measure(self.clock, "notify_interested_services"): + with Measure( + self.clock, name="notify_interested_services", server_name=self.server_name + ): self.is_processing = True try: upper_bound = -1 @@ -329,7 +332,11 @@ class ApplicationServicesHandler: users: Collection[Union[str, UserID]], ) -> None: logger.debug("Checking interested services for %s", stream_key) - with Measure(self.clock, "notify_interested_services_ephemeral"): + with Measure( + self.clock, + name="notify_interested_services_ephemeral", + server_name=self.server_name, + ): for service in services: if stream_key == StreamKeyType.TYPING: # Note that we don't persist the token (via set_appservice_stream_type_pos) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 80cb1cec9b..beb0e819c2 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -54,6 +54,7 @@ logger = logging.getLogger(__name__) class DelayedEventsHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._config = hs.config @@ -159,7 +160,9 @@ class DelayedEventsHandler: # Loop round handling deltas until we're up to date while True: - with Measure(self._clock, "delayed_events_delta"): + with Measure( + self._clock, name="delayed_events_delta", server_name=self.server_name + ): room_max_stream_ordering = self._store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index c6e44dae6a..e825626558 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -526,6 +526,8 @@ class DeviceHandler(DeviceWorkerHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.federation_sender = hs.get_federation_sender() self._account_data_handler = hs.get_account_data_handler() self._storage_controllers = hs.get_storage_controllers() @@ -1215,7 +1217,8 @@ class DeviceListUpdater(DeviceListWorkerUpdater): def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): self.store = hs.get_datastores().main self.federation = hs.get_federation_client() - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler self._notifier = hs.get_notifier() diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6f68b8f603..7c76c187bc 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -476,16 +476,16 @@ _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY = 7 * 24 * 60 * 60 * 1000 class EventCreationHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.validator = EventValidator() + self.event_builder_factory = hs.get_event_builder_factory() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.auth_blocking = hs.get_auth_blocking() self._event_auth_handler = hs.get_event_auth_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() - self.clock = hs.get_clock() - self.validator = EventValidator() self.profile_handler = hs.get_profile_handler() - self.event_builder_factory = hs.get_event_builder_factory() - self.server_name = hs.hostname self.notifier = hs.get_notifier() self.config = hs.config self.require_membership_for_aliases = ( diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index fb5d691d65..c652e333a6 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -747,6 +747,7 @@ class WorkerPresenceHandler(BasePresenceHandler): class PresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.wheel_timer: WheelTimer[str] = WheelTimer() self.notifier = hs.get_notifier() @@ -941,7 +942,9 @@ class PresenceHandler(BasePresenceHandler): now = self.clock.time_msec() - with Measure(self.clock, "presence_update_states"): + with Measure( + self.clock, name="presence_update_states", server_name=self.server_name + ): # NOTE: We purposefully don't await between now and when we've # calculated what we want to do with the new states, to avoid races. @@ -1497,7 +1500,9 @@ class PresenceHandler(BasePresenceHandler): async def _unsafe_process(self) -> None: # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "presence_delta"): + with Measure( + self.clock, name="presence_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return @@ -1759,6 +1764,7 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # Same with get_presence_router: # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler + self.server_name = hs.hostname self.get_presence_handler = hs.get_presence_handler self.get_presence_router = hs.get_presence_router self.clock = hs.get_clock() @@ -1792,7 +1798,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): user_id = user.to_string() stream_change_cache = self.store.presence_stream_cache - with Measure(self.clock, "presence.get_new_events"): + with Measure( + self.clock, name="presence.get_new_events", server_name=self.server_name + ): if from_key is not None: from_key = int(from_key) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a400e63fd5..7b99defac1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -329,6 +329,7 @@ class E2eeSyncResult: class SyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main self.notifier = hs.get_notifier() @@ -710,7 +711,9 @@ class SyncHandler: sync_config = sync_result_builder.sync_config - with Measure(self.clock, "ephemeral_by_room"): + with Measure( + self.clock, name="ephemeral_by_room", server_name=self.server_name + ): typing_key = since_token.typing_key if since_token else 0 room_ids = sync_result_builder.joined_room_ids @@ -783,7 +786,9 @@ class SyncHandler: and current token to send down to clients. newly_joined_room """ - with Measure(self.clock, "load_filtered_recents"): + with Measure( + self.clock, name="load_filtered_recents", server_name=self.server_name + ): timeline_limit = sync_config.filter_collection.timeline_limit() block_all_timeline = ( sync_config.filter_collection.blocks_all_room_timeline() @@ -1174,7 +1179,9 @@ class SyncHandler: # updates even if they occurred logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - with Measure(self.clock, "compute_state_delta"): + with Measure( + self.clock, name="compute_state_delta", server_name=self.server_name + ): # The memberships needed for events in the timeline. # Only calculated when `lazy_load_members` is on. members_to_fetch: Optional[Set[str]] = None @@ -1791,7 +1798,9 @@ class SyncHandler: # the DB. return RoomNotifCounts.empty() - with Measure(self.clock, "unread_notifs_for_room_id"): + with Measure( + self.clock, name="unread_notifs_for_room_id", server_name=self.server_name + ): return await self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 8d693fee30..bbef3a59a5 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -503,6 +503,7 @@ class TypingWriterHandler(FollowerTypingHandler): class TypingNotificationEventSource(EventSource[int, JsonMapping]): def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() # We can't call get_typing_handler here because there's a cycle: @@ -535,7 +536,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): appservice may be interested in. * The latest known room serial. """ - with Measure(self.clock, "typing.get_new_events_as"): + with Measure( + self.clock, name="typing.get_new_events_as", server_name=self.server_name + ): handler = self.get_typing_handler() events = [] @@ -571,7 +574,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): Find typing notifications for given rooms (> `from_token` and <= `to_token`) """ - with Measure(self.clock, "typing.get_new_events"): + with Measure( + self.clock, name="typing.get_new_events", server_name=self.server_name + ): from_key = int(from_key) handler = self.get_typing_handler() diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 1f692c79a0..5f9e96706a 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -237,7 +237,9 @@ class UserDirectoryHandler(StateDeltasHandler): # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "user_dir_delta"): + with Measure( + self.clock, name="user_dir_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self.pos == room_max_stream_ordering: return diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index a7742fcea8..4a47665abd 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -92,6 +92,7 @@ class MatrixFederationAgent: def __init__( self, + server_name: str, reactor: ISynapseReactor, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, @@ -100,6 +101,11 @@ class MatrixFederationAgent: _srv_resolver: Optional[SrvResolver] = None, _well_known_resolver: Optional[WellKnownResolver] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + """ + # proxy_reactor is not blocklisting reactor proxy_reactor = reactor @@ -127,6 +133,7 @@ class MatrixFederationAgent: if _well_known_resolver is None: _well_known_resolver = WellKnownResolver( + server_name, reactor, agent=BlocklistingAgentWrapper( ProxyAgent( diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 9a6bac7281..911cbac7ea 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -91,12 +91,19 @@ class WellKnownResolver: def __init__( self, + server_name: str, reactor: IReactorTime, agent: IAgent, user_agent: bytes, well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None, had_well_known_cache: Optional[TTLCache[bytes, bool]] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + """ + + self.server_name = server_name self._reactor = reactor self._clock = Clock(reactor) @@ -134,7 +141,13 @@ class WellKnownResolver: # TODO: should we linearise so that we don't end up doing two .well-known # requests for the same server in parallel? try: - with Measure(self._clock, "get_well_known"): + with Measure( + self._clock, + name="get_well_known", + # This should be our homeserver where the the code is running (used to + # label metrics) + server_name=self.server_name, + ): result: Optional[bytes] cache_period: float diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 97a863a118..67ea9cdb81 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -417,6 +417,7 @@ class MatrixFederationHttpClient: if hs.get_instance_name() in outbound_federation_restricted_to: # Talk to federation directly federation_agent: IAgent = MatrixFederationAgent( + self.server_name, self.reactor, tls_client_options_factory, user_agent.encode("ascii"), @@ -697,7 +698,11 @@ class MatrixFederationHttpClient: outgoing_requests_counter.labels(request.method).inc() try: - with Measure(self.clock, "outbound_request"): + with Measure( + self.clock, + name="outbound_request", + server_name=self.server_name, + ): # we don't want all the fancy cookie and redirect handling # that treq.request gives: just use the raw Agent. diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 86ac2c2395..7e508dba05 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -66,6 +66,21 @@ all_gauges: Dict[str, Collector] = {} HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") +SERVER_NAME_LABEL = "server_name" +""" +The `server_name` label is used to identify the homeserver that the metrics correspond +to. Because we support multiple instances of Synapse running in the same process and all +metrics are in a single global `REGISTRY`, we need to manually label any metrics. + +In the case of a Synapse homeserver, this should be set to the homeserver name +(`hs.hostname`). + +We're purposely not using the `instance` label for this purpose as that should be "The +: part of the target's URL that was scraped.". Also: "In Prometheus +terms, an endpoint you can scrape is called an *instance*, usually corresponding to a +single process." (source: https://prometheus.io/docs/concepts/jobs_instances/) +""" + class _RegistryProxy: @staticmethod @@ -192,7 +207,16 @@ class InFlightGauge(Generic[MetricsEntry], Collector): same key. Note that `callback` may be called on a separate thread. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) + with self._lock: self._registrations.setdefault(key, set()).add(callback) @@ -201,7 +225,17 @@ class InFlightGauge(Generic[MetricsEntry], Collector): key: Tuple[str, ...], callback: Callable[[MetricsEntry], None], ) -> None: - """Registers that we've exited a block with labels `key`.""" + """ + Registers that we've exited a block with labels `key`. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback + """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) with self._lock: self._registrations.setdefault(key, set()).discard(callback) @@ -225,7 +259,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): with self._lock: callbacks = set(self._registrations[key]) - in_flight.add_metric(key, len(callbacks)) + in_flight.add_metric(labels=key, value=len(callbacks)) metrics = self._metrics_class() metrics_by_key[key] = metrics @@ -239,7 +273,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): "_".join([self.name, name]), "", labels=self.labels ) for key, metrics in metrics_by_key.items(): - gauge.add_metric(key, getattr(metrics, name)) + gauge.add_metric(labels=key, value=getattr(metrics, name)) yield gauge def _register_with_collector(self) -> None: diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py index 6fa80a8eab..2ab65f9fd6 100644 --- a/synapse/module_api/callbacks/media_repository_callbacks.py +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -31,6 +31,7 @@ IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK = Callable[[str, int], Awaitabl class MediaRepositoryModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._get_media_config_for_user_callbacks: List[ GET_MEDIA_CONFIG_FOR_USER_CALLBACK @@ -57,7 +58,11 @@ class MediaRepositoryModuleApiCallbacks: async def get_media_config_for_user(self, user_id: str) -> Optional[JsonDict]: for callback in self._get_media_config_for_user_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Optional[JsonDict] = await delay_cancellation(callback(user_id)) if res: return res @@ -68,7 +73,11 @@ class MediaRepositoryModuleApiCallbacks: self, user_id: str, size: int ) -> bool: for callback in self._is_user_allowed_to_upload_media_of_size_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: bool = await delay_cancellation(callback(user_id, size)) if not res: return res diff --git a/synapse/module_api/callbacks/ratelimit_callbacks.py b/synapse/module_api/callbacks/ratelimit_callbacks.py index 64f9cc81e8..a580ea7d7c 100644 --- a/synapse/module_api/callbacks/ratelimit_callbacks.py +++ b/synapse/module_api/callbacks/ratelimit_callbacks.py @@ -43,6 +43,7 @@ GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK = Callable[ class RatelimitModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._get_ratelimit_override_for_user_callbacks: List[ GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK @@ -64,7 +65,11 @@ class RatelimitModuleApiCallbacks: self, user_id: str, limiter_name: str ) -> Optional[RatelimitOverride]: for callback in self._get_ratelimit_override_for_user_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Optional[RatelimitOverride] = await delay_cancellation( callback(user_id, limiter_name) ) diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index c43824f213..428e733979 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -356,6 +356,7 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM: Literal["NOT_SPAM"] = "NOT_SPAM" def __init__(self, hs: "synapse.server.HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] @@ -490,7 +491,11 @@ class SpamCheckerModuleApiCallbacks: generally discouraged as it doesn't support internationalization. """ for callback in self._check_event_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(event)) if res is False or res == self.NOT_SPAM: # This spam-checker accepts the event. @@ -543,7 +548,11 @@ class SpamCheckerModuleApiCallbacks: True if the event should be silently dropped """ for callback in self._should_drop_federated_event_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -565,7 +574,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, [Codes, Dict] otherwise. """ for callback in self._user_may_join_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(user_id, room_id, is_invited)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is True or res is self.NOT_SPAM: @@ -604,7 +617,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, invitee_userid, room_id) ) @@ -643,7 +660,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._federated_user_may_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(event)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is True or res is self.NOT_SPAM: @@ -686,7 +707,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_send_3pid_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, medium, address, room_id) ) @@ -722,7 +747,11 @@ class SpamCheckerModuleApiCallbacks: room_config: The room creation configuration which is the body of the /createRoom request """ for callback in self._user_may_create_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): checker_args = inspect.signature(callback) # Also ensure backwards compatibility with spam checker callbacks # that don't expect the room_config argument. @@ -786,7 +815,11 @@ class SpamCheckerModuleApiCallbacks: content: The content of the state event """ for callback in self._user_may_send_state_event_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): # We make a copy of the content to ensure that the spam checker cannot modify it. res = await delay_cancellation( callback(user_id, room_id, event_type, state_key, deepcopy(content)) @@ -814,7 +847,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._user_may_create_room_alias_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_alias)) if res is True or res is self.NOT_SPAM: continue @@ -847,7 +884,11 @@ class SpamCheckerModuleApiCallbacks: room_id: The ID of the room that would be published """ for callback in self._user_may_publish_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_id)) if res is True or res is self.NOT_SPAM: continue @@ -889,7 +930,11 @@ class SpamCheckerModuleApiCallbacks: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): checker_args = inspect.signature(callback) # Make a copy of the user profile object to ensure the spam checker cannot # modify it. @@ -938,7 +983,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_registration_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): behaviour = await delay_cancellation( callback(email_threepid, username, request_info, auth_provider_id) ) @@ -980,7 +1029,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_media_file_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(file_wrapper, file_info)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is False or res is self.NOT_SPAM: @@ -1027,7 +1080,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_login_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback( user_id, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 1f4f5b90c3..f20b98f73f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -129,7 +129,8 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs self.store = hs.get_datastores().main - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self._event_auth_handler = hs.get_event_auth_handler() self.should_calculate_push_rules = self.hs.config.push.enable_push diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index f3f8ddfd65..c29ed8d149 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -76,6 +76,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() @@ -122,7 +123,9 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_fed_send_events_parse"): + with Measure( + self.clock, name="repl_fed_send_events_parse", server_name=self.server_name + ): room_id = content["room_id"] backfilled = content["backfilled"] diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py index 01952a8d59..edda419a03 100644 --- a/synapse/replication/http/send_event.py +++ b/synapse/replication/http/send_event.py @@ -76,6 +76,7 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -121,7 +122,9 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict, event_id: str ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_event_parse"): + with Measure( + self.clock, name="repl_send_event_parse", server_name=self.server_name + ): event_dict = content["event"] room_ver = KNOWN_ROOM_VERSIONS[content["room_version"]] internal_metadata = content["internal_metadata"] diff --git a/synapse/replication/http/send_events.py b/synapse/replication/http/send_events.py index d965ce5492..15e363b3eb 100644 --- a/synapse/replication/http/send_events.py +++ b/synapse/replication/http/send_events.py @@ -77,6 +77,7 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -122,7 +123,9 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, payload: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_events_parse"): + with Measure( + self.clock, name="repl_send_events_parse", server_name=self.server_name + ): events_and_context = [] events = payload["events"] rooms = set() diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 0bd5478cd3..e71588f3de 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -75,6 +75,7 @@ class ReplicationDataHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._reactor = hs.get_reactor() @@ -342,7 +343,11 @@ class ReplicationDataHandler: waiting_list.add((position, deferred)) # We measure here to get in flight counts and average waiting time. - with Measure(self._clock, "repl.wait_for_stream_position"): + with Measure( + self._clock, + name="repl.wait_for_stream_position", + server_name=self.server_name, + ): logger.info( "Waiting for repl stream %r to reach %s (%s); currently at: %s", stream_name, diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index d647a2b332..0080a76f6f 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -78,6 +78,7 @@ class ReplicationStreamer: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() @@ -155,7 +156,11 @@ class ReplicationStreamer: while self.pending_updates: self.pending_updates = False - with Measure(self.clock, "repl.stream.get_updates"): + with Measure( + self.clock, + name="repl.stream.get_updates", + server_name=self.server_name, + ): all_streams = self.streams if self._replication_torture_level is not None: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 1c3e5d00a9..9c24525845 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -189,7 +189,8 @@ class StateHandler: """ def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self.hs = hs @@ -631,6 +632,7 @@ class StateResolutionHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.clock = hs.get_clock() self.resolve_linearizer = Linearizer(name="state_resolve_lock") @@ -747,7 +749,9 @@ class StateResolutionHandler: # which will be used as a cache key for future resolutions, but # not get persisted. - with Measure(self.clock, "state.create_group_ids"): + with Measure( + self.clock, name="state.create_group_ids", server_name=self.server_name + ): cache = _make_state_cache_entry(new_state, state_groups_ids) self._state_cache[group_names] = cache @@ -785,7 +789,9 @@ class StateResolutionHandler: a map from (type, state_key) to event_id. """ try: - with Measure(self.clock, "state._resolve_events") as m: + with Measure( + self.clock, name="state._resolve_events", server_name=self.server_name + ) as m: room_version_obj = KNOWN_ROOM_VERSIONS[room_version] if room_version_obj.state_res == StateResolutionVersions.V1: return await v1.resolve_events_with_store( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2309b1648e..548e7df930 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,6 +55,7 @@ class SQLBaseStore(metaclass=ABCMeta): hs: "HomeServer", ): self.hs = hs + self.server_name = hs.hostname self._clock = hs.get_clock() self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index f5131fe291..9f54430a22 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -337,6 +337,7 @@ class EventsPersistenceStorageController: assert stores.persist_events self.persist_events_store = stores.persist_events + self.server_name = hs.hostname self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id @@ -616,7 +617,11 @@ class EventsPersistenceStorageController: state_delta_for_room = None if not backfilled: - with Measure(self._clock, "_calculate_state_and_extrem"): + with Measure( + self._clock, + name="_calculate_state_and_extrem", + server_name=self.server_name, + ): # Work out the new "current state" for the room. # We do this by working out what the new extremities are and then # calculating the state from that. @@ -627,7 +632,11 @@ class EventsPersistenceStorageController: room_id, chunk ) - with Measure(self._clock, "calculate_chain_cover_index_for_events"): + with Measure( + self._clock, + name="calculate_chain_cover_index_for_events", + server_name=self.server_name, + ): # We now calculate chain ID/sequence numbers for any state events we're # persisting. We ignore out of band memberships as we're not in the room # and won't have their auth chain (we'll fix it up later if we join the @@ -719,7 +728,11 @@ class EventsPersistenceStorageController: break logger.debug("Calculating state delta for room %s", room_id) - with Measure(self._clock, "persist_events.get_new_state_after_events"): + with Measure( + self._clock, + name="persist_events.get_new_state_after_events", + server_name=self.server_name, + ): res = await self._get_new_state_after_events( room_id, ev_ctx_rm, @@ -746,7 +759,11 @@ class EventsPersistenceStorageController: # removed keys entirely. delta = DeltaState([], delta_ids) elif current_state is not None: - with Measure(self._clock, "persist_events.calculate_state_delta"): + with Measure( + self._clock, + name="persist_events.calculate_state_delta", + server_name=self.server_name, + ): delta = await self._calculate_state_delta(room_id, current_state) if delta: diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index f28f5d7e03..d79791fed4 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -68,6 +68,7 @@ class StateStorageController: """ def __init__(self, hs: "HomeServer", stores: "Databases"): + self.server_name = hs.hostname self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores @@ -812,7 +813,9 @@ class StateStorageController: state_group = object() assert state_group is not None - with Measure(self._clock, "get_joined_hosts"): + with Measure( + self._clock, name="get_joined_hosts", server_name=self.server_name + ): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 9cc0493307..2929b1d57a 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1246,7 +1246,9 @@ class EventsWorkerStore(SQLBaseStore): to event row. Note that it may well contain additional events that were not part of this request. """ - with Measure(self._clock, "_fetch_event_list"): + with Measure( + self._clock, name="_fetch_event_list", server_name=self.server_name + ): try: events_to_fetch = { event_id for events, _ in event_list for event_id in events diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 7ca73abb83..ce77e0b0d6 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -983,7 +983,11 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): `_get_user_ids_from_membership_event_ids` for any uncached events. """ - with Measure(self._clock, "get_joined_user_ids_from_state"): + with Measure( + self._clock, + name="get_joined_user_ids_from_state", + server_name=self.server_name, + ): users_in_room = set() member_event_ids = [ e_id for key, e_id in state.items() if key[0] == EventTypes.Member diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6a389f7a7e..608a4d4848 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -41,59 +41,100 @@ from synapse.logging.context import ( LoggingContext, current_context, ) -from synapse.metrics import InFlightGauge +from synapse.metrics import SERVER_NAME_LABEL, InFlightGauge from synapse.util import Clock logger = logging.getLogger(__name__) -block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"]) +# Metrics to see the number of and how much time is spend in various blocks of code. +# +block_counter = Counter( + "synapse_util_metrics_block_count", + documentation="The number of times this block has been called.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The number of times this block has been called.""" -block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"]) +block_timer = Counter( + "synapse_util_metrics_block_time_seconds", + documentation="The cumulative time spent executing this block across all calls, in seconds.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The cumulative time spent executing this block across all calls, in seconds.""" block_ru_utime = Counter( - "synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_utime_seconds", + documentation="Resource usage: user CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: user CPU time in seconds used in this block""" block_ru_stime = Counter( - "synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_stime_seconds", + documentation="Resource usage: system CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: system CPU time in seconds used in this block""" block_db_txn_count = Counter( - "synapse_util_metrics_block_db_txn_count", "", ["block_name"] + "synapse_util_metrics_block_db_txn_count", + documentation="Number of database transactions completed in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Number of database transactions completed in this block""" # seconds spent waiting for db txns, excluding scheduling time, in this block block_db_txn_duration = Counter( - "synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_txn_duration_seconds", + documentation="Seconds spent waiting for database txns, excluding scheduling time, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for database txns, excluding scheduling time, in this block""" # seconds spent waiting for a db connection, in this block block_db_sched_duration = Counter( - "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_sched_duration_seconds", + documentation="Seconds spent waiting for a db connection, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for a db connection, in this block""" # This is dynamically created in InFlightGauge.__init__. -class _InFlightMetric(Protocol): +class _BlockInFlightMetric(Protocol): + """ + Sub-metrics used for the `InFlightGauge` for blocks. + """ + real_time_max: float + """The longest observed duration of any single execution of this block, in seconds.""" real_time_sum: float + """The cumulative time spent executing this block across all calls, in seconds.""" -# Tracks the number of blocks currently active -in_flight: InFlightGauge[_InFlightMetric] = InFlightGauge( +in_flight: InFlightGauge[_BlockInFlightMetric] = InFlightGauge( "synapse_util_metrics_block_in_flight", - "", - labels=["block_name"], + desc="Tracks the number of blocks currently active", + labels=["block_name", SERVER_NAME_LABEL], + # Matches the fields in the `_BlockInFlightMetric` sub_metrics=["real_time_max", "real_time_sum"], ) - +"""Tracks the number of blocks currently active""" P = ParamSpec("P") R = TypeVar("R") -class HasClock(Protocol): +class HasClockAndServerName(Protocol): clock: Clock + """ + Used to measure functions + """ + server_name: str + """ + The homeserver name that this Measure is associated with (used to label the metric) + (`hs.hostname`). + """ def measure_func( @@ -101,8 +142,9 @@ def measure_func( ) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[R]]]: """Decorate an async method with a `Measure` context manager. - The Measure is created using `self.clock`; it should only be used to decorate - methods in classes defining an instance-level `clock` attribute. + The Measure is created using `self.clock` and `self.server_name; it should only be + used to decorate methods in classes defining an instance-level `clock` and + `server_name` attributes. Usage: @@ -116,16 +158,21 @@ def measure_func( with Measure(...): ... + Args: + name: The name of the metric to report (the block name) (used to label the + metric). Defaults to the name of the decorated function. """ def wrapper( - func: Callable[Concatenate[HasClock, P], Awaitable[R]], + func: Callable[Concatenate[HasClockAndServerName, P], Awaitable[R]], ) -> Callable[P, Awaitable[R]]: block_name = func.__name__ if name is None else name @wraps(func) - async def measured_func(self: HasClock, *args: P.args, **kwargs: P.kwargs) -> R: - with Measure(self.clock, block_name): + async def measured_func( + self: HasClockAndServerName, *args: P.args, **kwargs: P.kwargs + ) -> R: + with Measure(self.clock, name=block_name, server_name=self.server_name): r = await func(self, *args, **kwargs) return r @@ -142,19 +189,24 @@ class Measure: __slots__ = [ "clock", "name", + "server_name", "_logging_context", "start", ] - def __init__(self, clock: Clock, name: str) -> None: + def __init__(self, clock: Clock, *, name: str, server_name: str) -> None: """ Args: clock: An object with a "time()" method, which returns the current time in seconds. - name: The name of the metric to report. + name: The name of the metric to report (the block name) (used to label the + metric). + server_name: The homeserver name that this Measure is associated with (used to + label the metric) (`hs.hostname`). """ self.clock = clock self.name = name + self.server_name = server_name curr_context = current_context() if not curr_context: logger.warning( @@ -174,7 +226,7 @@ class Measure: self.start = self.clock.time() self._logging_context.__enter__() - in_flight.register((self.name,), self._update_in_flight) + in_flight.register((self.name, self.server_name), self._update_in_flight) logger.debug("Entering block %s", self.name) @@ -194,19 +246,20 @@ class Measure: duration = self.clock.time() - self.start usage = self.get_resource_usage() - in_flight.unregister((self.name,), self._update_in_flight) + in_flight.unregister((self.name, self.server_name), self._update_in_flight) self._logging_context.__exit__(exc_type, exc_val, exc_tb) try: - block_counter.labels(self.name).inc() - block_timer.labels(self.name).inc(duration) - block_ru_utime.labels(self.name).inc(usage.ru_utime) - block_ru_stime.labels(self.name).inc(usage.ru_stime) - block_db_txn_count.labels(self.name).inc(usage.db_txn_count) - block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) - block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) - except ValueError: - logger.warning("Failed to save metrics! Usage: %s", usage) + labels = {"block_name": self.name, SERVER_NAME_LABEL: self.server_name} + block_counter.labels(**labels).inc() + block_timer.labels(**labels).inc(duration) + block_ru_utime.labels(**labels).inc(usage.ru_utime) + block_ru_stime.labels(**labels).inc(usage.ru_stime) + block_db_txn_count.labels(**labels).inc(usage.db_txn_count) + block_db_txn_duration.labels(**labels).inc(usage.db_txn_duration_sec) + block_db_sched_duration.labels(**labels).inc(usage.db_sched_duration_sec) + except ValueError as exc: + logger.warning("Failed to save metrics! Usage: %s Error: %s", usage, exc) def get_resource_usage(self) -> ContextResourceUsage: """Get the resources used within this Measure block @@ -215,7 +268,7 @@ class Measure: """ return self._logging_context.get_resource_usage() - def _update_in_flight(self, metrics: _InFlightMetric) -> None: + def _update_in_flight(self, metrics: _BlockInFlightMetric) -> None: """Gets called when processing in flight metrics""" assert self.start is not None duration = self.clock.time() - self.start diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 9d8960315f..394315d2b0 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -86,6 +86,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 6b25e53c28..eb859ca47a 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -91,6 +91,7 @@ class MatrixFederationAgentTests(unittest.TestCase): "test_cache", timer=self.reactor.seconds ) self.well_known_resolver = WellKnownResolver( + "OUR_STUB_HOMESERVER_NAME", self.reactor, Agent(self.reactor, contextFactory=self.tls_factory), b"test-agent", @@ -269,6 +270,7 @@ class MatrixFederationAgentTests(unittest.TestCase): because it is created too early during setUp """ return MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. @@ -1011,6 +1013,7 @@ class MatrixFederationAgentTests(unittest.TestCase): # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. @@ -1018,6 +1021,7 @@ class MatrixFederationAgentTests(unittest.TestCase): ip_blocklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( + "OUR_STUB_HOMESERVER_NAME", cast(ISynapseReactor, self.reactor), Agent(self.reactor, contextFactory=tls_factory), b"test-agent", diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 58a7a9dc72..6c4145f2c2 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -68,6 +68,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): reactor, _ = get_clock() self.matrix_federation_agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", From 88785dbaeb30379a97c0f6ebfc51bff675605e54 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 16 Jul 2025 16:04:57 -0500 Subject: [PATCH 479/691] Refactor cache metrics to be homeserver-scoped (#18604) (add `server_name` label to cache metrics). Part of https://github.com/element-hq/synapse/issues/18592 --- changelog.d/18604.misc | 1 + synapse/api/auth/base.py | 2 +- synapse/api/auth/msc3861_delegated.py | 6 +- synapse/appservice/__init__.py | 8 +- synapse/appservice/api.py | 6 +- synapse/appservice/scheduler.py | 2 +- synapse/config/appservice.py | 3 +- synapse/federation/federation_client.py | 6 +- synapse/federation/federation_server.py | 17 +++- synapse/handlers/appservice.py | 2 +- synapse/handlers/device.py | 2 + synapse/handlers/directory.py | 2 +- synapse/handlers/initial_sync.py | 7 +- synapse/handlers/message.py | 5 +- synapse/handlers/profile.py | 1 + synapse/handlers/room.py | 6 +- synapse/handlers/room_list.py | 14 ++- synapse/handlers/room_summary.py | 6 +- synapse/handlers/sync.py | 12 +-- synapse/handlers/typing.py | 5 +- .../federation/matrix_federation_agent.py | 11 ++- .../http/federation/well_known_resolver.py | 17 ++-- synapse/http/matrixfederationclient.py | 12 +-- synapse/media/url_previewer.py | 1 + synapse/push/bulk_push_rule_evaluator.py | 6 +- synapse/replication/http/_base.py | 7 +- synapse/rest/client/login.py | 4 +- synapse/rest/client/sync.py | 2 + .../server_notices/server_notices_manager.py | 10 +-- synapse/state/__init__.py | 1 + synapse/storage/_base.py | 2 +- synapse/storage/controllers/state.py | 2 +- .../storage/databases/main/account_data.py | 4 +- synapse/storage/databases/main/appservice.py | 2 +- synapse/storage/databases/main/client_ips.py | 5 +- synapse/storage/databases/main/deviceinbox.py | 11 ++- synapse/storage/databases/main/devices.py | 27 ++++-- .../databases/main/event_federation.py | 5 +- .../storage/databases/main/events_worker.py | 6 +- synapse/storage/databases/main/presence.py | 5 +- synapse/storage/databases/main/push_rule.py | 5 +- synapse/storage/databases/main/receipts.py | 5 +- synapse/storage/databases/main/stream.py | 9 +- synapse/storage/databases/state/store.py | 11 ++- synapse/util/caches/__init__.py | 87 +++++++++++++------ synapse/util/caches/deferred_cache.py | 5 ++ synapse/util/caches/descriptors.py | 24 ++++- synapse/util/caches/dictionary_cache.py | 11 ++- synapse/util/caches/expiringcache.py | 11 ++- synapse/util/caches/lrucache.py | 52 +++++++++-- synapse/util/caches/response_cache.py | 19 +++- synapse/util/caches/stream_change_cache.py | 18 +++- synapse/util/caches/ttlcache.py | 24 ++++- synmark/suites/lrucache.py | 2 +- synmark/suites/lrucache_evict.py | 2 +- tests/api/test_auth.py | 47 ++++++---- tests/api/test_ratelimiting.py | 6 +- tests/appservice/test_api.py | 4 +- tests/appservice/test_appservice.py | 7 +- tests/config/test_cache.py | 14 +-- tests/handlers/test_appservice.py | 11 +-- tests/handlers/test_device.py | 4 +- tests/handlers/test_e2e_keys.py | 6 +- tests/handlers/test_oauth_delegation.py | 2 +- tests/handlers/test_typing.py | 4 +- tests/handlers/test_user_directory.py | 14 +-- .../test_matrix_federation_agent.py | 28 +++--- tests/metrics/test_metrics.py | 4 +- tests/push/test_push_rule_evaluator.py | 6 +- tests/replication/tcp/streams/test_typing.py | 4 +- .../test_federation_sender_shard.py | 4 +- .../test_module_cache_invalidation.py | 1 + tests/rest/client/test_account.py | 4 +- tests/rest/client/test_devices.py | 4 +- tests/rest/client/test_directory.py | 4 +- tests/rest/client/test_login.py | 11 ++- tests/rest/client/test_register.py | 8 +- tests/rest/client/test_rooms.py | 2 +- tests/storage/test_user_directory.py | 5 +- tests/test_mau.py | 8 +- tests/util/caches/test_deferred_cache.py | 45 +++++++--- tests/util/caches/test_descriptors.py | 46 ++++++++++ tests/util/caches/test_response_cache.py | 4 +- tests/util/caches/test_ttlcache.py | 4 +- tests/util/test_dict_cache.py | 2 +- tests/util/test_expiring_cache.py | 21 ++++- tests/util/test_lrucache.py | 52 ++++++----- tests/util/test_stream_change_cache.py | 38 ++++++-- 88 files changed, 694 insertions(+), 268 deletions(-) create mode 100644 changelog.d/18604.misc diff --git a/changelog.d/18604.misc b/changelog.d/18604.misc new file mode 100644 index 0000000000..c06fb23af5 --- /dev/null +++ b/changelog.d/18604.misc @@ -0,0 +1 @@ +Refactor cache metrics to be homeserver-scoped. diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 3126bc9f27..f97a71caf7 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -172,7 +172,7 @@ class BaseAuth: """ # It's ok if the app service is trying to use the sender from their registration - if app_service.sender == user_id: + if app_service.sender.to_string() == user_id: pass # Check to make sure the app service is allowed to control the user elif not app_service.is_interested_in_user(user_id): diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 0cfdf15d60..ad8f4e04f6 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -176,6 +176,7 @@ class MSC3861DelegatedAuth(BaseAuth): assert self._config.client_id, "No client_id provided" assert auth_method is not None, "Invalid client_auth_method provided" + self.server_name = hs.hostname self._clock = hs.get_clock() self._http_client = hs.get_proxied_http_client() self._hostname = hs.hostname @@ -206,8 +207,9 @@ class MSC3861DelegatedAuth(BaseAuth): # In this case, the device still exists and it's not the end of the world for # the old access token to continue working for a short time. self._introspection_cache: ResponseCache[str] = ResponseCache( - self._clock, - "token_introspection", + clock=self._clock, + name="token_introspection", + server_name=self.server_name, timeout_ms=120_000, # don't log because the keys are access tokens enable_logging=False, diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 6ee5240c4e..2d8d382e68 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -78,7 +78,7 @@ class ApplicationService: self, token: str, id: str, - sender: str, + sender: UserID, url: Optional[str] = None, namespaces: Optional[JsonDict] = None, hs_token: Optional[str] = None, @@ -96,6 +96,8 @@ class ApplicationService: self.hs_token = hs_token # The full Matrix ID for this application service's sender. self.sender = sender + # The application service user should be part of the server's domain. + self.server_name = sender.domain # nb must be called this for @cached self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist @@ -223,7 +225,7 @@ class ApplicationService: """ return ( # User is the appservice's configured sender_localpart user - user_id == self.sender + user_id == self.sender.to_string() # User is in the appservice's user namespace or self.is_user_in_namespace(user_id) ) @@ -347,7 +349,7 @@ class ApplicationService: def is_exclusive_user(self, user_id: str) -> bool: return ( self._is_exclusive(ApplicationService.NS_USERS, user_id) - or user_id == self.sender + or user_id == self.sender.to_string() ) def is_interested_in_protocol(self, protocol: str) -> bool: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 45371d6f3b..8c21e0951a 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -126,11 +126,15 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.clock = hs.get_clock() self.config = hs.config.appservice self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS + clock=hs.get_clock(), + name="as_protocol_meta", + server_name=self.server_name, + timeout_ms=HOUR_IN_MS, ) def _get_headers(self, service: "ApplicationService") -> Dict[bytes, List[bytes]]: diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index ab3f4e15fe..9d7fc0995a 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -319,7 +319,7 @@ class _ServiceQueuer: users: Set[str] = set() # The sender is always included - users.add(service.sender) + users.add(service.sender.to_string()) # All AS users that would receive the PDUs or EDUs sent to these rooms # are classed as 'interesting'. diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index dda6bcd1b7..81dbd330cc 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -122,8 +122,7 @@ def _load_appservice( localpart = as_info["sender_localpart"] if urlparse.quote(localpart) != localpart: raise ValueError("sender_localpart needs characters which are not URL encoded.") - user = UserID(localpart, hostname) - user_id = user.to_string() + user_id = UserID(localpart, hostname) # Rate limiting for users of this AS is on by default (excludes sender) rate_limited = as_info.get("rate_limited") diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2d1da70793..35c5ac6311 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -137,13 +137,14 @@ class FederationClient(FederationBase): self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() - self.hostname = hs.hostname + self.server_name = hs.hostname self.signing_key = hs.signing_key # Cache mapping `event_id` to a tuple of the event itself and the `pull_origin` # (which server we pulled the event from) self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( cache_name="get_pdu_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=120 * 1000, @@ -162,6 +163,7 @@ class FederationClient(FederationBase): Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]], ] = ExpiringCache( cache_name="get_room_hierarchy_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=5 * 60 * 1000, @@ -1068,7 +1070,7 @@ class FederationClient(FederationBase): # there's some we never care about ev = builder.create_local_event_from_event_dict( self._clock, - self.hostname, + self.server_name, self.signing_key, room_version=room_version, event_dict=pdu_dict, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2a7f5b2c4d..3e6b8b8493 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,10 @@ class FederationServer(FederationBase): # We cache results for transaction with the same ID self._transaction_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "fed_txn_handler", timeout_ms=30000 + clock=hs.get_clock(), + name="fed_txn_handler", + server_name=self.server_name, + timeout_ms=30000, ) self.transaction_actions = TransactionActions(self.store) @@ -169,10 +172,18 @@ class FederationServer(FederationBase): # We cache responses to state queries, as they take a while and often # come in waves. self._state_resp_cache: ResponseCache[Tuple[str, Optional[str]]] = ( - ResponseCache(hs.get_clock(), "state_resp", timeout_ms=30000) + ResponseCache( + clock=hs.get_clock(), + name="state_resp", + server_name=self.server_name, + timeout_ms=30000, + ) ) self._state_ids_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "state_ids_resp", timeout_ms=30000 + clock=hs.get_clock(), + name="state_ids_resp", + server_name=self.server_name, + timeout_ms=30000, ) self._federation_metrics_domains = ( diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5aefc73aba..8c5308b522 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -846,7 +846,7 @@ class ApplicationServicesHandler: # user not found; could be the AS though, so check. services = self.store.get_app_services() - service_list = [s for s in services if s.sender == user_id] + service_list = [s for s in services if s.sender.to_string() == user_id] return len(service_list) == 0 async def _check_user_exists(self, user_id: str) -> bool: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index e825626558..65fbb48768 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1215,6 +1215,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): "Handles incoming device list updates from federation and updates the DB" def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func @@ -1235,6 +1236,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): # resyncs. self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( cache_name="device_update_edu", + server_name=self.server_name, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 74c697960f..11284ccd0b 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -406,7 +406,7 @@ class DirectoryHandler: ] for service in interested_services: - if user_id == service.sender: + if user_id == service.sender.to_string(): # this user IS the app service so they can do whatever they like return True elif service.is_exclusive_alias(alias.to_string()): diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index bd3c87f5f4..75d64d2d50 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -60,6 +60,7 @@ logger = logging.getLogger(__name__) class InitialSyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.auth = hs.get_auth() self.state_handler = hs.get_state_handler() @@ -77,7 +78,11 @@ class InitialSyncHandler: bool, bool, ] - ] = ResponseCache(hs.get_clock(), "initial_sync_cache") + ] = ResponseCache( + clock=hs.get_clock(), + name="initial_sync_cache", + server_name=self.server_name, + ) self._event_serializer = hs.get_event_client_serializer() self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7c76c187bc..aa295fb6c8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -558,8 +558,9 @@ class EventCreationHandler: self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None if self._external_cache.is_enabled(): self._external_cache_joined_hosts_updates = ExpiringCache( - "_external_cache_joined_hosts_updates", - self.clock, + cache_name="_external_cache_joined_hosts_updates", + server_name=self.server_name, + clock=self.clock, expiry_ms=30 * 60 * 1000, ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 76aa90e11b..4958ab5e75 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -55,6 +55,7 @@ class ProfileHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self.clock = hs.get_clock() self.hs = hs diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b063e301e6..d8c4d0c20e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -119,6 +119,7 @@ class EventContext: class RoomCreationHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.auth = hs.get_auth() @@ -175,7 +176,10 @@ class RoomCreationHandler: # succession, only process the first attempt and return its result to # subsequent requests self._upgrade_response_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "room_upgrade", timeout_ms=FIVE_MINUTES_IN_MS + clock=hs.get_clock(), + name="room_upgrade", + server_name=self.server_name, + timeout_ms=FIVE_MINUTES_IN_MS, ) self._server_notices_mxid = hs.config.servernotices.server_notices_mxid diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 07eac71e2a..9d4307fb07 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -61,16 +61,26 @@ MAX_PUBLIC_ROOMS_IN_RESPONSE = 100 class RoomListHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.hs = hs self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] - ] = ResponseCache(hs.get_clock(), "room_list") + ] = ResponseCache( + clock=hs.get_clock(), + name="room_list", + server_name=self.server_name, + ) self.remote_response_cache: ResponseCache[ Tuple[str, Optional[int], Optional[str], bool, Optional[str]] - ] = ResponseCache(hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000) + ] = ResponseCache( + clock=hs.get_clock(), + name="remote_room_list", + server_name=self.server_name, + timeout_ms=30 * 1000, + ) async def get_local_public_room_list( self, diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 1f322ac263..838fee6a30 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -96,6 +96,7 @@ class RoomSummaryHandler: _PAGINATION_SESSION_VALIDITY_PERIOD_MS = 5 * 60 * 1000 def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._event_auth_handler = hs.get_event_auth_handler() self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -121,8 +122,9 @@ class RoomSummaryHandler: Optional[Tuple[str, ...]], ] ] = ResponseCache( - hs.get_clock(), - "get_room_hierarchy", + clock=hs.get_clock(), + name="get_room_hierarchy", + server_name=self.server_name, ) self._msc3266_enabled = hs.config.experimental.msc3266_enabled diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7b99defac1..69064e751a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -353,8 +353,9 @@ class SyncHandler: # cached result any more, and we could flush the entry from the cache to save # memory. self.response_cache: ResponseCache[SyncRequestKey] = ResponseCache( - hs.get_clock(), - "sync", + clock=hs.get_clock(), + name="sync", + server_name=self.server_name, timeout_ms=hs.config.caches.sync_response_cache_duration, ) @@ -362,8 +363,9 @@ class SyncHandler: self.lazy_loaded_members_cache: ExpiringCache[ Tuple[str, Optional[str]], LruCache[str, str] ] = ExpiringCache( - "lazy_loaded_members_cache", - self.clock, + cache_name="lazy_loaded_members_cache", + server_name=self.server_name, + clock=self.clock, max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) @@ -1134,7 +1136,7 @@ class SyncHandler: ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) + cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index bbef3a59a5..3c49655598 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -263,6 +263,7 @@ class TypingWriterHandler(FollowerTypingHandler): assert hs.get_instance_name() in hs.config.worker.writers.typing + self.server_name = hs.hostname self.auth = hs.get_auth() self.notifier = hs.get_notifier() self.event_auth_handler = hs.get_event_auth_handler() @@ -280,7 +281,9 @@ class TypingWriterHandler(FollowerTypingHandler): # caches which room_ids changed at which serials self._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", self._latest_room_serial + name="TypingStreamChangeCache", + server_name=self.server_name, + current_stream_pos=self._latest_room_serial, ) def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 4a47665abd..15609a799f 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -104,6 +104,13 @@ class MatrixFederationAgent: """ Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + tls_client_options_factory + user_agent + ip_allowlist + ip_blocklist + _srv_resolver + _well_known_resolver """ # proxy_reactor is not blocklisting reactor @@ -133,8 +140,8 @@ class MatrixFederationAgent: if _well_known_resolver is None: _well_known_resolver = WellKnownResolver( - server_name, - reactor, + server_name=server_name, + reactor=reactor, agent=BlocklistingAgentWrapper( ProxyAgent( reactor, diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 911cbac7ea..70242ad0ae 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -77,10 +77,6 @@ WELL_KNOWN_RETRY_ATTEMPTS = 3 logger = logging.getLogger(__name__) -_well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known") -_had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known") - - @attr.s(slots=True, frozen=True, auto_attribs=True) class WellKnownLookupResult: delegated_server: Optional[bytes] @@ -101,6 +97,11 @@ class WellKnownResolver: """ Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + agent + user_agent + well_known_cache + had_well_known_cache """ self.server_name = server_name @@ -108,10 +109,14 @@ class WellKnownResolver: self._clock = Clock(reactor) if well_known_cache is None: - well_known_cache = _well_known_cache + well_known_cache = TTLCache( + cache_name="well-known", server_name=server_name + ) if had_well_known_cache is None: - had_well_known_cache = _had_valid_well_known_cache + had_well_known_cache = TTLCache( + cache_name="had-valid-well-known", server_name=server_name + ) self._well_known_cache = well_known_cache self._had_valid_well_known_cache = had_well_known_cache diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 67ea9cdb81..0013b97723 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -417,12 +417,12 @@ class MatrixFederationHttpClient: if hs.get_instance_name() in outbound_federation_restricted_to: # Talk to federation directly federation_agent: IAgent = MatrixFederationAgent( - self.server_name, - self.reactor, - tls_client_options_factory, - user_agent.encode("ascii"), - hs.config.server.federation_ip_range_allowlist, - hs.config.server.federation_ip_range_blocklist, + server_name=self.server_name, + reactor=self.reactor, + tls_client_options_factory=tls_client_options_factory, + user_agent=user_agent.encode("ascii"), + ip_allowlist=hs.config.server.federation_ip_range_allowlist, + ip_blocklist=hs.config.server.federation_ip_range_blocklist, ) else: proxy_authorization_secret = hs.config.worker.worker_replication_secret diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 0c665e1942..eb0104e543 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -200,6 +200,7 @@ class UrlPreviewer: # JSON-encoded OG metadata self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( cache_name="url_previews", + server_name=self.server_name, clock=self.clock, # don't spider URLs more often than once an hour expiry_ms=ONE_HOUR, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f20b98f73f..fed9931930 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -128,6 +128,7 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname self.store = hs.get_datastores().main self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func @@ -137,10 +138,11 @@ class BulkPushRuleEvaluator: self._related_event_match_enabled = self.hs.config.experimental.msc3664_enabled self.room_push_rule_cache_metrics = register_cache( - "cache", - "room_push_rule_cache", + cache_type="cache", + cache_name="room_push_rule_cache", cache=[], # Meaningless size, as this isn't a cache that stores values, resizable=False, + server_name=self.server_name, ) async def _get_rules_for_event( diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 0002538680..31204a8384 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -121,9 +121,14 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): WAIT_FOR_STREAMS: ClassVar[bool] = True def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname + if self.CACHE: self.response_cache: ResponseCache[str] = ResponseCache( - hs.get_clock(), "repl." + self.NAME, timeout_ms=30 * 60 * 1000 + clock=hs.get_clock(), + name="repl." + self.NAME, + server_name=self.server_name, + timeout_ms=30 * 60 * 1000, ) # We reserve `instance_name` as a parameter to sending requests, so we diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 8a781f759c..aa0aa36cd9 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -314,7 +314,9 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token=should_issue_refresh_token, # The user represented by an appservice's configured sender_localpart # is not actually created in Synapse. - should_check_deactivated_or_locked=qualified_user_id != appservice.sender, + should_check_deactivated_or_locked=( + qualified_user_id != appservice.sender.to_string() + ), request_info=request_info, ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index bac02122d0..c9fb9dc4d3 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -111,6 +111,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main self.sync_handler = hs.get_sync_handler() @@ -125,6 +126,7 @@ class SyncRestServlet(RestServlet): self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, cache_name="sync_valid_filter", + server_name=self.server_name, ) # Ratelimiter for presence updates, keyed by requester. diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 001a290e87..19f86b5a56 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ SERVER_NOTICE_ROOM_TAG = "m.server_notice" class ServerNoticesManager: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self._store = hs.get_datastores().main self._config = hs.config self._account_data_handler = hs.get_account_data_handler() @@ -44,7 +45,6 @@ class ServerNoticesManager: self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() self._is_mine_id = hs.is_mine_id - self._server_name = hs.hostname self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -77,7 +77,7 @@ class ServerNoticesManager: assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) logger.info("Sending server notice to %s", user_id) @@ -151,7 +151,7 @@ class ServerNoticesManager: assert self._is_mine_id(user_id), "Cannot send server notices to remote users" requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) room_id = await self.maybe_get_notice_room_for_user(user_id) @@ -256,7 +256,7 @@ class ServerNoticesManager: """ assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) # Check whether the user has already joined or been invited to this room. If @@ -279,7 +279,7 @@ class ServerNoticesManager: if self._config.servernotices.server_notices_auto_join: user_requester = create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ) await self._room_member_handler.update_membership( requester=user_requester, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 9c24525845..976a98a58b 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -641,6 +641,7 @@ class StateResolutionHandler: self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( cache_name="state_cache", + server_name=self.server_name, clock=self.clock, max_len=100000, expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 548e7df930..d55c9e18ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,7 +55,7 @@ class SQLBaseStore(metaclass=ABCMeta): hs: "HomeServer", ): self.hs = hs - self.server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index d79791fed4..8997f4526f 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -68,7 +68,7 @@ class StateStorageController: """ def __init__(self, hs: "HomeServer", stores: "Databases"): - self.server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @cached self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index be5c494ed6..883ab93f7c 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -90,7 +90,9 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) account_max = self.get_max_account_data_stream_id() self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", account_max + name="AccountDataAndTagsChangeCache", + server_name=self.server_name, + current_stream_pos=account_max, ) self.db_pool.updates.register_background_index_update( diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 766c94fc14..9862e574fd 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -126,7 +126,7 @@ class ApplicationServiceWorkerStore(RoomMemberWorkerStore): The application service or None. """ for service in self.services_cache: - if service.sender == user_id: + if service.sender.to_string() == user_id: return service return None diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 69008804bd..cf7bc4ac69 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -421,6 +421,7 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke hs: "HomeServer", ): super().__init__(database, db_conn, hs) + self.server_name = hs.hostname if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to @@ -434,7 +435,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke # (user_id, access_token, ip,) -> last_seen self.client_ip_last_seen = LruCache[Tuple[str, str, str], int]( - cache_name="client_ip_last_seen", max_size=50000 + cache_name="client_ip_last_seen", + server_name=self.server_name, + max_size=50000, ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index a22eab2474..da10afbebe 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -94,6 +94,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): Tuple[str, Optional[str]], int ] = ExpiringCache( cache_name="last_device_delete_cache", + server_name=self.server_name, clock=self._clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -127,8 +128,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_inbox_stream_cache = StreamChangeCache( - "DeviceInboxStreamChangeCache", - min_device_inbox_id, + name="DeviceInboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_inbox_id, prefilled_cache=device_inbox_prefill, ) @@ -143,8 +145,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_federation_outbox_stream_cache = StreamChangeCache( - "DeviceFederationOutboxStreamChangeCache", - min_device_outbox_id, + name="DeviceFederationOutboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_outbox_id, prefilled_cache=device_outbox_prefill, ) diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 941d278e6c..f054c66102 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -128,8 +128,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_stream_cache = StreamChangeCache( - "DeviceListStreamChangeCache", - min_device_list_id, + name="DeviceListStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_id, prefilled_cache=device_list_prefill, ) @@ -142,8 +143,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_room_stream_cache = StreamChangeCache( - "DeviceListRoomStreamChangeCache", - min_device_list_room_id, + name="DeviceListRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_room_id, prefilled_cache=device_list_room_prefill, ) @@ -159,8 +161,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=1000, ) self._user_signature_stream_cache = StreamChangeCache( - "UserSignatureStreamChangeCache", - user_signature_stream_list_id, + name="UserSignatureStreamChangeCache", + server_name=self.server_name, + current_stream_pos=user_signature_stream_list_id, prefilled_cache=user_signature_stream_prefill, ) @@ -178,8 +181,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_federation_stream_cache = StreamChangeCache( - "DeviceListFederationStreamChangeCache", - device_list_federation_list_id, + name="DeviceListFederationStreamChangeCache", + server_name=self.server_name, + current_stream_pos=device_list_federation_list_id, prefilled_cache=device_list_federation_prefill, ) @@ -1769,11 +1773,16 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): hs: "HomeServer", ): super().__init__(database, db_conn, hs) + self.server_name = hs.hostname # Map of (user_id, device_id) -> bool. If there is an entry that implies # the device exists. self.device_id_exists_cache: LruCache[Tuple[str, str], Literal[True]] = ( - LruCache(cache_name="device_id_exists", max_size=10000) + LruCache( + cache_name="device_id_exists", + server_name=self.server_name, + max_size=10000, + ) ) async def store_device( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index dfc25d8935..8e623bf061 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -148,7 +148,10 @@ class EventFederationWorkerStore( # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( - 500000, "_event_auth_cache", size_callback=len + max_size=500000, + server_name=self.server_name, + cache_name="_event_auth_cache", + size_callback=len, ) # Flag used by unit tests to disable fallback when there is no chain cover diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 2929b1d57a..d9ef93f826 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -269,8 +269,9 @@ class EventsWorkerStore(SQLBaseStore): limit=1000, ) self._curr_state_delta_stream_cache: StreamChangeCache = StreamChangeCache( - "_curr_state_delta_stream_cache", - min_curr_state_delta_id, + name="_curr_state_delta_stream_cache", + server_name=self.server_name, + current_stream_pos=min_curr_state_delta_id, prefilled_cache=curr_state_delta_prefill, ) @@ -283,6 +284,7 @@ class EventsWorkerStore(SQLBaseStore): self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( + server_name=self.server_name, cache_name="*getEvent*", max_size=hs.config.caches.event_cache_size, # `extra_index_cb` Returns a tuple as that is the key type diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 065c885603..12cff1d352 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -108,8 +108,9 @@ class PresenceStore(PresenceBackgroundUpdateStore, CacheInvalidationWorkerStore) max_value=self._presence_id_gen.get_current_token(), ) self.presence_stream_cache = StreamChangeCache( - "PresenceStreamChangeCache", - min_presence_val, + name="PresenceStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_presence_val, prefilled_cache=presence_cache_prefill, ) diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 86c87f78bf..3bc977d497 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -163,8 +163,9 @@ class PushRulesWorkerStore( ) self.push_rules_stream_cache = StreamChangeCache( - "PushRulesStreamChangeCache", - push_rules_id, + name="PushRulesStreamChangeCache", + server_name=self.server_name, + current_stream_pos=push_rules_id, prefilled_cache=push_rules_prefill, ) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 9964331510..81f50467e7 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -158,8 +158,9 @@ class ReceiptsWorkerStore(SQLBaseStore): limit=10000, ) self._receipts_stream_cache = StreamChangeCache( - "ReceiptsRoomChangeCache", - min_receipts_stream_id, + name="ReceiptsRoomChangeCache", + server_name=self.server_name, + current_stream_pos=min_receipts_stream_id, prefilled_cache=receipts_stream_prefill, ) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 3fda49f31f..b6c6b69b22 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -617,12 +617,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): max_value=events_max, ) self._events_stream_cache = StreamChangeCache( - "EventsRoomStreamChangeCache", - min_event_val, + name="EventsRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_event_val, prefilled_cache=event_cache_prefill, ) self._membership_stream_cache = StreamChangeCache( - "MembershipStreamChangeCache", events_max + name="MembershipStreamChangeCache", + server_name=self.server_name, + current_stream_pos=events_max, ) self._stream_order_on_start = self.get_room_max_stream_ordering() diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index c1a66dcba0..9b3b7e086f 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -92,6 +92,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): ): super().__init__(database, db_conn, hs) self._state_deletion_store = state_deletion_store + self.server_name = hs.hostname # Originally the state store used a single DictionaryCache to cache the # event IDs for the state types in a given state group to avoid hammering @@ -123,14 +124,16 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): # vast majority of state in Matrix (today) is member events. self._state_group_cache: DictionaryCache[int, StateKey, str] = DictionaryCache( - "*stateGroupCache*", + name="*stateGroupCache*", + server_name=self.server_name, # TODO: this hasn't been tuned yet - 50000, + max_entries=50000, ) self._state_group_members_cache: DictionaryCache[int, StateKey, str] = ( DictionaryCache( - "*stateGroupMembersCache*", - 500000, + name="*stateGroupMembersCache*", + server_name=self.server_name, + max_entries=500000, ) ) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 76e6e139c6..3087ad6adc 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -31,6 +31,7 @@ from prometheus_client import REGISTRY from prometheus_client.core import Gauge from synapse.config.cache import add_resizable_cache +from synapse.metrics import SERVER_NAME_LABEL from synapse.util.metrics import DynamicCollectorRegistry logger = logging.getLogger(__name__) @@ -46,50 +47,65 @@ CACHE_METRIC_REGISTRY = DynamicCollectorRegistry() caches_by_name: Dict[str, Sized] = {} cache_size = Gauge( - "synapse_util_caches_cache_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_hits = Gauge( - "synapse_util_caches_cache_hits", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_hits", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_evicted = Gauge( "synapse_util_caches_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) cache_total = Gauge( - "synapse_util_caches_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_max_size = Gauge( - "synapse_util_caches_cache_max_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_max_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_memory_usage = Gauge( "synapse_util_caches_cache_size_bytes", "Estimated memory usage of the caches", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_size = Gauge( "synapse_util_caches_response_cache_size", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_hits = Gauge( "synapse_util_caches_response_cache_hits", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_evicted = Gauge( "synapse_util_caches_response_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_total = Gauge( - "synapse_util_caches_response_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_response_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) @@ -103,12 +119,17 @@ class EvictionReason(Enum): invalidation = auto() -@attr.s(slots=True, auto_attribs=True) +@attr.s(slots=True, auto_attribs=True, kw_only=True) class CacheMetric: + """ + Used to track cache metrics + """ + _cache: Sized _cache_type: str _cache_name: str _collect_callback: Optional[Callable] + _server_name: str hits: int = 0 misses: int = 0 @@ -145,34 +166,34 @@ class CacheMetric: def collect(self) -> None: try: + labels_base = { + "name": self._cache_name, + SERVER_NAME_LABEL: self._server_name, + } if self._cache_type == "response_cache": - response_cache_size.labels(self._cache_name).set(len(self._cache)) - response_cache_hits.labels(self._cache_name).set(self.hits) + response_cache_size.labels(**labels_base).set(len(self._cache)) + response_cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - response_cache_evicted.labels(self._cache_name, reason.name).set( - self.eviction_size_by_reason[reason] - ) - response_cache_total.labels(self._cache_name).set( - self.hits + self.misses - ) + response_cache_evicted.labels( + **{**labels_base, "reason": reason.name} + ).set(self.eviction_size_by_reason[reason]) + response_cache_total.labels(**labels_base).set(self.hits + self.misses) else: - cache_size.labels(self._cache_name).set(len(self._cache)) - cache_hits.labels(self._cache_name).set(self.hits) + cache_size.labels(**labels_base).set(len(self._cache)) + cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - cache_evicted.labels(self._cache_name, reason.name).set( + cache_evicted.labels(**{**labels_base, "reason": reason.name}).set( self.eviction_size_by_reason[reason] ) - cache_total.labels(self._cache_name).set(self.hits + self.misses) + cache_total.labels(**labels_base).set(self.hits + self.misses) max_size = getattr(self._cache, "max_size", None) if max_size: - cache_max_size.labels(self._cache_name).set(max_size) + cache_max_size.labels(**labels_base).set(max_size) if TRACK_MEMORY_USAGE: # self.memory_usage can be None if nothing has been inserted # into the cache yet. - cache_memory_usage.labels(self._cache_name).set( - self.memory_usage or 0 - ) + cache_memory_usage.labels(**labels_base).set(self.memory_usage or 0) if self._collect_callback: self._collect_callback() except Exception as e: @@ -181,9 +202,11 @@ class CacheMetric: def register_cache( + *, cache_type: str, cache_name: str, cache: Sized, + server_name: str, collect_callback: Optional[Callable] = None, resizable: bool = True, resize_callback: Optional[Callable] = None, @@ -196,6 +219,8 @@ def register_cache( cache_name: name of the cache cache: cache itself, which must implement __len__(), and may optionally implement a max_size property + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). collect_callback: If given, a function which is called during metric collection to update additional metrics. resizable: Whether this cache supports being resized, in which case either @@ -210,7 +235,13 @@ def register_cache( resize_callback = cache.set_cache_factor # type: ignore add_resizable_cache(cache_name, resize_callback) - metric = CacheMetric(cache, cache_type, cache_name, collect_callback) + metric = CacheMetric( + cache=cache, + cache_type=cache_type, + cache_name=cache_name, + server_name=server_name, + collect_callback=collect_callback, + ) metric_name = "cache_%s_%s" % (cache_type, cache_name) caches_by_name[cache_name] = cache CACHE_METRIC_REGISTRY.register_hook(metric_name, metric.collect) diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 14868fa4d3..0c6c912918 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -79,7 +79,9 @@ class DeferredCache(Generic[KT, VT]): def __init__( self, + *, name: str, + server_name: str, max_entries: int = 1000, tree: bool = False, iterable: bool = False, @@ -89,6 +91,8 @@ class DeferredCache(Generic[KT, VT]): """ Args: name: The name of the cache + server_name: server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). max_entries: Maximum amount of entries that the cache will hold tree: Use a TreeCache instead of a dict as the underlying cache type iterable: If True, count each item in the cached object as an entry, @@ -113,6 +117,7 @@ class DeferredCache(Generic[KT, VT]): # a Deferred. self.cache: LruCache[KT, VT] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=cache_type, size_callback=( diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 29a9586710..9630cd6d26 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -33,6 +33,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Sequence, Tuple, Type, @@ -153,6 +154,14 @@ class _CacheDescriptorBase: ) +class HasServerName(Protocol): + server_name: str + """ + The homeserver name that this cache is associated with (used to label the metric) + (`hs.hostname`). + """ + + class DeferredCacheDescriptor(_CacheDescriptorBase): """A method decorator that applies a memoizing cache around the function. @@ -200,6 +209,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): def __init__( self, + *, orig: Callable[..., Any], max_entries: int = 1000, num_args: Optional[int] = None, @@ -229,10 +239,20 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): self.prune_unread_entries = prune_unread_entries def __get__( - self, obj: Optional[Any], owner: Optional[Type] + self, obj: Optional[HasServerName], owner: Optional[Type] ) -> Callable[..., "defer.Deferred[Any]"]: + # We need access to instance-level `obj.server_name` attribute + assert obj is not None, ( + "Cannot call cached method from class (❌ `MyClass.cached_method()`) " + "and must be called from an instance (✅ `MyClass().cached_method()`). " + ) + assert obj.server_name is not None, ( + "The `server_name` attribute must be set on the object where `@cached` decorator is used." + ) + cache: DeferredCache[CacheKey, Any] = DeferredCache( name=self.name, + server_name=obj.server_name, max_entries=self.max_entries, tree=self.tree, iterable=self.iterable, @@ -490,7 +510,7 @@ class _CachedFunctionDescriptor: def __call__(self, orig: F) -> CachedFunction[F]: d = DeferredCacheDescriptor( - orig, + orig=orig, max_entries=self.max_entries, num_args=self.num_args, uncached_args=self.uncached_args, diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 14bd3ba3b0..168ddc51cd 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -127,7 +127,15 @@ class DictionaryCache(Generic[KT, DKT, DV]): for the '2' dict key. """ - def __init__(self, name: str, max_entries: int = 1000): + def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + max_entries + """ + # We use a single LruCache to store two different types of entries: # 1. Map from (key, dict_key) -> dict value (or sentinel, indicating # the key doesn't exist in the dict); and @@ -152,6 +160,7 @@ class DictionaryCache(Generic[KT, DKT, DV]): Union[_PerKeyValue, Dict[DKT, DV]], ] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=TreeCache, size_callback=len, diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 3198fdd2ed..4be4c6f01b 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -46,7 +46,9 @@ VT = TypeVar("VT") class ExpiringCache(Generic[KT, VT]): def __init__( self, + *, cache_name: str, + server_name: str, clock: Clock, max_len: int = 0, expiry_ms: int = 0, @@ -56,6 +58,8 @@ class ExpiringCache(Generic[KT, VT]): """ Args: cache_name: Name of this cache, used for logging. + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). clock max_len: Max size of dict. If the dict grows larger than this then the oldest items get automatically evicted. Default is 0, @@ -83,7 +87,12 @@ class ExpiringCache(Generic[KT, VT]): self.iterable = iterable - self.metrics = register_cache("expiring", cache_name, self) + self.metrics = register_cache( + cache_type="expiring", + cache_name=cache_name, + cache=self, + server_name=server_name, + ) if not self._expiry_ms: # Don't bother starting the loop if things never expire diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 2e5efa3a52..466362e79c 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -376,9 +376,43 @@ class LruCache(Generic[KT, VT]): If cache_type=TreeCache, all keys must be tuples. """ + @overload def __init__( self, + *, max_size: int, + server_name: str, + cache_name: str, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + @overload + def __init__( + self, + *, + max_size: int, + server_name: Literal[None] = None, + cache_name: Literal[None] = None, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + def __init__( + self, + *, + max_size: int, + server_name: Optional[str] = None, cache_name: Optional[str] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, @@ -392,8 +426,13 @@ class LruCache(Generic[KT, VT]): Args: max_size: The maximum amount of entries the cache can hold - cache_name: The name of this cache, for the prometheus metrics. If unset, - no metrics will be reported on this cache. + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). Must be set if `cache_name` is + set. If unset, no metrics will be reported on this cache. + + cache_name: The name of this cache, for the prometheus metrics. Must be set + if `server_name` is set. If unset, no metrics will be reported on this + cache. cache_type: type of underlying cache to be used. Typically one of dict @@ -457,11 +496,12 @@ class LruCache(Generic[KT, VT]): # do yet when we get resized. self._on_resize: Optional[Callable[[], None]] = None - if cache_name is not None: + if cache_name is not None and server_name is not None: metrics: Optional[CacheMetric] = register_cache( - "lru_cache", - cache_name, - self, + cache_type="lru_cache", + cache_name=cache_name, + cache=self, + server_name=server_name, collect_callback=metrics_collection_callback, ) else: diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 54b99134b9..49a9151916 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -103,18 +103,35 @@ class ResponseCache(Generic[KV]): def __init__( self, + *, clock: Clock, name: str, + server_name: str, timeout_ms: float = 0, enable_logging: bool = True, ): + """ + Args: + clock + name + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). + timeout_ms + enable_logging + """ self._result_cache: Dict[KV, ResponseCacheEntry] = {} self.clock = clock self.timeout_sec = timeout_ms / 1000.0 self._name = name - self._metrics = register_cache("response_cache", name, self, resizable=False) + self._metrics = register_cache( + cache_type="response_cache", + cache_name=name, + cache=self, + server_name=server_name, + resizable=False, + ) self._enable_logging = enable_logging def size(self) -> int: diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 5ac8643eef..2cffd352d8 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -73,11 +73,23 @@ class StreamChangeCache: def __init__( self, + *, name: str, + server_name: str, current_stream_pos: int, max_size: int = 10000, prefilled_cache: Optional[Mapping[EntityType, int]] = None, ) -> None: + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + current_stream_pos + max_size + prefilled_cache + """ + self._original_max_size: int = max_size self._max_size = math.floor(max_size) @@ -96,7 +108,11 @@ class StreamChangeCache: self.name = name self.metrics = caches.register_cache( - "cache", self.name, self._cache, resize_callback=self.set_cache_factor + cache_type="cache", + cache_name=self.name, + server_name=server_name, + cache=self._cache, + resize_callback=self.set_cache_factor, ) if prefilled_cache: diff --git a/synapse/util/caches/ttlcache.py b/synapse/util/caches/ttlcache.py index 26a088603a..18c3a1e51c 100644 --- a/synapse/util/caches/ttlcache.py +++ b/synapse/util/caches/ttlcache.py @@ -40,7 +40,21 @@ VT = TypeVar("VT") class TTLCache(Generic[KT, VT]): """A key/value cache implementation where each entry has its own TTL""" - def __init__(self, cache_name: str, timer: Callable[[], float] = time.time): + def __init__( + self, + *, + cache_name: str, + server_name: str, + timer: Callable[[], float] = time.time, + ): + """ + Args: + cache_name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + timer: Function used to get the current time in seconds since the epoch. + """ + # map from key to _CacheEntry self._data: Dict[KT, _CacheEntry[KT, VT]] = {} @@ -49,7 +63,13 @@ class TTLCache(Generic[KT, VT]): self._timer = timer - self._metrics = register_cache("ttl", cache_name, self, resizable=False) + self._metrics = register_cache( + cache_type="ttl", + cache_name=cache_name, + cache=self, + server_name=server_name, + resizable=False, + ) def set(self, key: KT, value: VT, ttl: float) -> None: """Add/update an entry in the cache diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index 49d200c43b..d109441e55 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -29,7 +29,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ - cache: LruCache[int, bool] = LruCache(loops) + cache: LruCache[int, bool] = LruCache(max_size=loops) start = perf_counter() diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index 77061625a9..00cfdd0447 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -30,7 +30,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ - cache: LruCache[int, bool] = LruCache(loops // 2) + cache: LruCache[int, bool] = LruCache(max_size=loops // 2) start = perf_counter() diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index bd229cf7e9..95a4683d03 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -60,7 +60,7 @@ class AuthTestCase(unittest.HomeserverTestCase): # modify its config instead of the hs' self.auth_blocking = AuthBlocking(hs) - self.test_user = "@foo:bar" + self.test_user_id = UserID.from_string("@foo:bar") self.test_token = b"_test_token_" # this is overridden for the appservice tests @@ -71,7 +71,7 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_user_valid_token(self) -> None: user_info = TokenLookupResult( - user_id=self.test_user, token_id=5, device_id="device" + user_id=self.test_user_id.to_string(), token_id=5, device_id="device" ) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) self.store.mark_access_token_as_used = AsyncMock(return_value=None) @@ -81,7 +81,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_user_bad_token(self) -> None: self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -96,7 +96,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_user_missing_token(self) -> None: - user_info = TokenLookupResult(user_id=self.test_user, token_id=5) + user_info = TokenLookupResult(user_id=self.test_user_id.to_string(), token_id=5) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) request = Mock(args={}) @@ -109,7 +109,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -119,7 +122,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_good_ip(self) -> None: from netaddr import IPSet @@ -127,7 +130,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id.to_string(), ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -138,7 +141,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_bad_ip(self) -> None: from netaddr import IPSet @@ -146,7 +149,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id, ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -176,7 +179,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_appservice_missing_token(self) -> None: - app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service = Mock(token="foobar", url="a_url", sender=self.test_user_id) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -191,7 +194,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_valid_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -215,7 +221,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_bad_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=False) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -238,7 +247,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"DOPPELDEVICE" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -270,7 +282,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"NOT_A_REAL_DEVICE_ID" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -436,7 +451,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), @@ -467,7 +482,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 93f4f98916..2e45d4e4d2 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -5,7 +5,7 @@ from synapse.appservice import ApplicationService from synapse.config.ratelimiting import RatelimitSettings from synapse.module_api import RatelimitOverride from synapse.module_api.callbacks.ratelimit_callbacks import RatelimitModuleApiCallbacks -from synapse.types import create_requester +from synapse.types import UserID, create_requester from tests import unittest @@ -40,7 +40,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=True, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) @@ -76,7 +76,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=False, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) diff --git a/tests/appservice/test_api.py b/tests/appservice/test_api.py index 0f19736540..8fcd928d31 100644 --- a/tests/appservice/test_api.py +++ b/tests/appservice/test_api.py @@ -25,7 +25,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.appservice import ApplicationService from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -41,7 +41,7 @@ class ApplicationServiceApiTestCase(unittest.HomeserverTestCase): self.api = hs.get_application_service_api() self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url=URL, token="unused", hs_token=TOKEN, diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 3fa4426638..620c2b907b 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -25,6 +25,7 @@ from unittest.mock import AsyncMock, Mock from twisted.internet import defer from synapse.appservice import ApplicationService, Namespace +from synapse.types import UserID from tests import unittest @@ -37,7 +38,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def setUp(self) -> None: self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url="some_url", token="some_token", ) @@ -226,11 +227,11 @@ class ApplicationServiceTestCase(unittest.TestCase): @defer.inlineCallbacks def test_interested_in_self(self) -> Generator["defer.Deferred[Any]", object, None]: # make sure invites get through - self.service.sender = "@appservice:name" + self.service.sender = UserID.from_string("@appservice:name") self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*")) self.event.type = "m.room.member" self.event.content = {"membership": "invite"} - self.event.state_key = self.service.sender + self.event.state_key = self.service.sender.to_string() self.assertTrue( ( yield self.service.is_interested_in_event( diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index aead73e059..deb6bade46 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -75,7 +75,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -96,7 +96,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 200) @@ -106,7 +106,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized to the new default cache size once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -126,7 +126,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 150) @@ -145,15 +145,15 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache_a: LruCache = LruCache(100) + cache_a: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) self.assertEqual(cache_a.max_size, 200) - cache_b: LruCache = LruCache(100) + cache_b: LruCache = LruCache(max_size=100) add_resizable_cache("*Cache_b*", cache_resize_callback=cache_b.set_cache_factor) self.assertEqual(cache_b.max_size, 300) - cache_c: LruCache = LruCache(100) + cache_c: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_c*", cache_resize_callback=cache_c.set_cache_factor) self.assertEqual(cache_c.max_size, 200) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1db630e9e4..25cf5269b8 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -43,6 +43,7 @@ from synapse.types import ( MultiWriterStreamToken, RoomStreamToken, StreamKeyType, + UserID, ) from synapse.util import Clock from synapse.util.stringutils import random_string @@ -1009,7 +1010,7 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces=namespaces, supports_ephemeral=True, @@ -1087,7 +1088,7 @@ class ApplicationServicesHandlerDeviceListsTestCase(unittest.HomeserverTestCase) appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces={ ApplicationService.NS_USERS: [ @@ -1151,9 +1152,9 @@ class ApplicationServicesHandlerOtkCountsTestCase(unittest.HomeserverTestCase): # Define an application service for the tests self._service_token = "VERYSECRET" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 080e6a7028..99a0c50211 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -34,7 +34,7 @@ from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from synapse.util.task_scheduler import TaskScheduler @@ -419,7 +419,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 70fc4263e7..323950c2f4 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -1457,7 +1457,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1525,7 +1525,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1751,7 +1751,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index fefa2f1135..20f2306d4c 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -726,7 +726,7 @@ class MSC3861OAuthDelegation(HomeserverTestCase): token="i_am_an_app_service", id="1234", namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 394315d2b0..1126b6f183 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -86,8 +86,8 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index b12ffc3665..0da423142c 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -31,7 +31,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import login, register, room, user_directory from synapse.server import HomeServer from synapse.storage.roommember import ProfileInfo -from synapse.types import JsonDict, UserProfile, create_requester +from synapse.types import JsonDict, UserID, UserProfile, create_requester from synapse.util import Clock from tests import unittest @@ -78,7 +78,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not match the regex above, so that tests # can distinguish the sender from the AS user. - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -196,7 +196,9 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): user = self.register_user("user", "pass") token = self.login(user, "pass") room = self.helper.create_room_as(user, is_public=True, tok=token) - self.helper.join(room, self.appservice.sender, tok=self.appservice.token) + self.helper.join( + room, self.appservice.sender.to_string(), tok=self.appservice.token + ) self._check_only_one_user_in_directory(user, room) def test_search_term_with_colon_in_it_does_not_raise(self) -> None: @@ -433,7 +435,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): def test_handle_local_profile_change_with_appservice_sender(self) -> None: # profile is not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) @@ -441,13 +443,13 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): profile_info = ProfileInfo(avatar_url="avatar_url", display_name="4L1c3") self.get_success( self.handler.handle_local_profile_change( - self.appservice.sender, profile_info + self.appservice.sender.to_string(), profile_info ) ) # profile is still not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index eb859ca47a..a1243b053d 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -85,16 +85,20 @@ class MatrixFederationAgentTests(unittest.TestCase): self.tls_factory = FederationPolicyForHTTPS(config) self.well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.had_well_known_cache: TTLCache[bytes, bool] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.well_known_resolver = WellKnownResolver( - "OUR_STUB_HOMESERVER_NAME", - self.reactor, - Agent(self.reactor, contextFactory=self.tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=self.reactor, + agent=Agent(self.reactor, contextFactory=self.tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ) @@ -270,7 +274,7 @@ class MatrixFederationAgentTests(unittest.TestCase): because it is created too early during setUp """ return MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", + server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. @@ -1013,7 +1017,7 @@ class MatrixFederationAgentTests(unittest.TestCase): # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", + server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. @@ -1021,10 +1025,10 @@ class MatrixFederationAgentTests(unittest.TestCase): ip_blocklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( - "OUR_STUB_HOMESERVER_NAME", - cast(ISynapseReactor, self.reactor), - Agent(self.reactor, contextFactory=tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=cast(ISynapseReactor, self.reactor), + agent=Agent(self.reactor, contextFactory=tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ), diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 2e7004df3a..dca8dd79b1 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -159,7 +159,9 @@ class CacheMetricsTests(unittest.HomeserverTestCase): Caches produce metrics reflecting their state when scraped. """ CACHE_NAME = "cache_metrics_test_fgjkbdfg" - cache: DeferredCache[str, str] = DeferredCache(CACHE_NAME, max_entries=777) + cache: DeferredCache[str, str] = DeferredCache( + name=CACHE_NAME, server_name=self.hs.hostname, max_entries=777 + ) items = { x.split(b"{")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index 3898532acf..98a3a22154 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -823,9 +823,9 @@ class TestBulkPushRuleEvaluator(unittest.HomeserverTestCase): # Define an application service so that we can register appservice users self._service_token = "some_token" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py index b1c2f5b03b..c8958189f8 100644 --- a/tests/replication/tcp/streams/test_typing.py +++ b/tests/replication/tcp/streams/test_typing.py @@ -139,7 +139,9 @@ class TypingStreamTestCase(BaseStreamTestCase): self.hs.get_replication_command_handler()._streams["typing"].last_token = 0 typing._latest_room_serial = 0 typing._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", typing._latest_room_serial + name="TypingStreamChangeCache", + server_name=self.hs.hostname, + current_stream_pos=typing._latest_room_serial, ) typing._reset() diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 6c4145f2c2..5b7ed95a23 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -68,8 +68,8 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): reactor, _ = get_clock() self.matrix_federation_agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/replication/test_module_cache_invalidation.py b/tests/replication/test_module_cache_invalidation.py index 1e7183edaa..8d5d0cce9a 100644 --- a/tests/replication/test_module_cache_invalidation.py +++ b/tests/replication/test_module_cache_invalidation.py @@ -35,6 +35,7 @@ KEY = "mykey" class TestCache: current_value = FIRST_VALUE + server_name = "test_server" # nb must be called this for @cached @cached() async def cached_function(self, user_id: str) -> str: diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index a85ea994de..5343b10e92 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -764,10 +764,10 @@ class WhoamiTestCase(unittest.HomeserverTestCase): as_token = "i_am_an_app_service" appservice = ApplicationService( - as_token, + token=as_token, id="1234", namespaces={"users": [{"regex": user_id, "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index dd3abdebac..b7230488e4 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -472,7 +472,7 @@ class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): id="msc4190", token="some_token", hs_token="some_token", - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), namespaces={ ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] }, @@ -483,7 +483,7 @@ class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): id="regular", token="other_token", hs_token="other_token", - sender="@as2:example.com", + sender=UserID.from_string("@as2:example.com"), namespaces={ ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] }, diff --git a/tests/rest/client/test_directory.py b/tests/rest/client/test_directory.py index ecf38493c3..6e499093cf 100644 --- a/tests/rest/client/test_directory.py +++ b/tests/rest/client/test_directory.py @@ -25,7 +25,7 @@ from synapse.appservice import ApplicationService from synapse.rest import admin from synapse.rest.client import directory, login, room from synapse.server import HomeServer -from synapse.types import RoomAlias +from synapse.types import RoomAlias, UserID from synapse.util import Clock from synapse.util.stringutils import random_string @@ -140,7 +140,7 @@ class DirectoryTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"aliases": [{"regex": "#asns-*", "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index c5c6604667..b8bcc235e9 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -51,7 +51,7 @@ from synapse.rest.client import account, devices, login, logout, profile, regist from synapse.rest.client.account import WhoamiRestServlet from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from tests import unittest @@ -1484,7 +1484,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.service = ApplicationService( id="unique_identifier", token="some_token", - sender="@asbot:example.com", + sender=UserID.from_string("@asbot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as_user.*", "exclusive": False} @@ -1496,7 +1496,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.another_service = ApplicationService( id="another__identifier", token="another_token", - sender="@as2bot:example.com", + sender=UserID.from_string("@as2bot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as2_user.*", "exclusive": False} @@ -1530,7 +1530,10 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): params = { "type": login.LoginRestServlet.APPSERVICE_TYPE, - "identifier": {"type": "m.id.user", "user": self.service.sender}, + "identifier": { + "type": "m.id.user", + "user": self.service.sender.to_string(), + }, } channel = self.make_request( b"POST", LOGIN_URL, params, access_token=self.service.token diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index b697bf6f67..638fbf0062 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -39,7 +39,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import account, account_validity, login, logout, register, sync from synapse.server import HomeServer from synapse.storage._base import db_to_json -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -75,7 +75,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -99,7 +99,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -129,7 +129,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), msc4190_device_management=True, ) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 3ba7584c65..8a6e6f118a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -1479,7 +1479,7 @@ class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index f97ca12d84..80f491aff9 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -36,6 +36,7 @@ from synapse.storage.databases.main.user_directory import ( _parse_words_with_icu, ) from synapse.storage.roommember import ProfileInfo +from synapse.types import UserID from synapse.util import Clock from tests.server import ThreadedMemoryReactorClock @@ -153,7 +154,7 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): token="i_am_an_app_service", id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -378,7 +379,7 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): # Join the AS sender to rooms owned by the normal user. public, private = self._create_rooms_and_inject_memberships( - user, token, self.appservice.sender + user, token, self.appservice.sender.to_string() ) # Rebuild the directory. diff --git a/tests/test_mau.py b/tests/test_mau.py index 714854cdf2..472965e022 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -29,7 +29,7 @@ from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService from synapse.rest.client import register, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -118,7 +118,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token, id="SomeASID", - sender="@as_sender:test", + sender=UserID.from_string("@as_sender:test"), namespaces={"users": [{"regex": "@as_*", "exclusive": True}]}, ) ) @@ -263,7 +263,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token_1, id="SomeASID", - sender="@as_sender_1:test", + sender=UserID.from_string("@as_sender_1:test"), namespaces={"users": [{"regex": "@as_1.*", "exclusive": True}]}, ) ) @@ -273,7 +273,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token_2, id="AnotherASID", - sender="@as_sender_2:test", + sender=UserID.from_string("@as_sender_2:test"), namespaces={"users": [{"regex": "@as_2.*", "exclusive": True}]}, ) ) diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py index f99f99237e..7017d6d70a 100644 --- a/tests/util/caches/test_deferred_cache.py +++ b/tests/util/caches/test_deferred_cache.py @@ -31,18 +31,24 @@ from tests.unittest import TestCase class DeferredCacheTestCase(TestCase): def test_empty(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) with self.assertRaises(KeyError): cache.get("foo") def test_hit(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) cache.prefill("foo", 123) self.assertEqual(self.successResultOf(cache.get("foo")), 123) def test_hit_deferred(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) origin_d: "defer.Deferred[int]" = defer.Deferred() set_d = cache.set("k1", origin_d) @@ -65,7 +71,9 @@ class DeferredCacheTestCase(TestCase): def test_callbacks(self) -> None: """Invalidation callbacks are called at the right time""" - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) callbacks = set() # start with an entry, with a callback @@ -98,7 +106,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(callbacks, {"set", "get"}) def test_set_fail(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) callbacks = set() # start with an entry, with a callback @@ -135,7 +145,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(callbacks, {"prefill", "get2"}) def test_get_immediate(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) d1: "defer.Deferred[int]" = defer.Deferred() cache.set("key1", d1) @@ -151,7 +163,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(v, 2) def test_invalidate(self) -> None: - cache: DeferredCache[Tuple[str], int] = DeferredCache("test") + cache: DeferredCache[Tuple[str], int] = DeferredCache( + name="test", server_name="test_server" + ) cache.prefill(("foo",), 123) cache.invalidate(("foo",)) @@ -159,7 +173,9 @@ class DeferredCacheTestCase(TestCase): cache.get(("foo",)) def test_invalidate_all(self) -> None: - cache: DeferredCache[str, str] = DeferredCache("testcache") + cache: DeferredCache[str, str] = DeferredCache( + name="testcache", server_name="test_server" + ) callback_record = [False, False] @@ -203,7 +219,10 @@ class DeferredCacheTestCase(TestCase): def test_eviction(self) -> None: cache: DeferredCache[int, str] = DeferredCache( - "test", max_entries=2, apply_cache_factor_from_config=False + name="test", + server_name="test_server", + max_entries=2, + apply_cache_factor_from_config=False, ) cache.prefill(1, "one") @@ -218,7 +237,10 @@ class DeferredCacheTestCase(TestCase): def test_eviction_lru(self) -> None: cache: DeferredCache[int, str] = DeferredCache( - "test", max_entries=2, apply_cache_factor_from_config=False + name="test", + server_name="test_server", + max_entries=2, + apply_cache_factor_from_config=False, ) cache.prefill(1, "one") @@ -237,7 +259,8 @@ class DeferredCacheTestCase(TestCase): def test_eviction_iterable(self) -> None: cache: DeferredCache[int, List[str]] = DeferredCache( - "test", + name="test", + server_name="test_server", max_entries=3, apply_cache_factor_from_config=False, iterable=True, diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 6af9dfaf56..7865a67709 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -66,6 +66,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> str: @@ -100,6 +101,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached(num_args=1) def fn(self, arg1: int, arg2: int) -> str: @@ -145,6 +147,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" obj = Cls() obj.mock.return_value = "fish" @@ -175,6 +178,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, kwarg1: int = 2) -> str: @@ -209,6 +213,8 @@ class DescriptorTestCase(unittest.TestCase): """If the wrapped function throws synchronously, things should continue to work""" class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached() def fn(self, arg1: int) -> NoReturn: raise SynapseError(100, "mai spoon iz too big!!1") @@ -232,6 +238,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: result: Optional[Deferred] = None call_count = 0 + server_name = "test_server" # nb must be called this for @cached @cached() def fn(self, arg1: int) -> Deferred: @@ -285,6 +292,8 @@ class DescriptorTestCase(unittest.TestCase): complete_lookup: Deferred = Deferred() class Cls: + server_name = "test_server" + @descriptors.cached() def fn(self, arg1: int) -> "Deferred[int]": @defer.inlineCallbacks @@ -327,6 +336,8 @@ class DescriptorTestCase(unittest.TestCase): the lookup function throws an exception""" class Cls: + server_name = "test_server" + @descriptors.cached() def fn(self, arg1: int) -> Deferred: @defer.inlineCallbacks @@ -369,6 +380,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int = 2, arg3: int = 3) -> str: @@ -406,6 +418,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached(iterable=True) def fn(self, arg1: int, arg2: int) -> Tuple[str, ...]: @@ -439,6 +452,8 @@ class DescriptorTestCase(unittest.TestCase): """If the wrapped function throws synchronously, things should continue to work""" class Cls: + server_name = "test_server" + @descriptors.cached(iterable=True) def fn(self, arg1: int) -> NoReturn: raise SynapseError(100, "mai spoon iz too big!!1") @@ -460,6 +475,8 @@ class DescriptorTestCase(unittest.TestCase): """Invalidations should cascade up through cache contexts""" class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached(cache_context=True) async def func1(self, key: str, cache_context: _CacheContext) -> int: return await self.func2(key, on_invalidate=cache_context.invalidate) @@ -486,6 +503,8 @@ class DescriptorTestCase(unittest.TestCase): complete_lookup: "Deferred[None]" = Deferred() class Cls: + server_name = "test_server" + @cached() async def fn(self, arg1: int) -> str: await complete_lookup @@ -517,6 +536,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False + server_name = "test_server" # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -562,6 +582,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): @defer.inlineCallbacks def test_passthrough(self) -> Generator["Deferred[Any]", object, None]: class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: return key @@ -576,6 +598,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -594,6 +618,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -612,6 +638,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): def test_invalidate_missing(self) -> None: class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: return key @@ -623,6 +651,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached(max_entries=10) def func(self, key: int) -> int: callcount[0] += 1 @@ -650,6 +680,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): d = defer.succeed(123) class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> "Deferred[int]": callcount[0] += 1 @@ -668,6 +700,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -701,6 +735,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached(max_entries=2) def func(self, key: str) -> str: callcount[0] += 1 @@ -738,6 +774,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -785,6 +823,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -850,6 +889,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int) -> None: @@ -893,6 +933,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -933,6 +974,8 @@ class CachedListDescriptorTestCase(unittest.TestCase): complete_lookup: "Deferred[None]" = Deferred() class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached() def fn(self, arg1: int) -> None: pass @@ -967,6 +1010,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False + server_name = "test_server" # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1010,6 +1054,8 @@ class CachedListDescriptorTestCase(unittest.TestCase): """ class Cls: + server_name = "test_server" + @descriptors.cached(tree=True) def fn(self, room_id: str, event_id: str) -> None: pass diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index e350967bba..30cd6ef0e4 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -46,7 +46,9 @@ class ResponseCacheTestCase(TestCase): self.reactor, self.clock = get_clock() def with_cache(self, name: str, ms: int = 0) -> ResponseCache: - return ResponseCache(self.clock, name, timeout_ms=ms) + return ResponseCache( + clock=self.clock, name=name, server_name="test_server", timeout_ms=ms + ) @staticmethod async def instant_return(o: str) -> str: diff --git a/tests/util/caches/test_ttlcache.py b/tests/util/caches/test_ttlcache.py index ae73df1841..a7b55ffecf 100644 --- a/tests/util/caches/test_ttlcache.py +++ b/tests/util/caches/test_ttlcache.py @@ -28,7 +28,9 @@ from tests import unittest class CacheTestCase(unittest.TestCase): def setUp(self) -> None: self.mock_timer = Mock(side_effect=lambda: 100.0) - self.cache: TTLCache[str, str] = TTLCache("test_cache", self.mock_timer) + self.cache: TTLCache[str, str] = TTLCache( + cache_name="test_cache", server_name="test_server", timer=self.mock_timer + ) def test_get(self) -> None: """simple set/get tests""" diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py index 5055e4aead..246e18fd15 100644 --- a/tests/util/test_dict_cache.py +++ b/tests/util/test_dict_cache.py @@ -28,7 +28,7 @@ from tests import unittest class DictCacheTestCase(unittest.TestCase): def setUp(self) -> None: self.cache: DictionaryCache[str, str, str] = DictionaryCache( - "foobar", max_entries=10 + name="foobar", server_name="test_server", max_entries=10 ) def test_simple_cache_hit_full(self) -> None: diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index e97e5cf77d..75bf50e644 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -33,7 +33,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - "test", cast(Clock, clock), max_len=1 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=1, ) cache["key"] = "value" @@ -43,7 +46,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - "test", cast(Clock, clock), max_len=2 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=2, ) cache["key"] = "value" @@ -59,7 +65,11 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_iterable_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, List[int]] = ExpiringCache( - "test", cast(Clock, clock), max_len=5, iterable=True + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=5, + iterable=True, ) cache["key"] = [1] @@ -79,7 +89,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_time_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, int] = ExpiringCache( - "test", cast(Clock, clock), expiry_ms=1000 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + expiry_ms=1000, ) cache["key"] = 1 diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 3f0d8139f8..b7acf58690 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -34,13 +34,13 @@ from tests.unittest import override_config class LruCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache["key"] = "value" self.assertEqual(cache.get("key"), "value") self.assertEqual(cache["key"], "value") def test_eviction(self) -> None: - cache: LruCache[int, int] = LruCache(2) + cache: LruCache[int, int] = LruCache(max_size=2) cache[1] = 1 cache[2] = 2 @@ -54,7 +54,7 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get(3), 3) def test_setdefault(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) self.assertEqual(cache.setdefault("key", 1), 1) self.assertEqual(cache.get("key"), 1) self.assertEqual(cache.setdefault("key", 2), 1) @@ -63,14 +63,16 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key"), 2) def test_pop(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) cache["key"] = 1 self.assertEqual(cache.pop("key"), 1) self.assertEqual(cache.pop("key"), None) def test_del_multi(self) -> None: # The type here isn't quite correct as they don't handle TreeCache well. - cache: LruCache[Tuple[str, str], str] = LruCache(4, cache_type=TreeCache) + cache: LruCache[Tuple[str, str], str] = LruCache( + max_size=4, cache_type=TreeCache + ) cache[("animal", "cat")] = "mew" cache[("animal", "dog")] = "woof" cache[("vehicles", "car")] = "vroom" @@ -89,21 +91,23 @@ class LruCacheTestCase(unittest.HomeserverTestCase): # Man from del_multi say "Yes". def test_clear(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) cache["key"] = 1 cache.clear() self.assertEqual(len(cache), 0) @override_config({"caches": {"per_cache_factors": {"mycache": 10}}}) def test_special_size(self) -> None: - cache: LruCache = LruCache(10, "mycache") + cache: LruCache = LruCache( + max_size=10, server_name="test_server", cache_name="mycache" + ) self.assertEqual(cache.max_size, 100) class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value") self.assertFalse(m.called) @@ -122,7 +126,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_multi_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value") self.assertFalse(m.called) @@ -141,7 +145,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_set(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -157,7 +161,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_pop(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -177,7 +181,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m3 = Mock() m4 = Mock() # The type here isn't quite correct as they don't handle TreeCache well. - cache: LruCache[Tuple[str, str], str] = LruCache(4, cache_type=TreeCache) + cache: LruCache[Tuple[str, str], str] = LruCache( + max_size=4, cache_type=TreeCache + ) cache.set(("a", "1"), "value", callbacks=[m1]) cache.set(("a", "2"), "value", callbacks=[m2]) @@ -199,7 +205,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_clear(self) -> None: m1 = Mock() m2 = Mock() - cache: LruCache[str, str] = LruCache(5) + cache: LruCache[str, str] = LruCache(max_size=5) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -216,7 +222,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m1 = Mock(name="m1") m2 = Mock(name="m2") m3 = Mock(name="m3") - cache: LruCache[str, str] = LruCache(2) + cache: LruCache[str, str] = LruCache(max_size=2) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -252,7 +258,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: - cache: LruCache[str, List[int]] = LruCache(5, size_callback=len) + cache: LruCache[str, List[int]] = LruCache(max_size=5, size_callback=len) cache["key1"] = [0] cache["key2"] = [1, 2] cache["key3"] = [3] @@ -275,7 +281,9 @@ class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_zero_size_drop_from_cache(self) -> None: """Test that `drop_from_cache` works correctly with 0-sized entries.""" - cache: LruCache[str, List[int]] = LruCache(5, size_callback=lambda x: 0) + cache: LruCache[str, List[int]] = LruCache( + max_size=5, size_callback=lambda x: 0 + ) cache["key1"] = [] self.assertEqual(len(cache), 0) @@ -299,7 +307,7 @@ class TimeEvictionTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(5, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache(max_size=5, clock=self.hs.get_clock()) # Check that we evict entries we haven't accessed for 30 minutes. cache["key1"] = 1 @@ -351,7 +359,7 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): mock_jemalloc_class.get_stat.return_value = 924288000 setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(4, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache(max_size=4, clock=self.hs.get_clock()) cache["key1"] = 1 cache["key2"] = 2 @@ -387,7 +395,9 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): def test_invalidate_simple(self) -> None: - cache: LruCache[str, int] = LruCache(10, extra_index_cb=lambda k, v: str(v)) + cache: LruCache[str, int] = LruCache( + max_size=10, extra_index_cb=lambda k, v: str(v) + ) cache["key1"] = 1 cache["key2"] = 2 @@ -400,7 +410,9 @@ class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key2"), 2) def test_invalidate_multi(self) -> None: - cache: LruCache[str, int] = LruCache(10, extra_index_cb=lambda k, v: str(v)) + cache: LruCache[str, int] = LruCache( + max_size=10, extra_index_cb=lambda k, v: str(v) + ) cache["key1"] = 1 cache["key2"] = 1 cache["key3"] = 2 diff --git a/tests/util/test_stream_change_cache.py b/tests/util/test_stream_change_cache.py index 9254bff79b..69a072cd36 100644 --- a/tests/util/test_stream_change_cache.py +++ b/tests/util/test_stream_change_cache.py @@ -15,7 +15,12 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): Providing a prefilled cache to StreamChangeCache will result in a cache with the prefilled-cache entered in. """ - cache = StreamChangeCache("#test", 1, prefilled_cache={"user@foo.com": 2}) + cache = StreamChangeCache( + name="#test", + server_name=self.hs.hostname, + current_stream_pos=1, + prefilled_cache={"user@foo.com": 2}, + ) self.assertTrue(cache.has_entity_changed("user@foo.com", 1)) def test_has_entity_changed(self) -> None: @@ -23,7 +28,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): StreamChangeCache.entity_has_changed will mark entities as changed, and has_entity_changed will observe the changed entities. """ - cache = StreamChangeCache("#test", 3) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=3 + ) cache.entity_has_changed("user@foo.com", 6) cache.entity_has_changed("bar@baz.net", 7) @@ -61,7 +68,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): StreamChangeCache.entity_has_changed will respect the max size and purge the oldest items upon reaching that max size. """ - cache = StreamChangeCache("#test", 1, max_size=2) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1, max_size=2 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -100,7 +109,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): entities since the given position. If the position is before the start of the known stream, it returns None instead. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -148,7 +159,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): stream position is before it, it will return True, otherwise False if the cache has no entries. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) # With no entities, it returns True for the past, present, and False for # the future. @@ -175,7 +188,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): stream position is earlier than the earliest known position, it will return all of the entities queried for. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -242,7 +257,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): recent point where the entity could have changed. If the entity is not known, the stream start is provided instead. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -260,7 +277,12 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): """ `StreamChangeCache.all_entities_changed(...)` will mark all entites as changed. """ - cache = StreamChangeCache("#test", 1, max_size=10) + cache = StreamChangeCache( + name="#test", + server_name=self.hs.hostname, + current_stream_pos=1, + max_size=10, + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) From 42297bfcebeafae5c2117a94f4b8821dd8d1caf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:06:54 +0100 Subject: [PATCH 480/691] Bump ruff from 0.12.2 to 0.12.3 (#18683) --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index ed5028c017..b106f20bfb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2447,30 +2447,30 @@ files = [ [[package]] name = "ruff" -version = "0.12.2" +version = "0.12.3" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.12.2-py3-none-linux_armv6l.whl", hash = "sha256:093ea2b221df1d2b8e7ad92fc6ffdca40a2cb10d8564477a987b44fd4008a7be"}, - {file = "ruff-0.12.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:09e4cf27cc10f96b1708100fa851e0daf21767e9709e1649175355280e0d950e"}, - {file = "ruff-0.12.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8ae64755b22f4ff85e9c52d1f82644abd0b6b6b6deedceb74bd71f35c24044cc"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eb3a6b2db4d6e2c77e682f0b988d4d61aff06860158fdb413118ca133d57922"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73448de992d05517170fc37169cbca857dfeaeaa8c2b9be494d7bcb0d36c8f4b"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8b94317cbc2ae4a2771af641739f933934b03555e51515e6e021c64441532d"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:45fc42c3bf1d30d2008023a0a9a0cfb06bf9835b147f11fe0679f21ae86d34b1"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce48f675c394c37e958bf229fb5c1e843e20945a6d962cf3ea20b7a107dcd9f4"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793d8859445ea47591272021a81391350205a4af65a9392401f418a95dfb75c9"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6932323db80484dda89153da3d8e58164d01d6da86857c79f1961934354992da"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6aa7e623a3a11538108f61e859ebf016c4f14a7e6e4eba1980190cacb57714ce"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2a4a20aeed74671b2def096bdf2eac610c7d8ffcbf4fb0e627c06947a1d7078d"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:71a4c550195612f486c9d1f2b045a600aeba851b298c667807ae933478fcef04"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4987b8f4ceadf597c927beee65a5eaf994c6e2b631df963f86d8ad1bdea99342"}, - {file = "ruff-0.12.2-py3-none-win32.whl", hash = "sha256:369ffb69b70cd55b6c3fc453b9492d98aed98062db9fec828cdfd069555f5f1a"}, - {file = "ruff-0.12.2-py3-none-win_amd64.whl", hash = "sha256:dca8a3b6d6dc9810ed8f328d406516bf4d660c00caeaef36eb831cf4871b0639"}, - {file = "ruff-0.12.2-py3-none-win_arm64.whl", hash = "sha256:48d6c6bfb4761df68bc05ae630e24f506755e702d4fb08f08460be778c7ccb12"}, - {file = "ruff-0.12.2.tar.gz", hash = "sha256:d7b4f55cd6f325cb7621244f19c873c565a08aff5a4ba9c69aa7355f3f7afd3e"}, + {file = "ruff-0.12.3-py3-none-linux_armv6l.whl", hash = "sha256:47552138f7206454eaf0c4fe827e546e9ddac62c2a3d2585ca54d29a890137a2"}, + {file = "ruff-0.12.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0a9153b000c6fe169bb307f5bd1b691221c4286c133407b8827c406a55282041"}, + {file = "ruff-0.12.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fa6b24600cf3b750e48ddb6057e901dd5b9aa426e316addb2a1af185a7509882"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2506961bf6ead54887ba3562604d69cb430f59b42133d36976421bc8bd45901"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4faaff1f90cea9d3033cbbcdf1acf5d7fb11d8180758feb31337391691f3df0"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40dced4a79d7c264389de1c59467d5d5cefd79e7e06d1dfa2c75497b5269a5a6"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0262d50ba2767ed0fe212aa7e62112a1dcbfd46b858c5bf7bbd11f326998bafc"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12371aec33e1a3758597c5c631bae9a5286f3c963bdfb4d17acdd2d395406687"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:560f13b6baa49785665276c963edc363f8ad4b4fc910a883e2625bdb14a83a9e"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023040a3499f6f974ae9091bcdd0385dd9e9eb4942f231c23c57708147b06311"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:883d844967bffff5ab28bba1a4d246c1a1b2933f48cb9840f3fdc5111c603b07"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2120d3aa855ff385e0e562fdee14d564c9675edbe41625c87eeab744a7830d12"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6b16647cbb470eaf4750d27dddc6ebf7758b918887b56d39e9c22cce2049082b"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e1417051edb436230023575b149e8ff843a324557fe0a265863b7602df86722f"}, + {file = "ruff-0.12.3-py3-none-win32.whl", hash = "sha256:dfd45e6e926deb6409d0616078a666ebce93e55e07f0fb0228d4b2608b2c248d"}, + {file = "ruff-0.12.3-py3-none-win_amd64.whl", hash = "sha256:a946cf1e7ba3209bdef039eb97647f1c77f6f540e5845ec9c114d3af8df873e7"}, + {file = "ruff-0.12.3-py3-none-win_arm64.whl", hash = "sha256:5f9c7c9c8f84c2d7f27e93674d27136fbf489720251544c4da7fb3d742e011b1"}, + {file = "ruff-0.12.3.tar.gz", hash = "sha256:f1b5a4b6668fd7b7ea3697d8d98857390b40c1320a63a178eee6be0899ea2d77"}, ] [[package]] @@ -3391,4 +3391,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "1629969968eaaf20a8f49ee3cc0075b25e5dee7e81503fff66042ece72793103" +content-hash = "c140ab1db9e5d89d251e76c68f62cfcf5c477a923ba79f0fda186278f12af901" diff --git a/pyproject.toml b/pyproject.toml index 1f42c26817..89cd57a931 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -317,7 +317,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.12.2" +ruff = "0.12.3" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From b07dc6a27d61aa568daa2dd3ce442ad3916b0e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:07:15 +0100 Subject: [PATCH 481/691] Bump types-jsonschema from 4.24.0.20250528 to 4.24.0.20250708 (#18682) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b106f20bfb..02d3c8edb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2970,14 +2970,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.24.0.20250528" +version = "4.24.0.20250708" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_jsonschema-4.24.0.20250528-py3-none-any.whl", hash = "sha256:6a906b5ff73ac11c8d1e0b6c30a9693e1e4e1ab56c56c932b3a7e081b86d187b"}, - {file = "types_jsonschema-4.24.0.20250528.tar.gz", hash = "sha256:7e28c64e0ae7980eeb158105b20663fc6a6b8f81d5f86ea6614aa0014417bd1e"}, + {file = "types_jsonschema-4.24.0.20250708-py3-none-any.whl", hash = "sha256:d574aa3421d178a8435cc898cf4cf5e5e8c8f37b949c8e3ceeff06da433a18bf"}, + {file = "types_jsonschema-4.24.0.20250708.tar.gz", hash = "sha256:a910e4944681cbb1b18a93ffb502e09910db788314312fc763df08d8ac2aadb7"}, ] [package.dependencies] From a07e26a93603420fc66353dfee2d383729c9e994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4rt?= Date: Thu, 17 Jul 2025 13:13:36 +0300 Subject: [PATCH 482/691] Fix sequence migration for autoincrement tables in synapse_port_db (#18677) Closes https://github.com/element-hq/synapse/issues/18053 - the sliding sync tables will now migrate properly. --- changelog.d/18677.bugfix | 1 + synapse/_scripts/synapse_port_db.py | 89 ++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18677.bugfix diff --git a/changelog.d/18677.bugfix b/changelog.d/18677.bugfix new file mode 100644 index 0000000000..3b443d2055 --- /dev/null +++ b/changelog.d/18677.bugfix @@ -0,0 +1 @@ +Fix sliding_sync_connections related errors when porting from SQLite to Postgres. diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 573c70696e..6c3e380355 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -190,13 +190,18 @@ APPEND_ONLY_TABLES = [ "users", ] +# These tables declare their id column with "PRIMARY KEY AUTOINCREMENT" on sqlite side +# and with "PRIMARY KEY GENERATED ALWAYS AS IDENTITY" on postgres side. This creates an +# implicit sequence that needs its value to be migrated separately. Additionally, +# inserting on postgres side needs to use the "OVERRIDING SYSTEM VALUE" modifier. +AUTOINCREMENT_TABLES = { + "sliding_sync_connections", + "sliding_sync_connection_positions", + "sliding_sync_connection_required_state", + "state_groups_pending_deletion", +} IGNORED_TABLES = { - # Porting the auto generated sequence in this table is non-trivial. - # None of the entries in this list are mandatory for Synapse to keep working. - # If state group disk space is an issue after the port, the - # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again. - "state_groups_pending_deletion", # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", @@ -284,11 +289,17 @@ class Store( return self.db_pool.runInteraction("execute_sql", r) def insert_many_txn( - self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple] + self, + txn: LoggingTransaction, + table: str, + headers: List[str], + rows: List[Tuple], + override_system_value: bool = False, ) -> None: - sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + sql = "INSERT INTO %s (%s) %s VALUES (%s)" % ( table, ", ".join(k for k in headers), + "OVERRIDING SYSTEM VALUE" if override_system_value else "", ", ".join("%s" for _ in headers), ) @@ -532,7 +543,13 @@ class Porter: def insert(txn: LoggingTransaction) -> None: assert headers is not None - self.postgres_store.insert_many_txn(txn, table, headers[1:], rows) + self.postgres_store.insert_many_txn( + txn, + table, + headers[1:], + rows, + override_system_value=table in AUTOINCREMENT_TABLES, + ) self.postgres_store.db_pool.simple_update_one_txn( txn, @@ -884,6 +901,19 @@ class Porter: ], ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_positions", "connection_position" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_required_state", "required_state_id" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connections", "connection_key" + ) + await self._setup_autoincrement_sequence( + "state_groups_pending_deletion", "sequence_number" + ) + # Step 3. Get tables. self.progress.set_state("Fetching tables") sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol( @@ -1216,6 +1246,49 @@ class Porter: "_setup_%s" % (sequence_name,), r ) + async def _setup_autoincrement_sequence( + self, + sqlite_table_name: str, + sqlite_id_column_name: str, + ) -> None: + """Set a sequence to the correct value. Use where id column was declared with PRIMARY KEY AUTOINCREMENT.""" + seq_name = await self._pg_get_serial_sequence( + sqlite_table_name, sqlite_id_column_name + ) + if seq_name is None: + raise Exception( + "implicit sequence not found for table " + sqlite_table_name + ) + + seq_value = await self.sqlite_store.db_pool.simple_select_one_onecol( + table="sqlite_sequence", + keyvalues={"name": sqlite_table_name}, + retcol="seq", + allow_none=True, + ) + if seq_value is None: + return + + def r(txn: LoggingTransaction) -> None: + sql = "ALTER SEQUENCE %s RESTART WITH" % (seq_name,) + txn.execute(sql + " %s", (seq_value + 1,)) + + await self.postgres_store.db_pool.runInteraction("_setup_%s" % (seq_name,), r) + + async def _pg_get_serial_sequence(self, table: str, column: str) -> Optional[str]: + """Returns the name of the postgres sequence associated with a column, or NULL.""" + + def r(txn: LoggingTransaction) -> Optional[str]: + txn.execute("SELECT pg_get_serial_sequence('%s', '%s')" % (table, column)) + result = txn.fetchone() + if not result: + return None + return result[0] + + return await self.postgres_store.db_pool.runInteraction( + "_pg_get_serial_sequence", r + ) + async def _setup_auth_chain_sequence(self) -> None: curr_chain_id: Optional[ int From 56c166cbf0cdcbe62420b9f842260f737e4db618 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 17 Jul 2025 12:28:17 +0200 Subject: [PATCH 483/691] Include `event_id` when getting state with `?format=event` (#18675) --- changelog.d/18675.feature | 1 + synapse/rest/client/room.py | 16 ++++++++++-- tests/rest/client/test_rooms.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18675.feature diff --git a/changelog.d/18675.feature b/changelog.d/18675.feature new file mode 100644 index 0000000000..7ff2445807 --- /dev/null +++ b/changelog.d/18675.feature @@ -0,0 +1 @@ +Include `event_id` when getting state with `?format=event`. Contributed by @tulir @ Beeper. diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 4600a87778..6b0deda0df 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -44,7 +44,11 @@ from synapse.api.errors import ( UnredactedContentDeletedError, ) from synapse.api.filtering import Filter -from synapse.events.utils import SerializeEventConfig, format_event_for_client_v2 +from synapse.events.utils import ( + SerializeEventConfig, + format_event_for_client_v2, + serialize_event, +) from synapse.http.server import HttpServer from synapse.http.servlet import ( ResolveRoomIdMixin, @@ -198,6 +202,7 @@ class RoomStateEventRestServlet(RestServlet): self.message_handler = hs.get_message_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() + self.clock = hs.get_clock() self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker @@ -268,7 +273,14 @@ class RoomStateEventRestServlet(RestServlet): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if format == "event": - event = format_event_for_client_v2(data.get_dict()) + event = serialize_event( + data, + self.clock.time_msec(), + config=SerializeEventConfig( + event_format=format_event_for_client_v2, + requester=requester, + ), + ) return 200, event elif format == "content": return 200, data.get_dict()["content"] diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 8a6e6f118a..48d33b8e17 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -552,6 +552,51 @@ class RoomStateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.result["body"]) self.assertEqual(channel.json_body, {"membership": "join"}) + def test_get_state_format_content(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=content` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel1 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=content" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel1.code, HTTPStatus.OK, channel1.json_body) + self.assertEqual(channel1.json_body, {"membership": "join"}) + channel2 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel2.code, HTTPStatus.OK, channel2.json_body) + # "content" is the default format. + self.assertEqual(channel1.json_body, channel2.json_body) + + def test_get_state_format_event(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=event` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=event" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["content"], {"membership": "join"}) + self.assertEqual(channel.json_body["room_id"], room_id) + self.assertRegex(channel.json_body["event_id"], r"\$.+") + self.assertEqual(channel.json_body["type"], "m.room.member") + self.assertEqual(channel.json_body["sender"], self.user_id) + self.assertEqual(channel.json_body["state_key"], self.user_id) + self.assertTrue(type(channel.json_body["origin_server_ts"]) is int) + class RoomsMemberListTestCase(RoomBase): """Tests /rooms/$room_id/members/list REST events.""" From 84991317d0ab2270a04cdea710fb45f5ec89f996 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 17 Jul 2025 09:15:07 -0400 Subject: [PATCH 484/691] Stabilize support for custom profile fields. (#18635) --- changelog.d/18635.feature | 1 + synapse/handlers/profile.py | 24 ++-- synapse/rest/client/capabilities.py | 30 ++-- synapse/rest/client/profile.py | 190 ++++--------------------- tests/rest/client/test_capabilities.py | 12 ++ tests/rest/client/test_profile.py | 57 +++----- 6 files changed, 88 insertions(+), 226 deletions(-) create mode 100644 changelog.d/18635.feature diff --git a/changelog.d/18635.feature b/changelog.d/18635.feature new file mode 100644 index 0000000000..af536f64d3 --- /dev/null +++ b/changelog.d/18635.feature @@ -0,0 +1 @@ +Support arbitrary profile fields. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4958ab5e75..da392e115f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -93,9 +93,7 @@ class ProfileHandler: if self.hs.is_mine(target_user): profileinfo = await self.store.get_profileinfo(target_user) - extra_fields = {} - if self.hs.config.experimental.msc4133_enabled: - extra_fields = await self.store.get_profile_fields(target_user) + extra_fields = await self.store.get_profile_fields(target_user) if ( profileinfo.display_name is None @@ -551,16 +549,16 @@ class ProfileHandler: # since then we send a null in the JSON response if avatar_url is not None: response["avatar_url"] = avatar_url - if self.hs.config.experimental.msc4133_enabled: - if just_field is None: - response.update(await self.store.get_profile_fields(user)) - elif just_field not in ( - ProfileFields.DISPLAYNAME, - ProfileFields.AVATAR_URL, - ): - response[just_field] = await self.store.get_profile_field( - user, just_field - ) + + if just_field is None: + response.update(await self.store.get_profile_fields(user)) + elif just_field not in ( + ProfileFields.DISPLAYNAME, + ProfileFields.AVATAR_URL, + ): + response[just_field] = await self.store.get_profile_field( + user, just_field + ) except StoreError as e: if e.code == 404: raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 8f3193fb47..a279db1cc5 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -92,22 +92,22 @@ class CapabilitiesRestServlet(RestServlet): "enabled": self.config.experimental.msc3664_enabled, } - if self.config.experimental.msc4133_enabled: - response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = { - "enabled": True, - } + disallowed_profile_fields = [] + response["capabilities"]["m.profile_fields"] = {"enabled": True} + if not self.config.registration.enable_set_displayname: + disallowed_profile_fields.append("displayname") + if not self.config.registration.enable_set_avatar_url: + disallowed_profile_fields.append("avatar_url") + if disallowed_profile_fields: + response["capabilities"]["m.profile_fields"]["disallowed"] = ( + disallowed_profile_fields + ) - # Ensure this is consistent with the legacy m.set_displayname and - # m.set_avatar_url. - disallowed = [] - if not self.config.registration.enable_set_displayname: - disallowed.append("displayname") - if not self.config.registration.enable_set_avatar_url: - disallowed.append("avatar_url") - if disallowed: - response["capabilities"]["uk.tcpip.msc4133.profile_fields"][ - "disallowed" - ] = disallowed + # For transition from unstable to stable identifiers. + if self.config.experimental.msc4133_enabled: + response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = response[ + "capabilities" + ]["m.profile_fields"] if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 8326d8017c..243245f739 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -57,161 +57,6 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: return propagate -class ProfileDisplaynameRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/displayname", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - displayname = await self.profile_handler.get_displayname(user) - - ret = {} - if displayname is not None: - ret["displayname"] = displayname - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - requester = await self.auth.get_user_by_req(request, allow_guest=True) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - - try: - new_name = content["displayname"] - except Exception: - raise SynapseError( - 400, "Missing key 'displayname'", errcode=Codes.MISSING_PARAM - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating displayname while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_displayname( - user, requester, new_name, is_admin, propagate=propagate - ) - - return 200, {} - - -class ProfileAvatarURLRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/avatar_url", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - avatar_url = await self.profile_handler.get_avatar_url(user) - - ret = {} - if avatar_url is not None: - ret["avatar_url"] = avatar_url - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - requester = await self.auth.get_user_by_req(request) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - try: - new_avatar_url = content["avatar_url"] - except KeyError: - raise SynapseError( - 400, "Missing key 'avatar_url'", errcode=Codes.MISSING_PARAM - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating avatar URL while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_avatar_url( - user, requester, new_avatar_url, is_admin, propagate=propagate - ) - - return 200, {} - - class ProfileRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) CATEGORY = "Event sending requests" @@ -244,12 +89,19 @@ class ProfileRestServlet(RestServlet): return 200, ret -class UnstableProfileFieldRestServlet(RestServlet): +class ProfileFieldRestServlet(RestServlet): PATTERNS = [ + *client_patterns( + "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + ), + *client_patterns( + "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + ), re.compile( - r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" - ) + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + ), ] + CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): @@ -304,7 +156,10 @@ class UnstableProfileFieldRestServlet(RestServlet): HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM ) - requester = await self.auth.get_user_by_req(request) + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -366,7 +221,10 @@ class UnstableProfileFieldRestServlet(RestServlet): HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM ) - requester = await self.auth.get_user_by_req(request) + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -413,11 +271,15 @@ class UnstableProfileFieldRestServlet(RestServlet): return 200, {} +class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): + re.compile( + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + ) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The specific displayname / avatar URL / custom field endpoints *must* appear - # before their corresponding generic profile endpoint. - ProfileDisplaynameRestServlet(hs).register(http_server) - ProfileAvatarURLRestServlet(hs).register(http_server) + # The specific field endpoint *must* appear before the generic profile endpoint. + ProfileFieldRestServlet(hs).register(http_server) ProfileRestServlet(hs).register(http_server) if hs.config.experimental.msc4133_enabled: UnstableProfileFieldRestServlet(hs).register(http_server) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 13831462e8..cdf31155fd 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -130,6 +130,10 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) @override_config({"enable_set_avatar_url": False}) def test_get_set_avatar_url_capabilities_avatar_url_disabled(self) -> None: @@ -141,6 +145,8 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) @override_config( { @@ -159,6 +165,10 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) self.assertEqual( capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], @@ -180,6 +190,8 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) self.assertEqual( capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 708402b792..49776d8e8c 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -484,38 +484,34 @@ class ProfileTestCase(unittest.HomeserverTestCase): # The client requested ?propagate=true, so it should have happened. self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_missing_custom_field(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_missing_custom_field_invalid_field_name(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/[custom_field]", + f"/_matrix/client/v3/profile/{self.owner}/[custom_field]", ) self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_custom_field_rejects_bad_username(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{urllib.parse.quote('@alice:')}/custom_field", + f"/_matrix/client/v3/profile/{urllib.parse.quote('@alice:')}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "test"}, access_token=self.owner_tok, ) @@ -523,7 +519,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {"custom_field": "test"}) @@ -531,7 +527,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Overwriting the field should work. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "new_Value"}, access_token=self.owner_tok, ) @@ -539,7 +535,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {"custom_field": "new_Value"}) @@ -547,7 +543,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Deleting the field should work. channel = self.make_request( "DELETE", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={}, access_token=self.owner_tok, ) @@ -555,12 +551,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_non_string(self) -> None: """Non-string fields are supported for custom fields.""" fields = { @@ -574,7 +569,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): for key, value in fields.items(): channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: value}, access_token=self.owner_tok, ) @@ -591,22 +586,20 @@ class ProfileTestCase(unittest.HomeserverTestCase): for key, value in fields.items(): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {key: value}) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_noauth(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "test"}, ) self.assertEqual(channel.code, 401, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.MISSING_TOKEN) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_size(self) -> None: """ Attempts to set a custom field name that is too long should get a 400 error. @@ -614,7 +607,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Key is missing. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/", + f"/_matrix/client/v3/profile/{self.owner}/", content={"": "test"}, access_token=self.owner_tok, ) @@ -625,7 +618,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "c" * 500 channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "test"}, access_token=self.owner_tok, ) @@ -634,7 +627,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "DELETE", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "test"}, access_token=self.owner_tok, ) @@ -644,14 +637,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Key doesn't match body. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"diff_key": "test"}, access_token=self.owner_tok, ) self.assertEqual(channel.code, 400, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.MISSING_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_profile_too_long(self) -> None: """ Attempts to set a custom field that would push the overall profile too large. @@ -664,7 +656,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "a" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "a" * 65498}, access_token=self.owner_tok, ) @@ -692,7 +684,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "b" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "1" + "a" * ADDITIONAL_CHARS}, access_token=self.owner_tok, ) @@ -722,7 +714,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "b" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "" + "a" * ADDITIONAL_CHARS}, access_token=self.owner_tok, ) @@ -732,17 +724,16 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "a" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: ""}, access_token=self.owner_tok, ) self.assertEqual(channel.code, 200, channel.result) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_displayname(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/displayname", + f"/_matrix/client/v3/profile/{self.owner}/displayname", content={"displayname": "test"}, access_token=self.owner_tok, ) @@ -751,11 +742,10 @@ class ProfileTestCase(unittest.HomeserverTestCase): displayname = self._get_displayname() self.assertEqual(displayname, "test") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_avatar_url(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/avatar_url", + f"/_matrix/client/v3/profile/{self.owner}/avatar_url", content={"avatar_url": "mxc://test/good"}, access_token=self.owner_tok, ) @@ -764,12 +754,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): avatar_url = self._get_avatar_url() self.assertEqual(avatar_url, "mxc://test/good") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_other(self) -> None: """Setting someone else's profile field should fail""" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.other}/custom_field", + f"/_matrix/client/v3/profile/{self.other}/custom_field", content={"custom_field": "test"}, access_token=self.owner_tok, ) From a0d6469069d0bd2b1aed2b0c78606d02ebb15ae9 Mon Sep 17 00:00:00 2001 From: Kim Brose <2803622+HarHarLinks@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:54:33 +0000 Subject: [PATCH 485/691] fix schema and docs of `rc_delayed_event_mgmt` (#18692) Signed-off-by: Kim Brose --- changelog.d/18692.doc | 1 + docs/usage/configuration/config_documentation.md | 5 ++--- schema/synapse-config.schema.yaml | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18692.doc diff --git a/changelog.d/18692.doc b/changelog.d/18692.doc new file mode 100644 index 0000000000..1514ef6fb8 --- /dev/null +++ b/changelog.d/18692.doc @@ -0,0 +1 @@ +Update `rc_delayed_event_mgmt` docs to the actual nesting level. Contributed by @HarHarLinks. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 6918559dea..e47a32f510 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1925,9 +1925,8 @@ This setting has the following sub-options: Default configuration: ```yaml rc_delayed_event_mgmt: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 ``` Example configuration: diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index d9a5a98496..9a244856e6 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2179,9 +2179,8 @@ properties: with a short timeout, or restarting several different delayed events all at once) without the risk of being ratelimited. default: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 examples: - per_second: 2.0 burst_count: 20.0 From f031105eee6a1b0f9d7231712ebb68c32f900d59 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Thu, 17 Jul 2025 17:15:11 +0200 Subject: [PATCH 486/691] Start and stop the Tokio runtime with the Twisted reactor (#18691) Fixes https://github.com/element-hq/synapse/issues/18659 This changes the Tokio runtime to be attached to the Twisted reactor. This way, the Tokio runtime starts when the Twisted reactor starts, and *not* when the module gets loaded. This is important as starting the runtime on module load meant that it broke when Synapse was started with `daemonize`/`synctl`, as forks only retain the calling threads, breaking the Tokio runtime. This also changes so that the HttpClient gets the Twisted reactor explicitly as parameter instead of loading it from `twisted.internet.reactor` --- Cargo.lock | 1 + changelog.d/18691.bugfix | 1 + rust/Cargo.toml | 1 + rust/src/http_client.rs | 227 ++++++++++++++++++-------- synapse/api/auth/msc3861_delegated.py | 3 +- synapse/synapse_rust/http_client.pyi | 4 +- 6 files changed, 164 insertions(+), 73 deletions(-) create mode 100644 changelog.d/18691.bugfix diff --git a/Cargo.lock b/Cargo.lock index abb3c6bed3..026be1b6e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,6 +1470,7 @@ dependencies = [ "lazy_static", "log", "mime", + "once_cell", "pyo3", "pyo3-log", "pythonize", diff --git a/changelog.d/18691.bugfix b/changelog.d/18691.bugfix new file mode 100644 index 0000000000..27bc09e4fd --- /dev/null +++ b/changelog.d/18691.bugfix @@ -0,0 +1 @@ +Fix the MAS integration not working when Synapse is started with `--daemonize` or using `synctl`. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dab32c8952..4f5ebb68b7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -52,6 +52,7 @@ reqwest = { version = "0.12.15", default-features = false, features = [ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +once_cell = "1.18.0" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index eda0197c74..b6cdf98f55 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,58 +12,149 @@ * . */ -use std::{collections::HashMap, future::Future, panic::AssertUnwindSafe, sync::LazyLock}; +use std::{collections::HashMap, future::Future}; use anyhow::Context; -use futures::{FutureExt, TryStreamExt}; -use pyo3::{exceptions::PyException, prelude::*, types::PyString}; +use futures::TryStreamExt; +use once_cell::sync::OnceCell; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; use reqwest::RequestBuilder; use tokio::runtime::Runtime; use crate::errors::HttpResponseException; -/// The tokio runtime that we're using to run async Rust libs. -static RUNTIME: LazyLock = LazyLock::new(|| { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build() - .unwrap() -}); +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); -/// A reference to the `Deferred` python class. -static DEFERRED_CLASS: LazyLock = LazyLock::new(|| { - Python::with_gil(|py| { - py.import("twisted.internet.defer") - .expect("module 'twisted.internet.defer' should be importable") - .getattr("Deferred") - .expect("module 'twisted.internet.defer' should have a 'Deferred' class") - .unbind() - }) -}); +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} -/// A reference to the twisted `reactor`. -static TWISTED_REACTOR: LazyLock> = LazyLock::new(|| { - Python::with_gil(|py| { - py.import("twisted.internet.reactor") - .expect("module 'twisted.internet.reactor' should be importable") - .unbind() - }) -}); +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; child_module.add_class::()?; - // Make sure we fail early if we can't build the lazy statics. - LazyLock::force(&RUNTIME); - LazyLock::force(&DEFERRED_CLASS); + // Make sure we fail early if we can't load some modules + defer(py)?; m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from - // synapse.synapse_rust import acl` work. + // synapse.synapse_rust import http_client` work. py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.http_client", child_module)?; @@ -72,26 +163,24 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> } #[pyclass] -#[derive(Clone)] struct HttpClient { client: reqwest::Client, + reactor: PyObject, } #[pymethods] impl HttpClient { #[new] - pub fn py_new(user_agent: &str) -> PyResult { - // The twisted reactor can only be imported after Synapse has been - // imported, to allow Synapse to change the twisted reactor. If we try - // and import the reactor too early twisted installs a default reactor, - // which can't be replaced. - LazyLock::force(&TWISTED_REACTOR); + pub fn py_new(reactor: Bound, user_agent: &str) -> PyResult { + // Make sure the runtime gets installed + let _ = runtime(&reactor)?; Ok(HttpClient { client: reqwest::Client::builder() .user_agent(user_agent) .build() .context("building reqwest client")?, + reactor: reactor.unbind(), }) } @@ -129,7 +218,7 @@ impl HttpClient { builder: RequestBuilder, response_limit: usize, ) -> PyResult> { - create_deferred(py, async move { + create_deferred(py, self.reactor.bind(py), async move { let response = builder.send().await.context("sending request")?; let status = response.status(); @@ -159,43 +248,51 @@ impl HttpClient { /// tokio runtime. /// /// Does not handle deferred cancellation or contextvars. -fn create_deferred(py: Python, fut: F) -> PyResult> +fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> where F: Future> + Send + 'static, - for<'a> O: IntoPyObject<'a>, + for<'a> O: IntoPyObject<'a> + Send + 'static, { - let deferred = DEFERRED_CLASS.bind(py).call0()?; + let deferred = defer(py)?.call_method0("Deferred")?; let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); - RUNTIME.spawn(async move { - // TODO: Is it safe to assert unwind safety here? I think so, as we - // don't use anything that could be tainted by the panic afterwards. - // Note that `.spawn(..)` asserts unwind safety on the future too. - let res = AssertUnwindSafe(fut).catch_unwind().await; + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; Python::with_gil(move |py| { // Flatten the panic into standard python error let res = match res { Ok(r) => r, - Err(panic_err) => { - let panic_message = get_panic_message(&panic_err); - Err(PyException::new_err( - PyString::new(py, panic_message).unbind(), - )) - } + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, }; + // Re-bind the reactor + let reactor = reactor.bind(py); + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` match res { Ok(obj) => { - TWISTED_REACTOR - .call_method(py, "callFromThread", (deferred_callback, obj), None) + reactor + .call_method("callFromThread", (deferred_callback, obj), None) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } Err(err) => { - TWISTED_REACTOR - .call_method(py, "callFromThread", (deferred_errback, err), None) + reactor + .call_method("callFromThread", (deferred_errback, err), None) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } } @@ -204,15 +301,3 @@ where Ok(deferred) } - -/// Try and get the panic message out of the panic -fn get_panic_message<'a>(panic_err: &'a (dyn std::any::Any + Send + 'static)) -> &'a str { - // Apparently this is how you extract the panic message from a panic - if let Some(str_slice) = panic_err.downcast_ref::<&str>() { - str_slice - } else if let Some(string) = panic_err.downcast_ref::() { - string - } else { - "unknown error" - } -} diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index ad8f4e04f6..581c9c1e74 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -184,7 +184,8 @@ class MSC3861DelegatedAuth(BaseAuth): self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users self._rust_http_client = HttpClient( - user_agent=self._http_client.user_agent.decode("utf8") + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), ) # # Token Introspection Cache diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 5fa6226fd5..cdc501e606 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -12,8 +12,10 @@ from typing import Awaitable, Mapping +from synapse.types import ISynapseReactor + class HttpClient: - def __init__(self, user_agent: str) -> None: ... + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: ... def get(self, url: str, response_limit: int) -> Awaitable[bytes]: ... def post( self, From f0f9a82ca4f3d6a9f4ae4514a9b085ccfd7f01d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 17:24:57 +0200 Subject: [PATCH 487/691] Bump lxml from 5.4.0 to 6.0.0 (#18631) Bumps [lxml](https://github.com/lxml/lxml) from 5.4.0 to 6.0.0.
Changelog

Sourced from lxml's changelog.

6.0.0 (2025-06-26)

Features added

  • GH#463: lxml.html.diff is faster and provides structurally better diffs. Original patch by Steven Fernandez.

  • GH#405: The factories Element and ElementTree can now be used in type hints.

  • GH#448: Parsing from memoryview and other buffers is supported to allow zero-copy parsing.

  • GH#437: lxml.html.builder was missing several HTML5 tag names. Patch by Nick Tarleton.

  • GH#458: CDATA can now be written into the incremental xmlfile() writer. Original patch by Lane Shaw.

  • A new parser option decompress=False was added that controls the automatic input decompression when using libxml2 2.15.0 or later. Disabling this option by default will effectively prevent decompression bombs when handling untrusted input. Code that depends on automatic decompression must enable this option. Note that libxml2 2.15.0 was not released yet, so this option currently has no effect but can already be used.

  • The set of compile time / runtime supported libxml2 feature names is available as etree.LIBXML_COMPILED_FEATURES and etree.LIBXML_FEATURES. This currently includes catalog, ftp, html, http, iconv, icu, lzma, regexp, schematron, xmlschema, xpath, zlib.

Bugs fixed

  • GH#353: Predicates in .find*() could mishandle tag indices if a default namespace is provided. Original patch by Luise K.

  • GH#272: The head and body properties of lxml.html elements failed if no such element was found. They now return None instead. Original patch by FVolral.

  • Tag names provided by code (API, not data) that are longer than INT_MAX could be truncated or mishandled in other ways.

  • .text_content() on lxml.html elements accidentally returned a "smart string" without additional information. It now returns a plain string.

  • LP#2109931: When building lxml with coverage reporting, it now disables the sys.monitoring support due to the lack of support in nedbat/coveragepy#1790

... (truncated)

Commits
  • 2a67034 Prepare release of 6.0.0.
  • e0b4e02 Update changelog.
  • d3f4dcf Build: Upgrade libxml2 to latest 2.14.4.
  • 014e51c Build: Add Windows arm64 wheel builds (GH-465)
  • d3914dc Only use "xmlCtxtIsStopped()" from libxml2 2.15.0 on since it fails to cover ...
  • 6e41390 Avoid reading the deprecated "disableSAX" attribute of "xmlParserCtxt".
  • f85da81 Use newer "language_level=3" in ElementPath module.
  • 787315e Build: bump pypa/cibuildwheel in the github-actions group (#464)
  • fb3adb1 Readme: Add project income report for 2024.
  • 8e61a75 Fit cached tuple more nicely into cachelines.
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=lxml&package-manager=pip&previous-version=5.4.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 231 ++++++++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 135 deletions(-) diff --git a/poetry.lock b/poetry.lock index 02d3c8edb2..ebdb8e3896 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1014,145 +1014,107 @@ pyasn1 = ">=0.4.6" [[package]] name = "lxml" -version = "5.4.0" +version = "6.0.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true -python-versions = ">=3.6" +python-versions = ">=3.8" groups = ["main"] markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, - {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696ea9e87442467819ac22394ca36cb3d01848dad1be6fac3fb612d3bd5a12cf"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef80aeac414f33c24b3815ecd560cee272786c3adfa5f31316d8b349bfade28"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9c2754cef6963f3408ab381ea55f47dabc6f78f4b8ebb0f0b25cf1ac1f7609"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a62cc23d754bb449d63ff35334acc9f5c02e6dae830d78dab4dd12b78a524f4"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f82125bc7203c5ae8633a7d5d20bcfdff0ba33e436e4ab0abc026a53a8960b7"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b67319b4aef1a6c56576ff544b67a2a6fbd7eaee485b241cabf53115e8908b8f"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:a8ef956fce64c8551221f395ba21d0724fed6b9b6242ca4f2f7beb4ce2f41997"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:0a01ce7d8479dce84fc03324e3b0c9c90b1ece9a9bb6a1b6c9025e7e4520e78c"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91505d3ddebf268bb1588eb0f63821f738d20e1e7f05d3c647a5ca900288760b"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3bcdde35d82ff385f4ede021df801b5c4a5bcdfb61ea87caabcebfc4945dc1b"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aea7c06667b987787c7d1f5e1dfcd70419b711cdb47d6b4bb4ad4b76777a0563"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7fb111eef4d05909b82152721a59c1b14d0f365e2be4c742a473c5d7372f4f5"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43d549b876ce64aa18b2328faff70f5877f8c6dede415f80a2f799d31644d776"}, - {file = "lxml-5.4.0-cp310-cp310-win32.whl", hash = "sha256:75133890e40d229d6c5837b0312abbe5bac1c342452cf0e12523477cd3aa21e7"}, - {file = "lxml-5.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:de5b4e1088523e2b6f730d0509a9a813355b7f5659d70eb4f319c76beea2e250"}, - {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9"}, - {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751"}, - {file = "lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4"}, - {file = "lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539"}, - {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4"}, - {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc"}, - {file = "lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f"}, - {file = "lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2"}, - {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0"}, - {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a"}, - {file = "lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82"}, - {file = "lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f"}, - {file = "lxml-5.4.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7be701c24e7f843e6788353c055d806e8bd8466b52907bafe5d13ec6a6dbaecd"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb54f7c6bafaa808f27166569b1511fc42701a7713858dddc08afdde9746849e"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97dac543661e84a284502e0cf8a67b5c711b0ad5fb661d1bd505c02f8cf716d7"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:c70e93fba207106cb16bf852e421c37bbded92acd5964390aad07cb50d60f5cf"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9c886b481aefdf818ad44846145f6eaf373a20d200b5ce1a5c8e1bc2d8745410"}, - {file = "lxml-5.4.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:fa0e294046de09acd6146be0ed6727d1f42ded4ce3ea1e9a19c11b6774eea27c"}, - {file = "lxml-5.4.0-cp36-cp36m-win32.whl", hash = "sha256:61c7bbf432f09ee44b1ccaa24896d21075e533cd01477966a5ff5a71d88b2f56"}, - {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, - {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, - {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, - {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, - {file = "lxml-5.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eaf24066ad0b30917186420d51e2e3edf4b0e2ea68d8cd885b14dc8afdcf6556"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b31a3a77501d86d8ade128abb01082724c0dfd9524f542f2f07d693c9f1175f"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e108352e203c7afd0eb91d782582f00a0b16a948d204d4dec8565024fafeea5"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a11a96c3b3f7551c8a8109aa65e8594e551d5a84c76bf950da33d0fb6dfafab7"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:ca755eebf0d9e62d6cb013f1261e510317a41bf4650f22963474a663fdfe02aa"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4cd915c0fb1bed47b5e6d6edd424ac25856252f09120e3e8ba5154b6b921860e"}, - {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:226046e386556a45ebc787871d6d2467b32c37ce76c2680f5c608e25823ffc84"}, - {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:b108134b9667bcd71236c5a02aad5ddd073e372fb5d48ea74853e009fe38acb6"}, - {file = "lxml-5.4.0-cp38-cp38-win32.whl", hash = "sha256:1320091caa89805df7dcb9e908add28166113dcd062590668514dbd510798c88"}, - {file = "lxml-5.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:073eb6dcdf1f587d9b88c8c93528b57eccda40209cf9be549d469b942b41d70b"}, - {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bda3ea44c39eb74e2488297bb39d47186ed01342f0022c8ff407c250ac3f498e"}, - {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ceaf423b50ecfc23ca00b7f50b64baba85fb3fb91c53e2c9d00bc86150c7e40"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:664cdc733bc87449fe781dbb1f309090966c11cc0c0cd7b84af956a02a8a4729"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67ed8a40665b84d161bae3181aa2763beea3747f748bca5874b4af4d75998f87"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b4a3bd174cc9cdaa1afbc4620c049038b441d6ba07629d89a83b408e54c35cd"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b0989737a3ba6cf2a16efb857fb0dfa20bc5c542737fddb6d893fde48be45433"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:dc0af80267edc68adf85f2a5d9be1cdf062f973db6790c1d065e45025fa26140"}, - {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:639978bccb04c42677db43c79bdaa23785dc7f9b83bfd87570da8207872f1ce5"}, - {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a99d86351f9c15e4a901fc56404b485b1462039db59288b203f8c629260a142"}, - {file = "lxml-5.4.0-cp39-cp39-win32.whl", hash = "sha256:3e6d5557989cdc3ebb5302bbdc42b439733a841891762ded9514e74f60319ad6"}, - {file = "lxml-5.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:a8c9b7f16b63e65bbba889acb436a1034a82d34fa09752d754f88d708eca80e1"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1b717b00a71b901b4667226bba282dd462c42ccf618ade12f9ba3674e1fabc55"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27a9ded0f0b52098ff89dd4c418325b987feed2ea5cc86e8860b0f844285d740"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7ce10634113651d6f383aa712a194179dcd496bd8c41e191cec2099fa09de5"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53370c26500d22b45182f98847243efb518d268374a9570409d2e2276232fd37"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6364038c519dffdbe07e3cf42e6a7f8b90c275d4d1617a69bb59734c1a2d571"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b12cb6527599808ada9eb2cd6e0e7d3d8f13fe7bbb01c6311255a15ded4c7ab4"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5f11a1526ebd0dee85e7b1e39e39a0cc0d9d03fb527f56d8457f6df48a10dc0c"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48b4afaf38bf79109bb060d9016fad014a9a48fb244e11b94f74ae366a64d252"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de6f6bb8a7840c7bf216fb83eec4e2f79f7325eca8858167b68708b929ab2172"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5cca36a194a4eb4e2ed6be36923d3cffd03dcdf477515dea687185506583d4c9"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b7c86884ad23d61b025989d99bfdd92a7351de956e01c61307cb87035960bcb1"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:53d9469ab5460402c19553b56c3648746774ecd0681b1b27ea74d5d8a3ef5590"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:56dbdbab0551532bb26c19c914848d7251d73edb507c3079d6805fa8bba5b706"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14479c2ad1cb08b62bb941ba8e0e05938524ee3c3114644df905d2331c76cd57"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32697d2ea994e0db19c1df9e40275ffe84973e4232b5c274f47e7c1ec9763cdd"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:24f6df5f24fc3385f622c0c9d63fe34604893bc1a5bdbb2dbf5870f85f9a404a"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:151d6c40bc9db11e960619d2bf2ec5829f0aaffb10b41dcf6ad2ce0f3c0b2325"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4025bf2884ac4370a3243c5aa8d66d3cb9e15d3ddd0af2d796eccc5f0244390e"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9459e6892f59ecea2e2584ee1058f5d8f629446eab52ba2305ae13a32a059530"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47fb24cc0f052f0576ea382872b3fc7e1f7e3028e53299ea751839418ade92a6"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50441c9de951a153c698b9b99992e806b71c1f36d14b154592580ff4a9d0d877"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ab339536aa798b1e17750733663d272038bf28069761d5be57cb4a9b0137b4f8"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9776af1aad5a4b4a1317242ee2bea51da54b2a7b7b48674be736d463c999f37d"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:63e7968ff83da2eb6fdda967483a7a023aa497d85ad8f05c3ad9b1f2e8c84987"}, - {file = "lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7"}, + {file = "lxml-6.0.0-cp310-cp310-win32.whl", hash = "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452"}, + {file = "lxml-6.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e"}, + {file = "lxml-6.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f"}, + {file = "lxml-6.0.0-cp311-cp311-win32.whl", hash = "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c"}, + {file = "lxml-6.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816"}, + {file = "lxml-6.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0"}, + {file = "lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a"}, + {file = "lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3"}, + {file = "lxml-6.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef"}, + {file = "lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181"}, + {file = "lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e"}, + {file = "lxml-6.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03"}, + {file = "lxml-6.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97"}, + {file = "lxml-6.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b"}, + {file = "lxml-6.0.0-cp38-cp38-win32.whl", hash = "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54"}, + {file = "lxml-6.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5"}, + {file = "lxml-6.0.0-cp39-cp39-win32.whl", hash = "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256"}, + {file = "lxml-6.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae"}, + {file = "lxml-6.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349"}, + {file = "lxml-6.0.0.tar.gz", hash = "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72"}, ] [package.extras] @@ -1160,7 +1122,6 @@ cssselect = ["cssselect (>=0.7)"] html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=3.0.11,<3.1.0)"] [[package]] name = "lxml-stubs" From cda922830eb99850b0a06fdf22721cbe318c6ca3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Jul 2025 11:57:19 -0500 Subject: [PATCH 488/691] Clean up `MetricsResource` and Prometheus hacks (#18687) Clean up `MetricsResource`, Prometheus hacks (`_set_prometheus_client_use_created_metrics`), and better document why we care about having a separate `metrics` listener type. These clean-up changes have been split out from https://github.com/element-hq/synapse/pull/18584 since that PR was closed. --- changelog.d/18687.misc | 1 + poetry.lock | 48 ++++++++-------- pyproject.toml | 4 +- synapse/app/_base.py | 33 ++++------- synapse/metrics/__init__.py | 78 +++++++++++++++++++++++++- synapse/metrics/_twisted_exposition.py | 45 --------------- tests/metrics/test_metrics.py | 31 ---------- tests/storage/test_event_metrics.py | 3 +- 8 files changed, 115 insertions(+), 128 deletions(-) create mode 100644 changelog.d/18687.misc delete mode 100644 synapse/metrics/_twisted_exposition.py diff --git a/changelog.d/18687.misc b/changelog.d/18687.misc new file mode 100644 index 0000000000..2d1ba2c790 --- /dev/null +++ b/changelog.d/18687.misc @@ -0,0 +1 @@ +Clean up `MetricsResource` and Prometheus hacks. diff --git a/poetry.lock b/poetry.lock index ebdb8e3896..4912432dbe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, @@ -435,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -478,7 +478,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -528,7 +528,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -865,7 +865,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1003,7 +1003,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1019,7 +1019,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, @@ -1260,7 +1260,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1493,7 +1493,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1699,7 +1699,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1720,7 +1720,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1736,7 +1736,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1996,7 +1996,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2056,7 +2056,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2081,7 +2081,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2109,7 +2109,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2474,7 +2474,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, @@ -2662,7 +2662,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2678,7 +2678,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2740,7 +2740,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2877,7 +2877,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3208,7 +3208,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3352,4 +3352,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "c140ab1db9e5d89d251e76c68f62cfcf5c477a923ba79f0fda186278f12af901" +content-hash = "a6965a294ca751ec2b5b0b92a050acc9afd4efb3e58550845dd32c60b74a70d1" diff --git a/pyproject.toml b/pyproject.toml index 89cd57a931..bd820bcb89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,7 +195,9 @@ pymacaroons = ">=0.13.0" msgpack = ">=0.5.2" phonenumbers = ">=8.2.0" # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. -prometheus-client = ">=0.4.0" +# `prometheus_client.metrics` was added in 0.5.0, so we require that too. +# We chose 0.6.0 as that is the current version in Debian Buster (oldstable). +prometheus-client = ">=0.6.0" # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 attrs = ">=19.2.0,!=21.1.0" diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 49ab5d680d..16aab93cd6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -286,6 +286,16 @@ def register_start( def listen_metrics(bind_addresses: StrCollection, port: int) -> None: """ Start Prometheus metrics server. + + This method runs the metrics server on a different port, in a different thread to + Synapse. This can make it more resilient to heavy load in Synapse causing metric + requests to be slow or timeout. + + Even though `start_http_server_prometheus(...)` uses `threading.Thread` behind the + scenes (where all threads share the GIL and only one thread can execute Python + bytecode at a time), this still works because the metrics thread can preempt the + Twisted reactor thread between bytecode boundaries and the metrics thread gets + scheduled with roughly equal priority to the Twisted reactor thread. """ from prometheus_client import start_http_server as start_http_server_prometheus @@ -293,32 +303,9 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) - _set_prometheus_client_use_created_metrics(False) start_http_server_prometheus(port, addr=host, registry=RegistryProxy) -def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: - """ - Sets whether prometheus_client should expose `_created`-suffixed metrics for - all gauges, histograms and summaries. - There is no programmatic way to disable this without poking at internals; - the proper way is to use an environment variable which prometheus_client - loads at import time. - - The motivation for disabling these `_created` metrics is that they're - a waste of space as they're not useful but they take up space in Prometheus. - """ - - import prometheus_client.metrics - - if hasattr(prometheus_client.metrics, "_use_created"): - prometheus_client.metrics._use_created = new_value - else: - logger.error( - "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" - ) - - def listen_manhole( bind_addresses: StrCollection, port: int, diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 7e508dba05..de750a5de2 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -25,6 +25,7 @@ import logging import os import platform import threading +from importlib import metadata from typing import ( Callable, Dict, @@ -41,7 +42,15 @@ from typing import ( ) import attr -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, Metric +from pkg_resources import parse_version +from prometheus_client import ( + CollectorRegistry, + Counter, + Gauge, + Histogram, + Metric, + generate_latest, +) from prometheus_client.core import ( REGISTRY, GaugeHistogramMetricFamily, @@ -49,11 +58,12 @@ from prometheus_client.core import ( ) from twisted.python.threadpool import ThreadPool +from twisted.web.resource import Resource +from twisted.web.server import Request # This module is imported for its side effects; flake8 needn't warn that it's unused. import synapse.metrics._reactor_metrics # noqa: F401 from synapse.metrics._gc import MIN_TIME_BETWEEN_GCS, install_gc_manager -from synapse.metrics._twisted_exposition import MetricsResource, generate_latest from synapse.metrics._types import Collector from synapse.types import StrSequence from synapse.util import SYNAPSE_VERSION @@ -81,6 +91,53 @@ terms, an endpoint you can scrape is called an *instance*, usually corresponding single process." (source: https://prometheus.io/docs/concepts/jobs_instances/) """ +CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" +""" +Content type of the latest text format for Prometheus metrics. + +Pulled directly from the prometheus_client library. +""" + + +def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: + """ + Sets whether prometheus_client should expose `_created`-suffixed metrics for + all gauges, histograms and summaries. + + There is no programmatic way in the old versions of `prometheus_client` to disable + this without poking at internals; the proper way in the old `prometheus_client` + versions (> `0.14.0` < `0.18.0`) is to use an environment variable which + prometheus_client loads at import time. For versions > `0.18.0`, we can use the + dedicated `disable_created_metrics()`/`enable_created_metrics()`. + + The motivation for disabling these `_created` metrics is that they're a waste of + space as they're not useful but they take up space in Prometheus. It's not the end + of the world if this doesn't work. + """ + import prometheus_client.metrics + + if hasattr(prometheus_client.metrics, "_use_created"): + prometheus_client.metrics._use_created = new_value + # Just log an error for old versions that don't support disabling the unecessary + # metrics. It's not the end of the world if this doesn't work as it just means extra + # wasted space taken up in Prometheus but things keep working. + elif parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): + logger.error( + "Can't disable `_created` metrics in prometheus_client (unsupported `prometheus_client` version, too old)" + ) + # If the attribute doesn't exist on a newer version, this is a sign that the brittle + # hack is broken. We should consider updating the minimum version of + # `prometheus_client` to a version (> `0.18.0`) where we can use dedicated + # `disable_created_metrics()`/`enable_created_metrics()` functions. + else: + raise Exception( + "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" + ) + + +# Set this globally so it applies wherever we generate/collect metrics +_set_prometheus_client_use_created_metrics(False) + class _RegistryProxy: @staticmethod @@ -508,6 +565,23 @@ def register_threadpool(name: str, threadpool: ThreadPool) -> None: ) +class MetricsResource(Resource): + """ + Twisted ``Resource`` that serves prometheus metrics. + """ + + isLeaf = True + + def __init__(self, registry: CollectorRegistry = REGISTRY): + self.registry = registry + + def render_GET(self, request: Request) -> bytes: + request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) + response = generate_latest(self.registry) + request.setHeader(b"Content-Length", str(len(response))) + return response + + __all__ = [ "Collector", "MetricsResource", diff --git a/synapse/metrics/_twisted_exposition.py b/synapse/metrics/_twisted_exposition.py deleted file mode 100644 index 9652ca83fb..0000000000 --- a/synapse/metrics/_twisted_exposition.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2019 Matrix.org Foundation C.I.C. -# Copyright 2015-2019 Prometheus Python Client Developers -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from prometheus_client import REGISTRY, CollectorRegistry, generate_latest - -from twisted.web.resource import Resource -from twisted.web.server import Request - -CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" - - -class MetricsResource(Resource): - """ - Twisted ``Resource`` that serves prometheus metrics. - """ - - isLeaf = True - - def __init__(self, registry: CollectorRegistry = REGISTRY): - self.registry = registry - - def render_GET(self, request: Request) -> bytes: - request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) - response = generate_latest(self.registry) - request.setHeader(b"Content-Length", str(len(response))) - return response diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index dca8dd79b1..e92d5f6dfa 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -18,14 +18,10 @@ # [This file includes modifications made by New Vector Limited] # # -from importlib import metadata from typing import Dict, Protocol, Tuple -from unittest.mock import patch -from pkg_resources import parse_version from prometheus_client.core import Sample -from synapse.app._base import _set_prometheus_client_use_created_metrics from synapse.metrics import REGISTRY, InFlightGauge, generate_latest from synapse.util.caches.deferred_cache import DeferredCache @@ -186,30 +182,3 @@ class CacheMetricsTests(unittest.HomeserverTestCase): self.assertEqual(items["synapse_util_caches_cache_size"], "1.0") self.assertEqual(items["synapse_util_caches_cache_max_size"], "777.0") - - -class PrometheusMetricsHackTestCase(unittest.HomeserverTestCase): - if parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): - skip = "prometheus-client too old" - - def test_created_metrics_disabled(self) -> None: - """ - Tests that a brittle hack, to disable `_created` metrics, works. - This involves poking at the internals of prometheus-client. - It's not the end of the world if this doesn't work. - - This test gives us a way to notice if prometheus-client changes - their internals. - """ - import prometheus_client.metrics - - PRIVATE_FLAG_NAME = "_use_created" - - # By default, the pesky `_created` metrics are enabled. - # Check this assumption is still valid. - self.assertTrue(getattr(prometheus_client.metrics, PRIVATE_FLAG_NAME)) - - with patch("prometheus_client.metrics") as mock: - setattr(mock, PRIVATE_FLAG_NAME, True) - _set_prometheus_client_use_created_metrics(False) - self.assertFalse(getattr(mock, PRIVATE_FLAG_NAME, False)) diff --git a/tests/storage/test_event_metrics.py b/tests/storage/test_event_metrics.py index 3f7ee86498..fc6e02545f 100644 --- a/tests/storage/test_event_metrics.py +++ b/tests/storage/test_event_metrics.py @@ -18,9 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # -from prometheus_client import generate_latest -from synapse.metrics import REGISTRY +from synapse.metrics import REGISTRY, generate_latest from synapse.types import UserID, create_requester from tests.unittest import HomeserverTestCase From 66504d1144c9620db378a43e1a982d807a0e6e63 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Jul 2025 15:20:42 -0500 Subject: [PATCH 489/691] Fix dirty `Cargo.lock` changes appearing after install (`base64`) (#18689) Normal install results in `Cargo.lock` changes constantly popping up for me as I navigate my branches. This was probably caused by some Depdendabot PR updating the `Cargo.toml` without `Cargo.lock` or something. ``` poetry install --extras all ``` In another PR, I've also added CI to ensure we don't leave `Cargo.lock` changes behind to avoid this annoyance in the future -> https://github.com/element-hq/synapse/pull/18693 --- changelog.d/18689.misc | 1 + rust/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18689.misc diff --git a/changelog.d/18689.misc b/changelog.d/18689.misc new file mode 100644 index 0000000000..51f0586e19 --- /dev/null +++ b/changelog.d/18689.misc @@ -0,0 +1 @@ +Fix dirty `Cargo.lock` changes appearing after install (`base64`). diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4f5ebb68b7..ab87de33ab 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -23,7 +23,7 @@ name = "synapse.synapse_rust" [dependencies] anyhow = "1.0.63" -base64 = "0.21.7" +base64 = "0.22.1" bytes = "1.6.0" headers = "0.4.0" http = "1.1.0" From 5ea2cf2484903a005f6a8af9fe8a0617e3cff373 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Jul 2025 09:06:14 +0200 Subject: [PATCH 490/691] Move device changes off the main process (#18581) The main goal of this PR is to handle device list changes onto multiple writers, off the main process, so that we can have logins happening whilst Synapse is rolling-restarting. This is quite an intrusive change, so I would advise to review this commit by commit; I tried to keep the history as clean as possible. There are a few things to consider: - the `device_list_key` in stream tokens becomes a `MultiWriterStreamToken`, which has a few implications in sync and on the storage layer - we had a split between `DeviceHandler` and `DeviceWorkerHandler` for master vs. worker process. I've kept this split, but making it rather writer vs. non-writer worker, using method overrides for doing replication calls when needed - there are a few operations that need to happen on a single worker at a time. Instead of using cross-worker locks, for now I made them run on the first writer on the list --------- Co-authored-by: Eric Eastwood --- changelog.d/18581.feature | 1 + .../complement/conf/start_for_complement.sh | 2 +- docker/configure_workers_and_start.py | 54 +- .../configuration/config_documentation.md | 2 + docs/workers.md | 21 +- schema/synapse-config.schema.yaml | 3 + synapse/config/workers.py | 46 +- synapse/handlers/appservice.py | 3 +- synapse/handlers/deactivate_account.py | 5 - synapse/handlers/device.py | 1053 ++++++++++------- synapse/handlers/devicemessage.py | 22 +- synapse/handlers/e2e_keys.py | 36 +- synapse/handlers/federation.py | 11 +- synapse/handlers/federation_event.py | 17 +- synapse/handlers/register.py | 4 - synapse/handlers/set_password.py | 16 +- synapse/handlers/sso.py | 8 - synapse/module_api/__init__.py | 7 - synapse/replication/http/__init__.py | 2 +- synapse/replication/http/devices.py | 185 ++- synapse/replication/tcp/client.py | 6 +- synapse/replication/tcp/handler.py | 7 + synapse/rest/admin/__init__.py | 4 +- synapse/rest/admin/devices.py | 26 +- synapse/rest/client/devices.py | 40 +- synapse/rest/client/keys.py | 5 +- synapse/rest/client/logout.py | 9 +- synapse/server.py | 12 +- synapse/storage/databases/main/devices.py | 935 ++++++++------- .../storage/databases/main/end_to_end_keys.py | 96 +- synapse/storage/databases/main/receipts.py | 21 +- .../storage/databases/main/registration.py | 408 ++++--- synapse/storage/databases/main/relations.py | 2 +- synapse/storage/databases/main/room.py | 5 + synapse/storage/databases/main/stream.py | 19 +- synapse/streams/events.py | 2 +- synapse/types/__init__.py | 54 +- tests/federation/test_federation_sender.py | 6 +- tests/handlers/test_device.py | 8 +- tests/handlers/test_e2e_keys.py | 4 +- tests/module_api/test_api.py | 4 +- tests/rest/admin/test_device.py | 4 +- 42 files changed, 1753 insertions(+), 1422 deletions(-) create mode 100644 changelog.d/18581.feature diff --git a/changelog.d/18581.feature b/changelog.d/18581.feature new file mode 100644 index 0000000000..c7b7017b0c --- /dev/null +++ b/changelog.d/18581.feature @@ -0,0 +1 @@ +Enable workers to write directly to the device lists stream and handle device list updates, reducing load on the main process. diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index a5e06396e2..da1b26a283 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -54,7 +54,6 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then export SYNAPSE_WORKER_TYPES="\ event_persister:2, \ background_worker, \ - frontend_proxy, \ event_creator, \ user_dir, \ media_repository, \ @@ -65,6 +64,7 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then client_reader, \ appservice, \ pusher, \ + device_lists:2, \ stream_writers=account_data+presence+receipts+to_device+typing" fi diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 102a88fad1..7909b9d932 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -178,6 +178,8 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/login$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/whoami$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$)", + "^/_matrix/client/(r0|v3)/delete_devices$", "^/_matrix/client/versions$", "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$", "^/_matrix/client/(r0|v3|unstable)/register$", @@ -194,6 +196,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$", "^/_matrix/client/(r0|v3|unstable)/capabilities$", "^/_matrix/client/(r0|v3|unstable)/notifications$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$", ], "shared_extra_conf": {}, "worker_extra_conf": "", @@ -265,13 +270,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, - "frontend_proxy": { - "app": "synapse.app.generic_worker", - "listener_resources": ["client", "replication"], - "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"], - "shared_extra_conf": {}, - "worker_extra_conf": "", - }, "account_data": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -306,6 +304,13 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "device_lists": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, "typing": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -412,16 +417,17 @@ def add_worker_roles_to_shared_config( # streams instance_map = shared_config.setdefault("instance_map", {}) - # This is a list of the stream_writers that there can be only one of. Events can be - # sharded, and therefore doesn't belong here. - singular_stream_writers = [ + # This is a list of the stream_writers. + stream_writers = { "account_data", + "events", + "device_lists", "presence", "receipts", "to_device", "typing", "push_rules", - ] + } # Worker-type specific sharding config. Now a single worker can fulfill multiple # roles, check each. @@ -431,28 +437,11 @@ def add_worker_roles_to_shared_config( if "federation_sender" in worker_types_set: shared_config.setdefault("federation_sender_instances", []).append(worker_name) - if "event_persister" in worker_types_set: - # Event persisters write to the events stream, so we need to update - # the list of event stream writers - shared_config.setdefault("stream_writers", {}).setdefault("events", []).append( - worker_name - ) - - # Map of stream writer instance names to host/ports combos - if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False): - instance_map[worker_name] = { - "path": f"/run/worker.{worker_port}", - } - else: - instance_map[worker_name] = { - "host": "localhost", - "port": worker_port, - } # Update the list of stream writers. It's convenient that the name of the worker # type is the same as the stream to write. Iterate over the whole list in case there # is more than one. for worker in worker_types_set: - if worker in singular_stream_writers: + if worker in stream_writers: shared_config.setdefault("stream_writers", {}).setdefault( worker, [] ).append(worker_name) @@ -876,6 +865,13 @@ def generate_worker_files( else: healthcheck_urls.append("http://localhost:%d/health" % (worker_port,)) + # Special case for event_persister: those are just workers that write to + # the `events` stream. For other workers, the worker name is the same + # name of the stream they write to, but for some reason it is not the + # case for event_persister. + if "event_persister" in worker_types_set: + worker_types_set.add("events") + # Update the shared config with sharding-related options if necessary add_worker_roles_to_shared_config( shared_config, worker_types_set, worker_name, worker_port diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e47a32f510..fc838a1f0e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4341,6 +4341,8 @@ This setting has the following sub-options: * `push_rules` (string): Name of a worker assigned to the `push_rules` stream. +* `device_lists` (string): Name of a worker assigned to the `device_lists` stream. + Example configuration: ```yaml stream_writers: diff --git a/docs/workers.md b/docs/workers.md index 45a00696f3..7881aeebbe 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -238,7 +238,8 @@ information. ^/_matrix/client/unstable/im.nheko.summary/summary/.*$ ^/_matrix/client/(r0|v3|unstable)/account/3pid$ ^/_matrix/client/(r0|v3|unstable)/account/whoami$ - ^/_matrix/client/(r0|v3|unstable)/devices$ + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$) ^/_matrix/client/versions$ ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ @@ -257,7 +258,9 @@ information. ^/_matrix/client/(r0|v3|unstable)/keys/changes$ ^/_matrix/client/(r0|v3|unstable)/keys/claim$ ^/_matrix/client/(r0|v3|unstable)/room_keys/ - ^/_matrix/client/(r0|v3|unstable)/keys/upload$ + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ # Registration/login requests ^/_matrix/client/(api/v1|r0|v3|unstable)/login$ @@ -282,7 +285,6 @@ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events - ^/_matrix/client/(api/v1|r0|v3|unstable)/devices/ # Account data requests ^/_matrix/client/(r0|v3|unstable)/.*/tags @@ -329,7 +331,6 @@ set to `true`), the following endpoints can be handled by the worker: ^/_synapse/admin/v2/users/[^/]+$ ^/_synapse/admin/v1/username_available$ ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ - # Only the GET method: ^/_synapse/admin/v1/users/[^/]+/devices$ Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) @@ -550,6 +551,18 @@ the stream writer for the `push_rules` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ +##### The `device_lists` stream + +The `device_lists` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly one of the workers +configured as stream writer for the `device_lists` stream: + + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices/ + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 9a244856e6..72b80253f8 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5383,6 +5383,9 @@ properties: push_rules: type: string description: Name of a worker assigned to the `push_rules` stream. + device_lists: + type: string + description: Name of a worker assigned to the `device_lists` stream. default: {} examples: - events: worker1 diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 69036f9b52..c0c8a13861 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -134,40 +134,44 @@ class WriterLocations: can only be a single instance. account_data: The instances that write to the account data streams. Currently can only be a single instance. - receipts: The instances that write to the receipts stream. Currently - can only be a single instance. + receipts: The instances that write to the receipts stream. presence: The instances that write to the presence stream. Currently can only be a single instance. push_rules: The instances that write to the push stream. Currently can only be a single instance. + device_lists: The instances that write to the device list stream. """ events: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) typing: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) to_device: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) account_data: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) receipts: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) presence: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) push_rules: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) + device_lists: List[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) @@ -358,7 +362,10 @@ class WorkerConfig(Config): ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: - if instance != "master" and instance not in self.instance_map: + if ( + instance != MAIN_PROCESS_INSTANCE_NAME + and instance not in self.instance_map + ): raise ConfigError( "Instance %r is configured to write %s but does not appear in `instance_map` config." % (instance, stream) @@ -397,6 +404,11 @@ class WorkerConfig(Config): "Must only specify one instance to handle `push` messages." ) + if len(self.writers.device_lists) == 0: + raise ConfigError( + "Must specify at least one instance to handle `device_lists` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) @@ -419,9 +431,12 @@ class WorkerConfig(Config): # # No effort is made to ensure only a single instance of these tasks is # running. - background_tasks_instance = config.get("run_background_tasks_on") or "master" + background_tasks_instance = ( + config.get("run_background_tasks_on") or MAIN_PROCESS_INSTANCE_NAME + ) self.run_background_tasks = ( - self.worker_name is None and background_tasks_instance == "master" + self.worker_name is None + and background_tasks_instance == MAIN_PROCESS_INSTANCE_NAME ) or self.worker_name == background_tasks_instance self.should_notify_appservices = self._should_this_worker_perform_duty( @@ -493,9 +508,10 @@ class WorkerConfig(Config): # 'don't run here'. new_option_should_run_here = None if new_option_name in config: - designated_worker = config[new_option_name] or "master" + designated_worker = config[new_option_name] or MAIN_PROCESS_INSTANCE_NAME new_option_should_run_here = ( - designated_worker == "master" and self.worker_name is None + designated_worker == MAIN_PROCESS_INSTANCE_NAME + and self.worker_name is None ) or designated_worker == self.worker_name legacy_option_should_run_here = None @@ -592,7 +608,7 @@ class WorkerConfig(Config): # If no worker instances are set we check if the legacy option # is set, which means use the main process. if legacy_option: - worker_instances = ["master"] + worker_instances = [MAIN_PROCESS_INSTANCE_NAME] if self.worker_app == legacy_app_name: if legacy_option: diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8c5308b522..93224d0c1b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -638,7 +638,8 @@ class ApplicationServicesHandler: # Fetch the users who have modified their device list since then. users_with_changed_device_lists = await self.store.get_all_devices_changed( - from_key, to_key=new_key + MultiWriterStreamToken(stream=from_key), + to_key=MultiWriterStreamToken(stream=new_key), ) # Filter out any users the application service is not interested in diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 42e53d920a..8d4d84bed1 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership from synapse.api.errors import SynapseError -from synapse.handlers.device import DeviceHandler from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import Codes, Requester, UserID, create_requester @@ -84,10 +83,6 @@ class DeactivateAccountHandler: Returns: True if identity server supports removing threepids, otherwise False. """ - - # This can only be called on the main process. - assert isinstance(self._device_handler, DeviceHandler) - # Check if this user can be deactivated if not await self._third_party_rules.check_can_deactivate_user( user_id, by_admin diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 65fbb48768..80d49fc18d 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -20,6 +20,7 @@ # # import logging +import random from threading import Lock from typing import ( TYPE_CHECKING, @@ -31,6 +32,7 @@ from typing import ( Optional, Set, Tuple, + cast, ) from synapse.api import errors @@ -48,6 +50,13 @@ from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) +from synapse.replication.http.devices import ( + ReplicationDeviceHandleRoomUnPartialStated, + ReplicationHandleNewDeviceUpdateRestServlet, + ReplicationMultiUserDevicesResyncRestServlet, + ReplicationNotifyDeviceUpdateRestServlet, + ReplicationNotifyUserSignatureUpdateRestServlet, +) from synapse.storage.databases.main.client_ips import DeviceLastConnectionInfo from synapse.storage.databases.main.roommember import EventIdMembership from synapse.storage.databases.main.state_deltas import StateDelta @@ -75,6 +84,7 @@ from synapse.util.retryutils import ( ) if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -84,32 +94,319 @@ MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL_MS = 24 * 60 * 60 * 1000 -class DeviceWorkerHandler: +def _check_device_name_length(name: Optional[str]) -> None: + """ + Checks whether a device name is longer than the maximum allowed length. + + Args: + name: The name of the device. + + Raises: + SynapseError: if the device name is too long. + """ + if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: + raise SynapseError( + 400, + "Device display name is too long (max %i)" % (MAX_DEVICE_DISPLAY_NAME_LEN,), + errcode=Codes.TOO_LARGE, + ) + + +class DeviceHandler: + """ + Handles most things related to devices. This doesn't do any writing to the + device list stream on its own, and will call to device list writers through + replication when necessary (see DeviceWriterHandler). + """ + device_list_updater: "DeviceListWorkerUpdater" + store: "GenericWorkerStore" def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = hs - self.store = hs.get_datastores().main + self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() self._appservice_handler = hs.get_application_service_handler() self._state_storage = hs.get_storage_controllers().state self._auth_handler = hs.get_auth_handler() + self._account_data_handler = hs.get_account_data_handler() self._event_sources = hs.get_event_sources() - self.server_name = hs.hostname self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._query_appservices_for_keys = ( hs.config.experimental.msc3984_appservice_key_query ) self._task_scheduler = hs.get_task_scheduler() + self._dont_notify_new_devices_for = ( + hs.config.registration.dont_notify_new_devices_for + ) + self.device_list_updater = DeviceListWorkerUpdater(hs) self._task_scheduler.register_action( self._delete_device_messages, DELETE_DEVICE_MSGS_TASK_NAME ) + self._device_list_writers = hs.config.worker.writers.device_lists + + # Ensure a few operations are only running on the first device list writer + # + # This is needed because of a few linearizers in the DeviceListUpdater, + # and avoid using cross-worker locks. + # + # The main logic update is that the DeviceListUpdater is now only + # instantiated on the first device list writer, and a few methods that + # were safe to move to any worker were moved to the DeviceListWorkerUpdater + # This must be kept in sync with DeviceListWorkerUpdater + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] + + self._notify_device_update_client = ( + ReplicationNotifyDeviceUpdateRestServlet.make_client(hs) + ) + self._notify_user_signature_update_client = ( + ReplicationNotifyUserSignatureUpdateRestServlet.make_client(hs) + ) + self._handle_new_device_update_client = ( + ReplicationHandleNewDeviceUpdateRestServlet.make_client(hs) + ) + self._handle_room_un_partial_stated_client = ( + ReplicationDeviceHandleRoomUnPartialStated.make_client(hs) + ) + + # The EDUs are handled on a single writer, as it needs to acquire a + # per-user lock, for which it is cheaper to use in-memory linearizers + # than cross-worker locks. + hs.get_federation_registry().register_instances_for_edu( + EduTypes.DEVICE_LIST_UPDATE, + [self._main_device_list_writer], + ) + + self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after + + if ( + hs.config.worker.run_background_tasks + and self._delete_stale_devices_after is not None + ): + self.clock.looping_call( + run_as_background_process, + DELETE_STALE_DEVICES_INTERVAL_MS, + "delete_stale_devices", + self._delete_stale_devices, + ) + + async def _delete_stale_devices(self) -> None: + """Background task that deletes devices which haven't been accessed for more than + a configured time period. + """ + # We should only be running this job if the config option is defined. + assert self._delete_stale_devices_after is not None + now_ms = self.clock.time_msec() + since_ms = now_ms - self._delete_stale_devices_after + devices = await self.store.get_local_devices_not_accessed_since(since_ms) + + for user_id, user_devices in devices.items(): + await self.delete_devices(user_id, user_devices) + + async def check_device_registered( + self, + user_id: str, + device_id: Optional[str], + initial_device_display_name: Optional[str] = None, + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> str: + """ + If the given device has not been registered, register it with the + supplied display name. + + If no device_id is supplied, we make one up. + + Args: + user_id: @user:id + device_id: device id supplied by client + initial_device_display_name: device display name from client + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from the SSO IdP. + Returns: + device id (generated if none was supplied) + """ + + _check_device_name_length(initial_device_display_name) + + # Check if we should send out device lists updates for this new device. + notify = user_id not in self._dont_notify_new_devices_for + + if device_id is not None: + new_device = await self.store.store_device( + user_id=user_id, + device_id=device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [device_id]) + return device_id + + # if the device id is not specified, we'll autogen one, but loop a few + # times in case of a clash. + attempts = 0 + while attempts < 5: + new_device_id = stringutils.random_string(10).upper() + new_device = await self.store.store_device( + user_id=user_id, + device_id=new_device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [new_device_id]) + return new_device_id + attempts += 1 + + raise errors.StoreError(500, "Couldn't generate a device ID.") + + @trace + async def delete_all_devices_for_user( + self, user_id: str, except_device_id: Optional[str] = None + ) -> None: + """Delete all of the user's devices + + Args: + user_id: The user to remove all devices from + except_device_id: optional device id which should not be deleted + """ + device_map = await self.store.get_devices_by_user(user_id) + if except_device_id is not None: + device_map.pop(except_device_id, None) + user_device_ids = device_map.keys() + await self.delete_devices(user_id, user_device_ids) + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Delete several devices + + Args: + user_id: The user to delete devices from. + device_ids: The list of device IDs to delete + """ + to_device_stream_id = self._event_sources.get_current_token().to_device_key + + try: + await self.store.delete_devices(user_id, device_ids) + except errors.StoreError as e: + if e.code == 404: + # no match + set_tag("error", True) + set_tag("reason", "User doesn't have that device id.") + else: + raise + + # Delete data specific to each device. Not optimised as its an + # experimental MSC. + if self.hs.config.experimental.msc3890_enabled: + for device_id in device_ids: + # Remove any local notification settings for this device in accordance + # with MSC3890. + await self._account_data_handler.remove_account_data_for_user( + user_id, + f"org.matrix.msc3890.local_notification_settings.{device_id}", + ) + + # If we're deleting a lot of devices, a bunch of them may not have any + # to-device messages queued up. We filter those out to avoid scheduling + # unnecessary tasks. + devices_with_messages = await self.store.get_devices_with_messages( + user_id, device_ids + ) + for device_id in devices_with_messages: + # Delete device messages asynchronously and in batches using the task scheduler + # We specify an upper stream id to avoid deleting non delivered messages + # if an user re-uses a device ID. + await self._task_scheduler.schedule_task( + DELETE_DEVICE_MSGS_TASK_NAME, + resource_id=device_id, + params={ + "user_id": user_id, + "device_id": device_id, + "up_to_stream_id": to_device_stream_id, + }, + ) + + await self._auth_handler.delete_access_tokens_for_devices( + user_id, device_ids=device_ids + ) + + # Pushers are deleted after `delete_access_tokens_for_user` is called so that + # modules using `on_logged_out` hook can use them if needed. + await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) + + await self.notify_device_update(user_id, device_ids) + + async def upsert_device( + self, user_id: str, device_id: str, display_name: Optional[str] = None + ) -> bool: + """Create or update a device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + display_name: The new display name for this device. + + Returns: + True if the device was created, False if it was updated. + + """ + + # Reject a new displayname which is too long. + _check_device_name_length(display_name) + + created = await self.store.store_device( + user_id, + device_id, + initial_device_display_name=display_name, + ) + + if not created: + await self.store.update_device( + user_id, + device_id, + new_display_name=display_name, + ) + + await self.notify_device_update(user_id, [device_id]) + return created + + async def update_device(self, user_id: str, device_id: str, content: dict) -> None: + """Update the given device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + content: body of update request + """ + + # Reject a new displayname which is too long. + new_display_name = content.get("display_name") + + _check_device_name_length(new_display_name) + + try: + await self.store.update_device( + user_id, device_id, new_display_name=new_display_name + ) + await self.notify_device_update(user_id, [device_id]) + except errors.StoreError as e: + if e.code == 404: + raise errors.NotFoundError() + else: + raise + @trace async def get_devices_by_user(self, user_id: str) -> List[JsonDict]: """ @@ -146,6 +443,98 @@ class DeviceWorkerHandler: """ return await self.store.get_dehydrated_device(user_id) + async def store_dehydrated_device( + self, + user_id: str, + device_id: Optional[str], + device_data: JsonDict, + initial_device_display_name: Optional[str] = None, + keys_for_device: Optional[JsonDict] = None, + ) -> str: + """Store a dehydrated device for a user, optionally storing the keys associated with + it as well. If the user had a previous dehydrated device, it is removed. + + Args: + user_id: the user that we are storing the device for + device_id: device id supplied by client + device_data: the dehydrated device information + initial_device_display_name: The display name to use for the device + keys_for_device: keys for the dehydrated device + Returns: + device id of the dehydrated device + """ + device_id = await self.check_device_registered( + user_id, + device_id, + initial_device_display_name, + ) + + time_now = self.clock.time_msec() + + old_device_id = await self.store.store_dehydrated_device( + user_id, device_id, device_data, time_now, keys_for_device + ) + + if old_device_id is not None: + await self.delete_devices(user_id, [old_device_id]) + + return device_id + + async def rehydrate_device( + self, user_id: str, access_token: str, device_id: str + ) -> dict: + """Process a rehydration request from the user. + + Args: + user_id: the user who is rehydrating the device + access_token: the access token used for the request + device_id: the ID of the device that will be rehydrated + Returns: + a dict containing {"success": True} + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + # If the dehydrated device was successfully deleted (the device ID + # matched the stored dehydrated device), then modify the access + # token and refresh token to use the dehydrated device's ID and + # copy the old device display name to the dehydrated device, + # and destroy the old device ID + old_device_id = await self.store.set_device_for_access_token( + access_token, device_id + ) + await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) + old_device = await self.store.get_device(user_id, old_device_id) + if old_device is None: + raise errors.NotFoundError() + await self.store.update_device(user_id, device_id, old_device["display_name"]) + # can't call self.delete_device because that will clobber the + # access token so call the storage layer directly + await self.store.delete_devices(user_id, [old_device_id]) + + # tell everyone that the old device is gone and that the dehydrated + # device has a new display name + await self.notify_device_update(user_id, [old_device_id, device_id]) + + return {"success": True} + + async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: + """ + Delete a stored dehydrated device. + + Args: + user_id: the user_id to delete the device from + device_id: id of the dehydrated device to delete + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + await self.delete_devices(user_id, [device_id]) + @trace async def get_device(self, user_id: str, device_id: str) -> JsonDict: """Retrieve the given device @@ -484,10 +873,53 @@ class DeviceWorkerHandler: gone from partial to full state. """ - # TODO(faster_joins): worker mode support - # https://github.com/matrix-org/synapse/issues/12994 - logger.error( - "Trying handling device list state for partial join: not supported on workers." + await self._handle_room_un_partial_stated_client( + instance_name=random.choice(self._device_list_writers), + room_id=room_id, + ) + + @trace + @measure_func("notify_device_update") + async def notify_device_update( + self, user_id: str, device_ids: StrCollection + ) -> None: + """Notify that a user's device(s) has changed. Pokes the notifier, and + remote servers if the user is local. + + Args: + user_id: The Matrix ID of the user who's device list has been updated. + device_ids: The device IDs that have changed. + """ + await self._notify_device_update_client( + instance_name=random.choice(self._device_list_writers), + user_id=user_id, + device_ids=list(device_ids), + ) + + async def notify_user_signature_update( + self, + from_user_id: str, + user_ids: List[str], + ) -> None: + """Notify a device writer that a user have made new signatures of other users. + + Args: + from_user_id: The Matrix ID of the user who's signatures have been updated. + user_ids: The Matrix IDs of the users that have changed. + """ + await self._notify_user_signature_update_client( + instance_name=random.choice(self._device_list_writers), + from_user_id=from_user_id, + user_ids=user_ids, + ) + + async def handle_new_device_update(self) -> None: + """Wake up a device writer to send local device list changes as federation outbound pokes.""" + # This is only sent to the first device writer to avoid cross-worker + # locks in _handle_new_device_update_async, as it makes assumptions + # about being the only instance running. + await self._handle_new_device_update_client( + instance_name=self._device_list_writers[0], ) DEVICE_MSGS_DELETE_BATCH_LIMIT = 1000 @@ -511,39 +943,43 @@ class DeviceWorkerHandler: device_id=device_id, from_stream_id=from_stream_id, to_stream_id=up_to_stream_id, - limit=DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, + limit=DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, ) if from_stream_id is None: return TaskStatus.COMPLETE, None, None - await self.clock.sleep(DeviceHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0) + await self.clock.sleep( + DeviceWriterHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0 + ) -class DeviceHandler(DeviceWorkerHandler): - device_list_updater: "DeviceListUpdater" +class DeviceWriterHandler(DeviceHandler): + """ + Superclass of the DeviceHandler which gets instantiated on workers that can + write to the device list stream. + """ def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.server_name = hs.hostname # nb must be called this for @measure_func - self.clock = hs.get_clock() # nb must be called this for @measure_func - self.federation_sender = hs.get_federation_sender() - self._account_data_handler = hs.get_account_data_handler() + # We only need to poke the federation sender explicitly if its on the + # same instance. Other federation sender instances will get notified by + # `synapse.app.generic_worker.FederationSenderHandler` when it sees it + # in the device lists stream. + self.federation_sender = None + if hs.should_send_federation(): + self.federation_sender = hs.get_federation_sender() + self._storage_controllers = hs.get_storage_controllers() - self.db_pool = hs.get_datastores().main.db_pool - self._dont_notify_new_devices_for = ( - hs.config.registration.dont_notify_new_devices_for - ) - - self.device_list_updater = DeviceListUpdater(hs, self) - - federation_registry = hs.get_federation_registry() - - federation_registry.register_edu_handler( - EduTypes.DEVICE_LIST_UPDATE, - self.device_list_updater.incoming_device_list_update, + # There are a few things that are only handled on the main device list + # writer to avoid cross-worker locks + # + # This mainly concerns the `DeviceListUpdater` class, which is only + # instantiated on the first device list writer. + self._is_main_device_list_writer = ( + hs.get_instance_name() == self._main_device_list_writer ) # Whether `_handle_new_device_update_async` is currently processing. @@ -553,250 +989,22 @@ class DeviceHandler(DeviceWorkerHandler): # processing. self._handle_new_device_update_new_data = False - # On start up check if there are any updates pending. - hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - - self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after - - # Ideally we would run this on a worker and condition this on the - # "run_background_tasks_on" setting, but this would mean making the notification - # of device list changes over federation work on workers, which is nontrivial. - if self._delete_stale_devices_after is not None: - self.clock.looping_call( - run_as_background_process, - DELETE_STALE_DEVICES_INTERVAL_MS, - "delete_stale_devices", - self._delete_stale_devices, + # Only the main device list writer handles device list EDUs and converts + # device list updates to outbound federation pokes. This allows us to + # use in-memory per-user locks instead of cross-worker locks, and + # simplifies the logic for converting outbound pokes. This makes the + # device_list writers a little bit unbalanced in terms of load, but + # still unlocks local device changes (and therefore login/logouts) when + # rolling-restarting Synapse. + if self._is_main_device_list_writer: + # On start up check if there are any updates pending. + hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) + self.device_list_updater = DeviceListUpdater(hs, self) + hs.get_federation_registry().register_edu_handler( + EduTypes.DEVICE_LIST_UPDATE, + self.device_list_updater.incoming_device_list_update, ) - def _check_device_name_length(self, name: Optional[str]) -> None: - """ - Checks whether a device name is longer than the maximum allowed length. - - Args: - name: The name of the device. - - Raises: - SynapseError: if the device name is too long. - """ - if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: - raise SynapseError( - 400, - "Device display name is too long (max %i)" - % (MAX_DEVICE_DISPLAY_NAME_LEN,), - errcode=Codes.TOO_LARGE, - ) - - async def check_device_registered( - self, - user_id: str, - device_id: Optional[str], - initial_device_display_name: Optional[str] = None, - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> str: - """ - If the given device has not been registered, register it with the - supplied display name. - - If no device_id is supplied, we make one up. - - Args: - user_id: @user:id - device_id: device id supplied by client - initial_device_display_name: device display name from client - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from the SSO IdP. - Returns: - device id (generated if none was supplied) - """ - - self._check_device_name_length(initial_device_display_name) - - # Check if we should send out device lists updates for this new device. - notify = user_id not in self._dont_notify_new_devices_for - - if device_id is not None: - new_device = await self.store.store_device( - user_id=user_id, - device_id=device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [device_id]) - return device_id - - # if the device id is not specified, we'll autogen one, but loop a few - # times in case of a clash. - attempts = 0 - while attempts < 5: - new_device_id = stringutils.random_string(10).upper() - new_device = await self.store.store_device( - user_id=user_id, - device_id=new_device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [new_device_id]) - return new_device_id - attempts += 1 - - raise errors.StoreError(500, "Couldn't generate a device ID.") - - async def _delete_stale_devices(self) -> None: - """Background task that deletes devices which haven't been accessed for more than - a configured time period. - """ - # We should only be running this job if the config option is defined. - assert self._delete_stale_devices_after is not None - now_ms = self.clock.time_msec() - since_ms = now_ms - self._delete_stale_devices_after - devices = await self.store.get_local_devices_not_accessed_since(since_ms) - - for user_id, user_devices in devices.items(): - await self.delete_devices(user_id, user_devices) - - @trace - async def delete_all_devices_for_user( - self, user_id: str, except_device_id: Optional[str] = None - ) -> None: - """Delete all of the user's devices - - Args: - user_id: The user to remove all devices from - except_device_id: optional device id which should not be deleted - """ - device_map = await self.store.get_devices_by_user(user_id) - if except_device_id is not None: - device_map.pop(except_device_id, None) - user_device_ids = device_map.keys() - await self.delete_devices(user_id, user_device_ids) - - async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: - """Delete several devices - - Args: - user_id: The user to delete devices from. - device_ids: The list of device IDs to delete - """ - to_device_stream_id = self._event_sources.get_current_token().to_device_key - - try: - await self.store.delete_devices(user_id, device_ids) - except errors.StoreError as e: - if e.code == 404: - # no match - set_tag("error", True) - set_tag("reason", "User doesn't have that device id.") - else: - raise - - # Delete data specific to each device. Not optimised as its an - # experimental MSC. - if self.hs.config.experimental.msc3890_enabled: - for device_id in device_ids: - # Remove any local notification settings for this device in accordance - # with MSC3890. - await self._account_data_handler.remove_account_data_for_user( - user_id, - f"org.matrix.msc3890.local_notification_settings.{device_id}", - ) - - # If we're deleting a lot of devices, a bunch of them may not have any - # to-device messages queued up. We filter those out to avoid scheduling - # unnecessary tasks. - devices_with_messages = await self.store.get_devices_with_messages( - user_id, device_ids - ) - for device_id in devices_with_messages: - # Delete device messages asynchronously and in batches using the task scheduler - # We specify an upper stream id to avoid deleting non delivered messages - # if an user re-uses a device ID. - await self._task_scheduler.schedule_task( - DELETE_DEVICE_MSGS_TASK_NAME, - resource_id=device_id, - params={ - "user_id": user_id, - "device_id": device_id, - "up_to_stream_id": to_device_stream_id, - }, - ) - - await self._auth_handler.delete_access_tokens_for_devices( - user_id, device_ids=device_ids - ) - - # Pushers are deleted after `delete_access_tokens_for_user` is called so that - # modules using `on_logged_out` hook can use them if needed. - await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) - - await self.notify_device_update(user_id, device_ids) - - async def upsert_device( - self, user_id: str, device_id: str, display_name: Optional[str] = None - ) -> bool: - """Create or update a device - - Args: - user_id: The user to update devices of. - device_id: The device to update. - display_name: The new display name for this device. - - Returns: - True if the device was created, False if it was updated. - - """ - - # Reject a new displayname which is too long. - self._check_device_name_length(display_name) - - created = await self.store.store_device( - user_id, - device_id, - initial_device_display_name=display_name, - ) - - if not created: - await self.store.update_device( - user_id, - device_id, - new_display_name=display_name, - ) - - await self.notify_device_update(user_id, [device_id]) - return created - - async def update_device(self, user_id: str, device_id: str, content: dict) -> None: - """Update the given device - - Args: - user_id: The user to update devices of. - device_id: The device to update. - content: body of update request - """ - - # Reject a new displayname which is too long. - new_display_name = content.get("display_name") - - self._check_device_name_length(new_display_name) - - try: - await self.store.update_device( - user_id, device_id, new_display_name=new_display_name - ) - await self.notify_device_update(user_id, [device_id]) - except errors.StoreError as e: - if e.code == 404: - raise errors.NotFoundError() - else: - raise - @trace @measure_func("notify_device_update") async def notify_device_update( @@ -839,7 +1047,7 @@ class DeviceHandler(DeviceWorkerHandler): # We may need to do some processing asynchronously for local user IDs. if self.hs.is_mine_id(user_id): - self._handle_new_device_update_async() + await self.handle_new_device_update() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] @@ -859,97 +1067,16 @@ class DeviceHandler(DeviceWorkerHandler): StreamKeyType.DEVICE_LIST, position, users=[from_user_id] ) - async def store_dehydrated_device( - self, - user_id: str, - device_id: Optional[str], - device_data: JsonDict, - initial_device_display_name: Optional[str] = None, - keys_for_device: Optional[JsonDict] = None, - ) -> str: - """Store a dehydrated device for a user, optionally storing the keys associated with - it as well. If the user had a previous dehydrated device, it is removed. + async def handle_new_device_update(self) -> None: + # _handle_new_device_update_async is only called on the first device + # writer, as it makes assumptions about only having one instance running + # at a time. If this is not the first device writer, we defer to the + # superclass, which will make the call go through replication. + if not self._is_main_device_list_writer: + return await super().handle_new_device_update() - Args: - user_id: the user that we are storing the device for - device_id: device id supplied by client - device_data: the dehydrated device information - initial_device_display_name: The display name to use for the device - keys_for_device: keys for the dehydrated device - Returns: - device id of the dehydrated device - """ - device_id = await self.check_device_registered( - user_id, - device_id, - initial_device_display_name, - ) - - time_now = self.clock.time_msec() - - old_device_id = await self.store.store_dehydrated_device( - user_id, device_id, device_data, time_now, keys_for_device - ) - - if old_device_id is not None: - await self.delete_devices(user_id, [old_device_id]) - - return device_id - - async def rehydrate_device( - self, user_id: str, access_token: str, device_id: str - ) -> dict: - """Process a rehydration request from the user. - - Args: - user_id: the user who is rehydrating the device - access_token: the access token used for the request - device_id: the ID of the device that will be rehydrated - Returns: - a dict containing {"success": True} - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - # If the dehydrated device was successfully deleted (the device ID - # matched the stored dehydrated device), then modify the access - # token and refresh token to use the dehydrated device's ID and - # copy the old device display name to the dehydrated device, - # and destroy the old device ID - old_device_id = await self.store.set_device_for_access_token( - access_token, device_id - ) - await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) - old_device = await self.store.get_device(user_id, old_device_id) - if old_device is None: - raise errors.NotFoundError() - await self.store.update_device(user_id, device_id, old_device["display_name"]) - # can't call self.delete_device because that will clobber the - # access token so call the storage layer directly - await self.store.delete_devices(user_id, [old_device_id]) - - # tell everyone that the old device is gone and that the dehydrated - # device has a new display name - await self.notify_device_update(user_id, [old_device_id, device_id]) - - return {"success": True} - - async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: - """ - Delete a stored dehydrated device. - - Args: - user_id: the user_id to delete the device from - device_id: id of the dehydrated device to delete - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - await self.delete_devices(user_id, [device_id]) + self._handle_new_device_update_async() + return @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: @@ -959,12 +1086,27 @@ class DeviceHandler(DeviceWorkerHandler): This happens in the background so as not to block the original request that generated the device update. """ + # This should only ever be called on the main device list writer, as it + # expects to only have a single instance of this loop running at a time. + # See `handle_new_device_update`. + assert self._is_main_device_list_writer + if self._handle_new_device_update_is_processing: self._handle_new_device_update_new_data = True return self._handle_new_device_update_is_processing = True + # Note that this logic only deals with the minimum stream ID, and not + # the full stream token. This means that oubound pokes are only sent + # once every writer on the device_lists stream has caught up. This is + # fine, it may only introduces a bit of lag on the outbound pokes. + # To fix this, 'device_lists_changes_converted_stream_position' would + # need to include the full stream token instead of just a stream ID. + # We could also consider have each writer converting their own device + # list updates, but that can quickly become complex to handle changes in + # the list of device writers. + # The stream ID we processed previous iteration (if any), and the set of # hosts we've already poked about for this update. This is so that we # don't poke the same remote server about the same update repeatedly. @@ -976,7 +1118,7 @@ class DeviceHandler(DeviceWorkerHandler): while True: self._handle_new_device_update_new_data = False - max_stream_id = self.store.get_device_stream_token() + max_stream_id = self.store.get_device_stream_token().stream rows = await self.store.get_uncoverted_outbound_room_pokes( stream_id, room_id ) @@ -1041,7 +1183,7 @@ class DeviceHandler(DeviceWorkerHandler): # Notify replication that we've updated the device list stream. self.notifier.notify_replication() - if hosts: + if hosts and self.federation_sender: logger.info( "Sending device list update notif for %r to: %r", user_id, @@ -1161,9 +1303,10 @@ class DeviceHandler(DeviceWorkerHandler): # Notify things that device lists need to be sent out. self.notifier.notify_replication() - await self.federation_sender.send_device_messages( - potentially_changed_hosts, immediate=False - ) + if self.federation_sender: + await self.federation_sender.send_device_messages( + potentially_changed_hosts, immediate=False + ) def _update_device_from_client_ips( @@ -1180,19 +1323,21 @@ def _update_device_from_client_ips( class DeviceListWorkerUpdater: - "Handles incoming device list updates from federation and contacts the main process over replication" + "Handles incoming device list updates from federation and contacts the main device list writer over replication" def __init__(self, hs: "HomeServer"): - from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, - ) - + self.store = hs.get_datastores().main + self._notifier = hs.get_notifier() + # On which instance the DeviceListUpdater is running + # Must be kept in sync with DeviceHandler + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] self._multi_user_device_resync_client = ( ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) ) async def multi_user_device_resync( - self, user_ids: List[str], mark_failed_as_stale: bool = True + self, + user_ids: List[str], ) -> Dict[str, Optional[JsonMapping]]: """ Like `user_device_resync` but operates on multiple users **from the same origin** @@ -1201,27 +1346,104 @@ class DeviceListWorkerUpdater: Returns: Dict from User ID to the same Dict as `user_device_resync`. """ - # mark_failed_as_stale is not sent. Ensure this doesn't break expectations. - assert mark_failed_as_stale if not user_ids: # Shortcut empty requests return {} - return await self._multi_user_device_resync_client(user_ids=user_ids) + # This uses a per-user-id lock; to avoid using cross-worker locks, we + # forward the request to the main device list writer. + # See DeviceListUpdater + return await self._multi_user_device_resync_client( + instance_name=self._main_device_list_writer, + user_ids=user_ids, + ) + + async def process_cross_signing_key_update( + self, + user_id: str, + master_key: Optional[JsonDict], + self_signing_key: Optional[JsonDict], + ) -> List[str]: + """Process the given new master and self-signing key for the given remote user. + + Args: + user_id: The ID of the user these keys are for. + master_key: The dict of the cross-signing master key as returned by the + remote server. + self_signing_key: The dict of the cross-signing self-signing key as returned + by the remote server. + + Return: + The device IDs for the given keys. + """ + device_ids = [] + + current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) + current_keys = current_keys_map.get(user_id) or {} + + if master_key and master_key != current_keys.get("master"): + await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) + _, verify_key = get_verify_key_from_cross_signing_key(master_key) + # verify_key is a VerifyKey from signedjson, which uses + # .version to denote the portion of the key ID after the + # algorithm and colon, which is the device ID + device_ids.append(verify_key.version) + if self_signing_key and self_signing_key != current_keys.get("self_signing"): + await self.store.set_e2e_cross_signing_key( + user_id, "self_signing", self_signing_key + ) + _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) + device_ids.append(verify_key.version) + + return device_ids + + async def handle_room_un_partial_stated(self, room_id: str) -> None: + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + """ + + pending_updates = ( + await self.store.get_pending_remote_device_list_updates_for_room(room_id) + ) + + for user_id, device_id in pending_updates: + logger.info( + "Got pending device list update in room %s: %s / %s", + room_id, + user_id, + device_id, + ) + position = await self.store.add_device_change_to_streams( + user_id, + [device_id], + room_ids=[room_id], + ) + + if not position: + # This should only happen if there are no updates, which + # shouldn't happen when we've passed in a non-empty set of + # device IDs. + continue + + self._notifier.on_new_event( + StreamKeyType.DEVICE_LIST, position, rooms=[room_id] + ) class DeviceListUpdater(DeviceListWorkerUpdater): - "Handles incoming device list updates from federation and updates the DB" + """Handles incoming device list updates from federation and updates the DB. + + This is only instanciated on the first device list writer, as it uses + in-process linearizers for some operations.""" + + def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): + super().__init__(hs) - def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): - self.server_name = hs.hostname - self.store = hs.get_datastores().main self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._notifier = hs.get_notifier() self._remote_edu_linearizer = Linearizer(name="remote_device_list") self._resync_linearizer = Linearizer(name="remote_device_resync") @@ -1646,74 +1868,3 @@ class DeviceListUpdater(DeviceListWorkerUpdater): self._seen_updates[user_id] = {stream_id} return result, False - - async def process_cross_signing_key_update( - self, - user_id: str, - master_key: Optional[JsonDict], - self_signing_key: Optional[JsonDict], - ) -> List[str]: - """Process the given new master and self-signing key for the given remote user. - - Args: - user_id: The ID of the user these keys are for. - master_key: The dict of the cross-signing master key as returned by the - remote server. - self_signing_key: The dict of the cross-signing self-signing key as returned - by the remote server. - - Return: - The device IDs for the given keys. - """ - device_ids = [] - - current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) - current_keys = current_keys_map.get(user_id) or {} - - if master_key and master_key != current_keys.get("master"): - await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) - _, verify_key = get_verify_key_from_cross_signing_key(master_key) - # verify_key is a VerifyKey from signedjson, which uses - # .version to denote the portion of the key ID after the - # algorithm and colon, which is the device ID - device_ids.append(verify_key.version) - if self_signing_key and self_signing_key != current_keys.get("self_signing"): - await self.store.set_e2e_cross_signing_key( - user_id, "self_signing", self_signing_key - ) - _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) - device_ids.append(verify_key.version) - - return device_ids - - async def handle_room_un_partial_stated(self, room_id: str) -> None: - """Handles sending appropriate device list updates in a room that has - gone from partial to full state. - """ - - pending_updates = ( - await self.store.get_pending_remote_device_list_updates_for_room(room_id) - ) - - for user_id, device_id in pending_updates: - logger.info( - "Got pending device list update in room %s: %s / %s", - room_id, - user_id, - device_id, - ) - position = await self.store.add_device_change_to_streams( - user_id, - [device_id], - room_ids=[room_id], - ) - - if not position: - # This should only happen if there are no updates, which - # shouldn't happen when we've passed in a non-empty set of - # device IDs. - continue - - self.device_handler.notifier.on_new_event( - StreamKeyType.DEVICE_LIST, position, rooms=[room_id] - ) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index e56bdb4072..b43cbd9c15 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -33,9 +33,6 @@ from synapse.logging.opentracing import ( log_kv, set_tag, ) -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util import json_encoder from synapse.util.stringutils import random_string @@ -56,9 +53,9 @@ class DeviceMessageHandler: self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self.is_mine = hs.is_mine + self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() - self.device_handler = hs.get_device_handler() # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by @@ -80,18 +77,6 @@ class DeviceMessageHandler: hs.config.worker.writers.to_device, ) - # The handler to call when we think a user's device list might be out of - # sync. We do all device list resyncing on the master instance, so if - # we're on a worker we hit the device resync replication API. - if hs.config.worker.worker_app is None: - self._multi_user_device_resync = ( - hs.get_device_handler().device_list_updater.multi_user_device_resync - ) - else: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - # a rate limiter for room key requests. The keys are # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( @@ -213,7 +198,10 @@ class DeviceMessageHandler: await self.store.mark_remote_users_device_caches_as_stale((sender_user_id,)) # Immediately attempt a resync in the background - run_in_background(self._multi_user_device_resync, user_ids=[sender_user_id]) + run_in_background( + self.device_handler.device_list_updater.multi_user_device_resync, + user_ids=[sender_user_id], + ) async def send_device_message( self, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 6171aaf29f..b9abad2188 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -32,10 +32,9 @@ from twisted.internet import defer from synapse.api.constants import EduTypes from synapse.api.errors import CodeMessageException, Codes, NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import log_kv, set_tag, tag_args, trace -from synapse.replication.http.devices import ReplicationUploadKeysForUserRestServlet from synapse.types import ( JsonDict, JsonMapping, @@ -76,8 +75,10 @@ class E2eKeysHandler: federation_registry = hs.get_federation_registry() - is_master = hs.config.worker.worker_app is None - if is_master: + # Only the first writer in the list should handle EDUs for signing key + # updates, so that we can use an in-memory linearizer instead of worker locks. + edu_writer = hs.config.worker.writers.device_lists[0] + if hs.get_instance_name() == edu_writer: edu_updater = SigningKeyEduUpdater(hs) # Only register this edu handler on master as it requires writing @@ -92,11 +93,14 @@ class E2eKeysHandler: EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, edu_updater.incoming_signing_key_update, ) - - self.device_key_uploader = self.upload_device_keys_for_user else: - self.device_key_uploader = ( - ReplicationUploadKeysForUserRestServlet.make_client(hs) + federation_registry.register_instances_for_edu( + EduTypes.SIGNING_KEY_UPDATE, + [edu_writer], + ) + federation_registry.register_instances_for_edu( + EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, + [edu_writer], ) # doesn't really work as part of the generic query API, because the @@ -847,7 +851,7 @@ class E2eKeysHandler: # TODO: Validate the JSON to make sure it has the right keys. device_keys = keys.get("device_keys", None) if device_keys: - await self.device_key_uploader( + await self.upload_device_keys_for_user( user_id=user_id, device_id=device_id, keys={"device_keys": device_keys}, @@ -904,9 +908,6 @@ class E2eKeysHandler: device_keys: the `device_keys` of an /keys/upload request. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - time_now = self.clock.time_msec() device_keys = keys["device_keys"] @@ -998,9 +999,6 @@ class E2eKeysHandler: user_id: the user uploading the keys keys: the signing keys """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - # if a master key is uploaded, then check it. Otherwise, load the # stored master key, to check signatures on other keys if "master_key" in keys: @@ -1091,9 +1089,6 @@ class E2eKeysHandler: Raises: SynapseError: if the signatures dict is not valid. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - failures = {} # signatures to be stored. Each item will be a SignatureListItem @@ -1467,9 +1462,6 @@ class E2eKeysHandler: A tuple of the retrieved key content, the key's ID and the matching VerifyKey. If the key cannot be retrieved, all values in the tuple will instead be None. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - try: remote_result = await self.federation.query_user_devices( user.domain, user.to_string() @@ -1770,7 +1762,7 @@ class SigningKeyEduUpdater: self.clock = hs.get_clock() device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler self._remote_edu_linearizer = Linearizer(name="remote_signing_key") diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f7806e67a9..c709ed2c63 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -698,10 +698,19 @@ class FederationHandler: # We may want to reset the partial state info if it's from an # old, failed partial state join. # https://github.com/matrix-org/synapse/issues/13000 + + # FIXME: Ideally, we would store the full stream token here + # not just the minimum stream ID, so that we can compute an + # accurate list of device changes when un-partial-ing the + # room. The only side effect of this is that we may send + # extra unecessary device list outbound pokes through + # federation, which is harmless. + device_lists_stream_id = self.store.get_device_stream_token().stream + await self.store.store_partial_state_room( room_id=room_id, servers=ret.servers_in_room, - device_lists_stream_id=self.store.get_device_stream_token(), + device_lists_stream_id=device_lists_stream_id, joined_via=origin, ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 1e738f484f..5cec2b01e5 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -77,9 +77,6 @@ from synapse.logging.opentracing import ( trace, ) from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.replication.http.federation import ( ReplicationFederationSendEventsRestServlet, ) @@ -180,12 +177,7 @@ class FederationEventHandler: self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self._send_events = ReplicationFederationSendEventsRestServlet.make_client(hs) - if hs.config.worker.worker_app: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - else: - self._device_list_updater = hs.get_device_handler().device_list_updater + self._device_list_updater = hs.get_device_handler().device_list_updater # When joining a room we need to queue any events for that room up. # For each room, a list of (pdu, origin) tuples. @@ -1544,12 +1536,7 @@ class FederationEventHandler: await self._store.mark_remote_users_device_caches_as_stale((sender,)) # Immediately attempt a resync in the background - if self._config.worker.worker_app: - await self._multi_user_device_resync(user_ids=[sender]) - else: - await self._device_list_updater.multi_user_device_resync( - user_ids=[sender] - ) + await self._device_list_updater.multi_user_device_resync(user_ids=[sender]) except Exception: logger.exception("Failed to resync device for %s", sender) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 1e1f0c79c8..6322d980d4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -44,7 +44,6 @@ from synapse.api.errors import ( ) from synapse.appservice import ApplicationService from synapse.config.server import is_threepid_reserved -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import assert_params_in_dict from synapse.replication.http.login import RegisterDeviceReplicationServlet from synapse.replication.http.register import ( @@ -840,9 +839,6 @@ class RegistrationHandler: refresh_token = None refresh_token_id = None - # This can only run on the main process. - assert isinstance(self.device_handler, DeviceHandler) - registered_device_id = await self.device_handler.check_device_registered( user_id, device_id, diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 94301add9e..54116a9b72 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -21,7 +21,6 @@ import logging from typing import TYPE_CHECKING, Optional from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.types import Requester if TYPE_CHECKING: @@ -36,17 +35,7 @@ class SetPasswordHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() - - # We don't need the device handler if password changing is disabled. - # This allows us to instantiate the SetPasswordHandler on the workers - # that have admin APIs for MAS - if self._auth_handler.can_change_password(): - # This can only be instantiated on the main process. - device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) - self._device_handler: Optional[DeviceHandler] = device_handler - else: - self._device_handler = None + self._device_handler = hs.get_device_handler() async def set_password( self, @@ -58,9 +47,6 @@ class SetPasswordHandler: if not self._auth_handler.can_change_password(): raise SynapseError(403, "Password change disabled", errcode=Codes.FORBIDDEN) - # We should have this available only if password changing is enabled. - assert self._device_handler is not None - try: await self.store.user_set_password_hash(user_id, password_hash) except StoreError as e: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 0dd64618e0..48f7ba094e 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -46,7 +46,6 @@ from twisted.web.server import Request from synapse.api.constants import LoginType, ProfileFields from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError from synapse.config.sso import SsoAttributeRequirement -from synapse.handlers.device import DeviceHandler from synapse.handlers.register import init_counters_for_auth_provider from synapse.handlers.ui_auth import UIAuthSessionDataConstants from synapse.http import get_request_user_agent @@ -1181,8 +1180,6 @@ class SsoHandler: ) -> None: """Revoke any devices and in-flight logins tied to a provider session. - Can only be called from the main process. - Args: auth_provider_id: A unique identifier for this SSO provider, e.g. "oidc" or "saml". @@ -1191,11 +1188,6 @@ class SsoHandler: sessions belonging to other users and log an error. """ - # It is expected that this is the main process. - assert isinstance(self._device_handler, DeviceHandler), ( - "revoking SSO sessions can only be called on the main process" - ) - # Invalidate any running user-mapping sessions to_delete = [] for session_id, session in self._username_mapping_sessions.items(): diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index dd58d3aedc..f039cd54c3 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -66,7 +66,6 @@ from synapse.handlers.auth import ( ON_LOGGED_OUT_CALLBACK, AuthHandler, ) -from synapse.handlers.device import DeviceHandler from synapse.handlers.push_rules import RuleSpec, check_actions from synapse.http.client import SimpleHttpClient from synapse.http.server import ( @@ -925,8 +924,6 @@ class ModuleApi: ) -> Generator["defer.Deferred[Any]", Any, None]: """Invalidate an access token for a user - Can only be called from the main process. - Added in Synapse v0.25.0. Args: @@ -939,10 +936,6 @@ class ModuleApi: Raises: synapse.api.errors.AuthError: the access token is invalid """ - assert isinstance(self._device_handler, DeviceHandler), ( - "invalidate_access_token can only be called on the main process" - ) - # see if the access token corresponds to a device user_info = yield defer.ensureDeferred( self._auth.get_user_by_access_token(access_token) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index d500051714..555444fa3d 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -59,10 +59,10 @@ class ReplicationRestResource(JsonResource): account_data.register_servlets(hs, self) push.register_servlets(hs, self) state.register_servlets(hs, self) + devices.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: login.register_servlets(hs, self) register.register_servlets(hs, self) - devices.register_servlets(hs, self) delayed_events.register_servlets(hs, self) diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index 08cf9eff97..974d83bb8b 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -34,6 +34,92 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class ReplicationNotifyDeviceUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user's device list has changed. + + Request format: + + POST /_synapse/replication/notify_device_update/:user_id + + { + "device_ids": ["JLAFKJWSCS", "JLAFKJWSCS"] + } + """ + + NAME = "notify_device_update" + PATH_ARGS = ("user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, device_ids: List[str] + ) -> JsonDict: + return {"device_ids": device_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> Tuple[int, JsonDict]: + device_ids = content["device_ids"] + + span = active_span() + if span: + span.set_tag("user_id", user_id) + span.set_tag("device_ids", f"{device_ids!r}") + + await self.device_handler.notify_device_update(user_id, device_ids) + + return 200, {} + + +class ReplicationNotifyUserSignatureUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user have made new signatures of other users. + + Request format: + + POST /_synapse/replication/notify_user_signature_update/:from_user_id + + { + "user_ids": ["@alice:example.org", "@bob:example.org", ...] + } + """ + + NAME = "notify_user_signature_update" + PATH_ARGS = ("from_user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload(from_user_id: str, user_ids: List[str]) -> JsonDict: # type: ignore[override] + return {"user_ids": user_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, from_user_id: str + ) -> Tuple[int, JsonDict]: + user_ids = content["user_ids"] + + span = active_span() + if span: + span.set_tag("from_user_id", from_user_id) + span.set_tag("user_ids", f"{user_ids!r}") + + await self.device_handler.notify_user_signature_update(from_user_id, user_ids) + + return 200, {} + + class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): """Ask master to resync the device list for multiple users from the same remote server by contacting their server. @@ -73,11 +159,7 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) - from synapse.handlers.device import DeviceHandler - - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_list_updater = handler.device_list_updater + self.device_list_updater = hs.get_device_handler().device_list_updater self.store = hs.get_datastores().main self.clock = hs.get_clock() @@ -103,32 +185,10 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): return 200, multi_user_devices +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): - """Ask master to upload keys for the user and send them out over federation to - update other servers. - - For now, only the master is permitted to handle key upload requests; - any worker can handle key query requests (since they're read-only). - - Calls to e2e_keys_handler.upload_keys_for_user(user_id, device_id, keys) on - the main process to accomplish this. - - Request format for this endpoint (borrowed and expanded from KeyUploadServlet): - - POST /_synapse/replication/upload_keys_for_user - - { - "user_id": "", - "device_id": "", - "keys": { - ....this part can be found in KeyUploadServlet in rest/client/keys.py.... - or as defined in https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload - } - } - - Response is equivalent to ` /_matrix/client/v3/keys/upload` found in KeyUploadServlet - - """ + """Unused endpoint, kept for backwards compatibility during rollout.""" NAME = "upload_keys_for_user" PATH_ARGS = () @@ -165,6 +225,71 @@ class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): return 200, results +class ReplicationHandleNewDeviceUpdateRestServlet(ReplicationEndpoint): + """Wake up a device writer to send local device list changes as federation outbound pokes. + + Request format: + + POST /_synapse/replication/handle_new_device_update + + {} + """ + + NAME = "handle_new_device_update" + PATH_ARGS = () + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload() -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_new_device_update() + return 200, {} + + +class ReplicationDeviceHandleRoomUnPartialStated(ReplicationEndpoint): + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + + Request format: + + POST /_synapse/replication/device_handle_room_un_partial_stated/:room_id + + {} + """ + + NAME = "device_handle_room_un_partial_stated" + PATH_ARGS = ("room_id",) + CACHE = True + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload(room_id: str) -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, room_id: str + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_room_un_partial_stated(room_id) + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationNotifyDeviceUpdateRestServlet(hs).register(http_server) + ReplicationNotifyUserSignatureUpdateRestServlet(hs).register(http_server) ReplicationMultiUserDevicesResyncRestServlet(hs).register(http_server) + ReplicationHandleNewDeviceUpdateRestServlet(hs).register(http_server) ReplicationUploadKeysForUserRestServlet(hs).register(http_server) + ReplicationDeviceHandleRoomUnPartialStated(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index e71588f3de..b99f11f7c6 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -116,7 +116,11 @@ class ReplicationDataHandler: all_room_ids: Set[str] = set() if stream_name == DeviceListsStream.NAME: if any(not row.is_signature and not row.hosts_calculated for row in rows): - prev_token = self.store.get_device_stream_token() + # This only uses the minimum stream position on the device lists + # stream, which means that we may process a device list change + # twice in case of concurrent writes. This is fine, as this only + # triggers cache invalidation, which is harmless if done twice. + prev_token = self.store.get_device_stream_token().stream all_room_ids = await self.store.get_all_device_list_changes( prev_token, token ) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 1fafbb48c3..e434bed3e5 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -72,6 +72,7 @@ from synapse.replication.tcp.streams import ( ToDeviceStream, TypingStream, ) +from synapse.replication.tcp.streams._base import DeviceListsStream if TYPE_CHECKING: from synapse.server import HomeServer @@ -185,6 +186,12 @@ class ReplicationCommandHandler: continue + if isinstance(stream, DeviceListsStream): + if hs.get_instance_name() in hs.config.worker.writers.device_lists: + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index e55cdc0470..32df4b244c 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -51,7 +51,6 @@ from synapse.rest.admin.background_updates import ( from synapse.rest.admin.devices import ( DeleteDevicesRestServlet, DeviceRestServlet, - DevicesGetRestServlet, DevicesRestServlet, ) from synapse.rest.admin.event_reports import ( @@ -375,4 +374,5 @@ def register_servlets_for_msc3861_delegation( UserRestServletV2(hs).register(http_server) UsernameAvailableRestServlet(hs).register(http_server) UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) - DevicesGetRestServlet(hs).register(http_server) + DeviceRestServlet(hs).register(http_server) + DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 09baf8ce21..c488bce58e 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -23,7 +23,6 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.errors import NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import ( RestServlet, assert_params_in_dict, @@ -51,9 +50,7 @@ class DeviceRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine @@ -113,7 +110,7 @@ class DeviceRestServlet(RestServlet): return HTTPStatus.OK, {} -class DevicesGetRestServlet(RestServlet): +class DevicesRestServlet(RestServlet): """ Retrieve the given user's devices @@ -158,19 +155,6 @@ class DevicesGetRestServlet(RestServlet): return HTTPStatus.OK, {"devices": devices, "total": len(devices)} - -class DevicesRestServlet(DevicesGetRestServlet): - """ - Retrieve the given user's devices - """ - - PATTERNS = admin_patterns("/users/(?P[^/]*)/devices$", "v2") - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - assert isinstance(self.device_worker_handler, DeviceHandler) - self.device_handler = self.device_worker_handler - async def on_POST( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: @@ -194,7 +178,7 @@ class DevicesRestServlet(DevicesGetRestServlet): if not isinstance(device_id, str): raise SynapseError(HTTPStatus.BAD_REQUEST, "device_id must be a string") - await self.device_handler.check_device_registered( + await self.device_worker_handler.check_device_registered( user_id=user_id, device_id=device_id ) @@ -211,9 +195,7 @@ class DeleteDevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 0b075cc2f2..5667af20d4 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, List, Optional, Tuple from synapse._pydantic_compat import Extra, StrictStr from synapse.api import errors from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -91,7 +90,6 @@ class DeleteDevicesRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler self.auth_handler = hs.get_auth_handler() @@ -147,7 +145,6 @@ class DeviceRestServlet(RestServlet): self.auth_handler = hs.get_auth_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled - self._is_main_process = hs.config.worker.worker_app is None async def on_GET( self, request: SynapseRequest, device_id: str @@ -179,14 +176,6 @@ class DeviceRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - # Can only be run on main process, as changes to device lists must - # happen on main. - if not self._is_main_process: - error_message = "DELETE on /devices/ must be routed to main process" - logger.error(error_message) - raise SynapseError(500, error_message) - assert isinstance(self.device_handler, DeviceHandler) - requester = await self.auth.get_user_by_req(request) try: @@ -231,14 +220,6 @@ class DeviceRestServlet(RestServlet): async def on_PUT( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - # Can only be run on main process, as changes to device lists must - # happen on main. - if not self._is_main_process: - error_message = "PUT on /devices/ must be routed to main process" - logger.error(error_message) - raise SynapseError(500, error_message) - assert isinstance(self.device_handler, DeviceHandler) - requester = await self.auth.get_user_by_req(request, allow_guest=True) body = parse_and_validate_json_object_from_request(request, self.PutBody) @@ -317,7 +298,6 @@ class DehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -377,7 +357,6 @@ class ClaimDehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler class PostBody(RequestBodyModel): @@ -517,7 +496,6 @@ class DehydratedDeviceV2Servlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.e2e_keys_handler = hs.get_e2e_keys_handler() self.device_handler = handler @@ -595,18 +573,14 @@ class DehydratedDeviceV2Servlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if ( - hs.config.worker.worker_app is None - and not hs.config.experimental.msc3861.enabled - ): + if not hs.config.experimental.msc3861.enabled: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) DeviceRestServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - if hs.config.experimental.msc2697_enabled: - DehydratedDeviceServlet(hs).register(http_server) - ClaimDehydratedDeviceServlet(hs).register(http_server) - if hs.config.experimental.msc3814_enabled: - DehydratedDeviceV2Servlet(hs).register(http_server) - DehydratedDeviceEventsServlet(hs).register(http_server) + if hs.config.experimental.msc2697_enabled: + DehydratedDeviceServlet(hs).register(http_server) + ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc3814_enabled: + DehydratedDeviceV2Servlet(hs).register(http_server) + DehydratedDeviceEventsServlet(hs).register(http_server) diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 7025662fdc..09749b840f 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -504,6 +504,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: OneTimeKeyServlet(hs).register(http_server) if hs.config.experimental.msc3983_appservice_otk_claims: UnstableOneTimeKeyServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - SigningKeyUploadServlet(hs).register(http_server) - SignaturesUploadServlet(hs).register(http_server) + SigningKeyUploadServlet(hs).register(http_server) + SignaturesUploadServlet(hs).register(http_server) diff --git a/synapse/rest/client/logout.py b/synapse/rest/client/logout.py index e6b4a34d51..206865e989 100644 --- a/synapse/rest/client/logout.py +++ b/synapse/rest/client/logout.py @@ -22,7 +22,6 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -42,9 +41,7 @@ class LogoutRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( @@ -71,9 +68,7 @@ class LogoutAllRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( diff --git a/synapse/server.py b/synapse/server.py index fd16abb9ea..5270f7792d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -69,7 +69,7 @@ from synapse.handlers.auth import AuthHandler, PasswordAuthProvider from synapse.handlers.cas import CasHandler from synapse.handlers.deactivate_account import DeactivateAccountHandler from synapse.handlers.delayed_events import DelayedEventsHandler -from synapse.handlers.device import DeviceHandler, DeviceWorkerHandler +from synapse.handlers.device import DeviceHandler, DeviceWriterHandler from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.directory import DirectoryHandler from synapse.handlers.e2e_keys import E2eKeysHandler @@ -586,11 +586,11 @@ class HomeServer(metaclass=abc.ABCMeta): ) @cache_in_self - def get_device_handler(self) -> DeviceWorkerHandler: - if self.config.worker.worker_app: - return DeviceWorkerHandler(self) - else: - return DeviceHandler(self) + def get_device_handler(self) -> DeviceHandler: + if self.get_instance_name() in self.config.worker.writers.device_lists: + return DeviceWriterHandler(self) + + return DeviceHandler(self) @cache_in_self def get_device_message_handler(self) -> DeviceMessageHandler: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index f054c66102..6ed9f85800 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -27,7 +27,6 @@ from typing import ( Dict, Iterable, List, - Literal, Mapping, Optional, Set, @@ -61,12 +60,12 @@ from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import ( JsonDict, JsonMapping, + MultiWriterStreamToken, StrCollection, get_verify_key_from_cross_signing_key, ) from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.caches.lrucache import LruCache from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter @@ -86,6 +85,9 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): + _device_list_id_gen: MultiWriterIdGenerator + _instance_name: str + def __init__( self, database: DatabasePool, @@ -115,7 +117,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ), ], sequence_name="device_lists_sequence", - writers=["master"], + writers=hs.config.worker.writers.device_lists, + ) + + self._is_device_list_writer = ( + self._instance_name in hs.config.worker.writers.device_lists ) device_list_max = self._device_list_id_gen.get_current_token() @@ -244,8 +250,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): for room_id in room_ids: self._device_list_room_stream_cache.entity_has_changed(room_id, token) - def get_device_stream_token(self) -> int: - return self._device_list_id_gen.get_current_token() + def get_device_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator(self._device_list_id_gen) def get_device_stream_id_generator(self) -> MultiWriterIdGenerator: return self._device_list_id_gen @@ -286,6 +292,183 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) + async def store_device( + self, + user_id: str, + device_id: str, + initial_device_display_name: Optional[str], + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> bool: + """Ensure the given device is known; add it to the store if not + + Args: + user_id: id of user associated with the device + device_id: id of device + initial_device_display_name: initial displayname of the device. + Ignored if device exists. + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from a OIDC login. + + Returns: + Whether the device was inserted or an existing device existed with that ID. + + Raises: + StoreError: if the device is already in use + """ + try: + inserted = await self.db_pool.simple_upsert( + "devices", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + values={}, + insertion_values={ + "display_name": initial_device_display_name, + "hidden": False, + }, + desc="store_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + + if not inserted: + # if the device already exists, check if it's a real device, or + # if the device ID is reserved by something else + hidden = await self.db_pool.simple_select_one_onecol( + "devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + retcol="hidden", + ) + if hidden: + raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) + + if auth_provider_id and auth_provider_session_id: + await self.db_pool.simple_insert( + "device_auth_providers", + values={ + "user_id": user_id, + "device_id": device_id, + "auth_provider_id": auth_provider_id, + "auth_provider_session_id": auth_provider_session_id, + }, + desc="store_device_auth_provider", + ) + + return inserted + except StoreError: + raise + except Exception as e: + logger.error( + "store_device with device_id=%s(%r) user_id=%s(%r)" + " display_name=%s(%r) failed: %s", + type(device_id).__name__, + device_id, + type(user_id).__name__, + user_id, + type(initial_device_display_name).__name__, + initial_device_display_name, + e, + ) + raise StoreError(500, "Problem storing device.") + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Deletes several devices. + + Args: + user_id: The ID of the user which owns the devices + device_ids: The IDs of the devices to delete + """ + + def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: + self.db_pool.simple_delete_many_txn( + txn, + table="devices", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id, "hidden": False}, + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="device_auth_providers", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id}, + ) + + # Also delete associated e2e keys. + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + + # We're bulk deleting potentially many devices at once, so + # let's not invalidate the cache for each device individually. + # Instead, we will invalidate the cache for the user as a whole. + self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id,) + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id,) + ) + + for batch in batch_iter(device_ids, 1000): + await self.db_pool.runInteraction( + "delete_devices", _delete_devices_txn, batch + ) + + async def update_device( + self, user_id: str, device_id: str, new_display_name: Optional[str] = None + ) -> None: + """Update a device. Only updates the device if it is not marked as + hidden. + + Args: + user_id: The ID of the user which owns the device + device_id: The ID of the device to update + new_display_name: new displayname for device; None to leave unchanged + Raises: + StoreError: if the device is not found + """ + updates = {} + if new_display_name is not None: + updates["display_name"] = new_display_name + if not updates: + return None + await self.db_pool.simple_update_one( + table="devices", + keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, + updatevalues=updates, + desc="update_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + @cached(tree=True) async def get_device( self, user_id: str, device_id: str @@ -379,7 +562,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): - The list of updates, where each update is a pair of EDU type and EDU contents. """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we only + # record the last stream position we've sent to a destination. This + # means we have to wait for all the writers to catch up before sending + # device list updates, which is fine. + now_stream_id = self.get_device_stream_token().stream if from_stream_id == now_stream_id: return now_stream_id, [] @@ -756,6 +943,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): Returns: The new stream ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.runInteraction( @@ -878,8 +1068,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_all_devices_changed( self, - from_key: int, - to_key: int, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, ) -> Set[str]: """Get all users whose devices have changed in the given range. @@ -894,7 +1084,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): (exclusive) until `to_key` (inclusive). """ - result = self._device_list_stream_cache.get_all_entities_changed(from_key) + result = self._device_list_stream_cache.get_all_entities_changed( + from_key.stream + ) if result.hit: # We know which users might have changed devices. @@ -910,24 +1102,34 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # If the cache didn't tell us anything, we just need to query the full # range. sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? """ rows = await self.db_pool.execute( "get_all_devices_changed", sql, - from_key, - to_key, + from_key.stream, + to_key.get_max_stream_pos(), ) - return {u for (u,) in rows} + return { + user_id + for (user_id, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + } @cancellable async def get_users_whose_devices_changed( self, - from_key: int, + from_key: MultiWriterStreamToken, user_ids: Collection[str], - to_key: Optional[int] = None, + to_key: Optional[MultiWriterStreamToken] = None, ) -> Set[str]: """Get set of users whose devices have changed since `from_key` that are in the given list of user_ids. @@ -947,7 +1149,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # Get set of users who *may* have changed. Users not in the returned # list have definitely not changed. user_ids_to_check = self._device_list_stream_cache.get_entities_changed( - user_ids, from_key + user_ids, from_key.stream ) # If an empty set was returned, there's nothing to do. @@ -955,11 +1157,16 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): return set() if to_key is None: - to_key = self._device_list_id_gen.get_current_token() + to_key = self.get_device_stream_token() - def _get_users_whose_devices_changed_txn(txn: LoggingTransaction) -> Set[str]: + def _get_users_whose_devices_changed_txn( + txn: LoggingTransaction, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, + ) -> Set[str]: sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? AND %s """ @@ -970,17 +1177,32 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): clause, args = make_in_list_sql_clause( txn.database_engine, "user_id", chunk ) - txn.execute(sql % (clause,), [from_key, to_key] + args) - changes.update(user_id for (user_id,) in txn) + txn.execute( + sql % (clause,), + [from_key.stream, to_key.get_max_stream_pos()] + args, + ) + changes.update( + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + ) return changes return await self.db_pool.runInteraction( - "get_users_whose_devices_changed", _get_users_whose_devices_changed_txn + "get_users_whose_devices_changed", + _get_users_whose_devices_changed_txn, + from_key, + to_key, ) async def get_users_whose_signatures_changed( - self, user_id: str, from_key: int + self, user_id: str, from_key: MultiWriterStreamToken ) -> Set[str]: """Get the users who have new cross-signing signatures made by `user_id` since `from_key`. @@ -993,18 +1215,31 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): A set of user IDs with updated signatures. """ - if self._user_signature_stream_cache.has_entity_changed(user_id, from_key): - sql = """ - SELECT DISTINCT user_ids FROM user_signature_stream - WHERE from_user_id = ? AND stream_id > ? - """ - rows = await self.db_pool.execute( - "get_users_whose_signatures_changed", sql, user_id, from_key - ) - return {user for row in rows for user in db_to_json(row[0])} - else: + if not self._user_signature_stream_cache.has_entity_changed( + user_id, from_key.stream + ): return set() + sql = """ + SELECT user_ids, stream_id, instance_name + FROM user_signature_stream + WHERE from_user_id = ? AND stream_id > ? + """ + rows = await self.db_pool.execute( + "get_users_whose_signatures_changed", sql, user_id, from_key.stream + ) + return { + user + for (user_ids, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=None, + instance_name=instance_name, + pos=stream_id, + ) + for user in db_to_json(user_ids) + } + async def get_all_device_list_changes_for_remotes( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> Tuple[List[Tuple[int, tuple]], int, bool]: @@ -1258,9 +1493,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): if keys: device_keys = keys.get("device_keys", None) if device_keys: - # Type ignore - this function is defined on EndToEndKeyStore which we do - # have access to due to hs.get_datastore() "magic" - self._set_e2e_device_keys_txn( # type: ignore[attr-defined] + self._set_e2e_device_keys_txn( txn, user_id, device_id, time, device_keys ) @@ -1490,7 +1723,10 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_device_list_changes_in_rooms( - self, room_ids: Collection[str], from_id: int, to_id: int + self, + room_ids: Collection[str], + from_token: MultiWriterStreamToken, + to_token: MultiWriterStreamToken, ) -> Optional[Set[str]]: """Return the set of users whose devices have changed in the given rooms since the given stream ID. @@ -1503,41 +1739,50 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): min_stream_id = await self._get_min_device_lists_changes_in_room() - if min_stream_id > from_id: + # Return early if there are no rows to process in device_lists_changes_in_room + if min_stream_id > from_token.stream: return None changed_room_ids = self._device_list_room_stream_cache.get_entities_changed( - room_ids, from_id + room_ids, from_token.stream ) if not changed_room_ids: return set() sql = """ - SELECT DISTINCT user_id FROM device_lists_changes_in_room + SELECT user_id, stream_id, instance_name + FROM device_lists_changes_in_room WHERE {clause} AND stream_id > ? AND stream_id <= ? """ def _get_device_list_changes_in_rooms_txn( txn: LoggingTransaction, - clause: str, - args: List[Any], + chunk: list[str], ) -> Set[str]: - txn.execute(sql.format(clause=clause), args) - return {user_id for (user_id,) in txn} - - changes = set() - for chunk in batch_iter(changed_room_ids, 1000): clause, args = make_in_list_sql_clause( self.database_engine, "room_id", chunk ) - args.append(from_id) - args.append(to_id) + args.append(from_token.stream) + args.append(to_token.get_max_stream_pos()) + txn.execute(sql.format(clause=clause), args) + return { + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_token, + high=to_token, + instance_name=instance_name, + pos=stream_id, + ) + } + + changes = set() + for chunk in batch_iter(changed_room_ids, 1000): changes |= await self.db_pool.runInteraction( "get_device_list_changes_in_rooms", _get_device_list_changes_in_rooms_txn, - clause, - args, + chunk, ) return changes @@ -1605,371 +1850,6 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): desc="get_destinations_for_device", ) - -class DeviceBackgroundUpdateStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._instance_name = hs.get_instance_name() - - self.db_pool.updates.register_background_index_update( - "device_lists_stream_idx", - index_name="device_lists_stream_user_id", - table="device_lists_stream", - columns=["user_id", "device_id"], - ) - - # create a unique index on device_lists_remote_cache - self.db_pool.updates.register_background_index_update( - "device_lists_remote_cache_unique_idx", - index_name="device_lists_remote_cache_unique_id", - table="device_lists_remote_cache", - columns=["user_id", "device_id"], - unique=True, - ) - - # And one on device_lists_remote_extremeties - self.db_pool.updates.register_background_index_update( - "device_lists_remote_extremeties_unique_idx", - index_name="device_lists_remote_extremeties_unique_idx", - table="device_lists_remote_extremeties", - columns=["user_id"], - unique=True, - ) - - # once they complete, we can remove the old non-unique indexes. - self.db_pool.updates.register_background_update_handler( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, - self._drop_device_list_streams_non_unique_indexes, - ) - - # clear out duplicate device list outbound pokes - self.db_pool.updates.register_background_update_handler( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - self._remove_duplicate_outbound_pokes, - ) - - self.db_pool.updates.register_background_index_update( - "device_lists_changes_in_room_by_room_index", - index_name="device_lists_changes_in_room_by_room_idx", - table="device_lists_changes_in_room", - columns=["room_id", "stream_id"], - ) - - async def _drop_device_list_streams_non_unique_indexes( - self, progress: JsonDict, batch_size: int - ) -> int: - def f(conn: LoggingDatabaseConnection) -> None: - txn = conn.cursor() - txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") - txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") - txn.close() - - await self.db_pool.runWithConnection(f) - await self.db_pool.updates._end_background_update( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES - ) - return 1 - - async def _remove_duplicate_outbound_pokes( - self, progress: JsonDict, batch_size: int - ) -> int: - # for some reason, we have accumulated duplicate entries in - # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less - # efficient. - # - # For each duplicate, we delete all the existing rows and put one back. - - last_row = progress.get( - "last_row", - {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, - ) - - def _txn(txn: LoggingTransaction) -> int: - clause, args = make_tuple_comparison_clause( - [ - ("stream_id", last_row["stream_id"]), - ("destination", last_row["destination"]), - ("user_id", last_row["user_id"]), - ("device_id", last_row["device_id"]), - ] - ) - sql = f""" - SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts - FROM device_lists_outbound_pokes - WHERE {clause} - GROUP BY stream_id, destination, user_id, device_id - HAVING count(*) > 1 - ORDER BY stream_id, destination, user_id, device_id - LIMIT ? - """ - txn.execute(sql, args + [batch_size]) - rows = txn.fetchall() - - stream_id, destination, user_id, device_id = None, None, None, None - for stream_id, destination, user_id, device_id, _ in rows: - self.db_pool.simple_delete_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - }, - ) - - self.db_pool.simple_insert_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "instance_name": self._instance_name, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - "sent": False, - }, - ) - - if rows: - self.db_pool.updates._background_update_progress_txn( - txn, - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - { - "last_row": { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - } - }, - ) - - return len(rows) - - rows = await self.db_pool.runInteraction( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn - ) - - if not rows: - await self.db_pool.updates._end_background_update( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES - ) - - return rows - - -class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - self.server_name = hs.hostname - - # Map of (user_id, device_id) -> bool. If there is an entry that implies - # the device exists. - self.device_id_exists_cache: LruCache[Tuple[str, str], Literal[True]] = ( - LruCache( - cache_name="device_id_exists", - server_name=self.server_name, - max_size=10000, - ) - ) - - async def store_device( - self, - user_id: str, - device_id: str, - initial_device_display_name: Optional[str], - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> bool: - """Ensure the given device is known; add it to the store if not - - Args: - user_id: id of user associated with the device - device_id: id of device - initial_device_display_name: initial displayname of the device. - Ignored if device exists. - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from a OIDC login. - - Returns: - Whether the device was inserted or an existing device existed with that ID. - - Raises: - StoreError: if the device is already in use - """ - key = (user_id, device_id) - if self.device_id_exists_cache.get(key, None): - return False - - try: - inserted = await self.db_pool.simple_upsert( - "devices", - keyvalues={ - "user_id": user_id, - "device_id": device_id, - }, - values={}, - insertion_values={ - "display_name": initial_device_display_name, - "hidden": False, - }, - desc="store_device", - ) - await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) - - if not inserted: - # if the device already exists, check if it's a real device, or - # if the device ID is reserved by something else - hidden = await self.db_pool.simple_select_one_onecol( - "devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - retcol="hidden", - ) - if hidden: - raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) - - if auth_provider_id and auth_provider_session_id: - await self.db_pool.simple_insert( - "device_auth_providers", - values={ - "user_id": user_id, - "device_id": device_id, - "auth_provider_id": auth_provider_id, - "auth_provider_session_id": auth_provider_session_id, - }, - desc="store_device_auth_provider", - ) - - self.device_id_exists_cache.set(key, True) - return inserted - except StoreError: - raise - except Exception as e: - logger.error( - "store_device with device_id=%s(%r) user_id=%s(%r)" - " display_name=%s(%r) failed: %s", - type(device_id).__name__, - device_id, - type(user_id).__name__, - user_id, - type(initial_device_display_name).__name__, - initial_device_display_name, - e, - ) - raise StoreError(500, "Problem storing device.") - - async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: - """Deletes several devices. - - Args: - user_id: The ID of the user which owns the devices - device_ids: The IDs of the devices to delete - """ - - def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: - self.db_pool.simple_delete_many_txn( - txn, - table="devices", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id, "hidden": False}, - ) - - self.db_pool.simple_delete_many_txn( - txn, - table="device_auth_providers", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id}, - ) - - # Also delete associated e2e keys. - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_device_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_one_time_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="dehydrated_devices", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_fallback_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - - # We're bulk deleting potentially many devices at once, so - # let's not invalidate the cache for each device individually. - # Instead, we will invalidate the cache for the user as a whole. - self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) - self._invalidate_cache_and_stream( - txn, self.count_e2e_one_time_keys, (user_id,) - ) - self._invalidate_cache_and_stream( - txn, self.get_e2e_unused_fallback_key_types, (user_id,) - ) - - for batch in batch_iter(device_ids, 1000): - await self.db_pool.runInteraction( - "delete_devices", _delete_devices_txn, batch - ) - - for device_id in device_ids: - self.device_id_exists_cache.invalidate((user_id, device_id)) - - async def update_device( - self, user_id: str, device_id: str, new_display_name: Optional[str] = None - ) -> None: - """Update a device. Only updates the device if it is not marked as - hidden. - - Args: - user_id: The ID of the user which owns the device - device_id: The ID of the device to update - new_display_name: new displayname for device; None to leave unchanged - Raises: - StoreError: if the device is not found - """ - updates = {} - if new_display_name is not None: - updates["display_name"] = new_display_name - if not updates: - return None - await self.db_pool.simple_update_one( - table="devices", - keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, - updatevalues=updates, - desc="update_device", - ) - await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) - async def update_remote_device_list_cache_entry( self, user_id: str, device_id: str, content: JsonDict, stream_id: str ) -> None: @@ -2008,8 +1888,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): table="device_lists_remote_cache", keyvalues={"user_id": user_id, "device_id": device_id}, ) - - txn.call_after(self.device_id_exists_cache.invalidate, (user_id, device_id)) else: self.db_pool.simple_upsert_txn( txn, @@ -2102,6 +1980,10 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): The maximum stream ID of device list updates that were added to the database, or None if no updates were added. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not device_ids: return None @@ -2110,8 +1992,11 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): def add_device_changes_txn( txn: LoggingTransaction, batch_device_ids: StrCollection, - stream_ids: List[int], - ) -> None: + ) -> int: + stream_ids = self._device_list_id_gen.get_next_mult_txn( + txn, len(device_ids) + ) + self._add_device_change_to_stream_txn( txn, user_id, @@ -2128,18 +2013,17 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): context, ) - for batch_device_ids in batch_iter(device_ids, 1000): - async with self._device_list_id_gen.get_next_mult( - len(device_ids) - ) as stream_ids: - await self.db_pool.runInteraction( - "add_device_change_to_stream", - add_device_changes_txn, - batch_device_ids, - stream_ids, - ) + return stream_ids[-1] - return stream_ids[-1] + last_stream_id: Optional[int] = None + for batch_device_ids in batch_iter(device_ids, 1000): + last_stream_id = await self.db_pool.runInteraction( + "add_device_change_to_stream", + add_device_changes_txn, + batch_device_ids, + ) + + return last_stream_id def _add_device_change_to_stream_txn( self, @@ -2338,7 +2222,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): A list of user ID, device ID, room ID, stream ID and optional opentracing context, in order of ascending (stream ID, room ID). """ - sql = """ SELECT user_id, device_id, room_id, stream_id, opentracing_context FROM device_lists_changes_in_room @@ -2391,6 +2274,10 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Queue the device update to be sent to the given set of hosts, calculated from the room ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not hosts: return @@ -2419,6 +2306,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Add a device list update to the table tracking remote device list updates during partial joins. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.simple_upsert( @@ -2441,6 +2331,14 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): the room. """ + # The device list stream is a multi-writer stream, but when we partially + # join a room, we only record the minimum stream ID. This means that we + # may be returning a device update that was already sent through + # federation here in case of concurrent writes. This is absolutely fine, + # sending a device update multiple times through federation is safe + + # FIXME: record the full multi-writer stream token with individual + # writer positions at the time of the join to avoid this min_device_stream_id = await self.db_pool.simple_select_one_onecol( table="partial_state_rooms", keyvalues={ @@ -2511,3 +2409,176 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): }, desc="set_device_change_last_converted_pos", ) + + +class DeviceBackgroundUpdateStore(SQLBaseStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._instance_name = hs.get_instance_name() + + self.db_pool.updates.register_background_index_update( + "device_lists_stream_idx", + index_name="device_lists_stream_user_id", + table="device_lists_stream", + columns=["user_id", "device_id"], + ) + + # create a unique index on device_lists_remote_cache + self.db_pool.updates.register_background_index_update( + "device_lists_remote_cache_unique_idx", + index_name="device_lists_remote_cache_unique_id", + table="device_lists_remote_cache", + columns=["user_id", "device_id"], + unique=True, + ) + + # And one on device_lists_remote_extremeties + self.db_pool.updates.register_background_index_update( + "device_lists_remote_extremeties_unique_idx", + index_name="device_lists_remote_extremeties_unique_idx", + table="device_lists_remote_extremeties", + columns=["user_id"], + unique=True, + ) + + # once they complete, we can remove the old non-unique indexes. + self.db_pool.updates.register_background_update_handler( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, + self._drop_device_list_streams_non_unique_indexes, + ) + + # clear out duplicate device list outbound pokes + self.db_pool.updates.register_background_update_handler( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + self._remove_duplicate_outbound_pokes, + ) + + self.db_pool.updates.register_background_index_update( + "device_lists_changes_in_room_by_room_index", + index_name="device_lists_changes_in_room_by_room_idx", + table="device_lists_changes_in_room", + columns=["room_id", "stream_id"], + ) + + async def _drop_device_list_streams_non_unique_indexes( + self, progress: JsonDict, batch_size: int + ) -> int: + def f(conn: LoggingDatabaseConnection) -> None: + txn = conn.cursor() + txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") + txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") + txn.close() + + await self.db_pool.runWithConnection(f) + await self.db_pool.updates._end_background_update( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES + ) + return 1 + + async def _remove_duplicate_outbound_pokes( + self, progress: JsonDict, batch_size: int + ) -> int: + # for some reason, we have accumulated duplicate entries in + # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less + # efficient. + # + # For each duplicate, we delete all the existing rows and put one back. + + last_row = progress.get( + "last_row", + {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, + ) + + def _txn(txn: LoggingTransaction) -> int: + clause, args = make_tuple_comparison_clause( + [ + ("stream_id", last_row["stream_id"]), + ("destination", last_row["destination"]), + ("user_id", last_row["user_id"]), + ("device_id", last_row["device_id"]), + ] + ) + sql = f""" + SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts + FROM device_lists_outbound_pokes + WHERE {clause} + GROUP BY stream_id, destination, user_id, device_id + HAVING count(*) > 1 + ORDER BY stream_id, destination, user_id, device_id + LIMIT ? + """ + txn.execute(sql, args + [batch_size]) + rows = txn.fetchall() + + stream_id, destination, user_id, device_id = None, None, None, None + for stream_id, destination, user_id, device_id, _ in rows: + self.db_pool.simple_delete_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + }, + ) + + self.db_pool.simple_insert_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "instance_name": self._instance_name, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + "sent": False, + }, + ) + + if rows: + self.db_pool.updates._background_update_progress_txn( + txn, + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + { + "last_row": { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + } + }, + ) + + return len(rows) + + rows = await self.db_pool.runInteraction( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn + ) + + if not rows: + await self.db_pool.updates._end_background_update( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES + ) + + return rows + + +class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 0700b0087b..a4a8aafa0c 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -59,7 +59,7 @@ from synapse.storage.database import ( from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict, JsonMapping +from synapse.types import JsonDict, JsonMapping, MultiWriterStreamToken from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList from synapse.util.cancellation import cancellable @@ -120,6 +120,20 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker self.hs.config.federation.allow_device_name_lookup_over_federation ) + self._cross_signing_id_gen = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="e2e_cross_signing_keys", + instance_name=self._instance_name, + tables=[ + ("e2e_cross_signing_keys", "instance_name", "stream_id"), + ], + sequence_name="e2e_cross_signing_keys_sequence", + # No one reads the stream positions, so we're allowed to have an empty list of writers + writers=[], + ) + def process_replication_rows( self, stream_name: str, @@ -145,7 +159,12 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker Returns: (stream_id, devices) """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we *need* to + # give out the stream_id as an integer in the federation API. + # This means that we'll potentially return the same data twice with a + # different stream_id, and invalidate cache more often than necessary, + # which is fine overall. + now_stream_id = self.get_device_stream_token().stream # We need to be careful with the caching here, as we need to always # return *all* persisted devices, however there may be a lag between a @@ -164,8 +183,10 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker # have to check for potential invalidations after the # `now_stream_id`. sql = """ - SELECT user_id FROM device_lists_stream + SELECT 1 + FROM device_lists_stream WHERE stream_id >= ? AND user_id = ? + LIMIT 1 """ rows = await self.db_pool.execute( "get_e2e_device_keys_for_federation_query_check", @@ -1117,7 +1138,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) @abc.abstractmethod - def get_device_stream_token(self) -> int: + def get_device_stream_token(self) -> MultiWriterStreamToken: """Get the current stream id from the _device_list_id_gen""" ... @@ -1540,27 +1561,44 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker impl, ) + async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: + def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: + log_kv( + { + "message": "Deleting keys for device", + "device_id": device_id, + "user_id": user_id, + } + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id, device_id) + ) + self.db_pool.simple_delete_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) + ) -class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._cross_signing_id_gen = MultiWriterIdGenerator( - db_conn=db_conn, - db=database, - notifier=hs.get_replication_notifier(), - stream_name="e2e_cross_signing_keys", - instance_name=self._instance_name, - tables=[ - ("e2e_cross_signing_keys", "instance_name", "stream_id"), - ], - sequence_name="e2e_cross_signing_keys_sequence", - writers=["master"], + await self.db_pool.runInteraction( + "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn ) async def set_e2e_device_keys( @@ -1754,3 +1792,13 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): ], desc="add_e2e_signing_key", ) + + +class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 81f50467e7..16af68108d 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -36,7 +36,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from synapse.api.constants import EduTypes from synapse.replication.tcp.streams import ReceiptsStream @@ -167,25 +166,7 @@ class ReceiptsWorkerStore(SQLBaseStore): def get_max_receipt_stream_id(self) -> MultiWriterStreamToken: """Get the current max stream ID for receipts stream""" - min_pos = self._receipts_id_gen.get_current_token() - - positions = {} - if isinstance(self._receipts_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._receipts_id_gen.get_positions().items() - if p > min_pos - } - - return MultiWriterStreamToken( - stream=min_pos, instance_map=immutabledict(positions) - ) + return MultiWriterStreamToken.from_generator(self._receipts_id_gen) def get_receipt_stream_id_for_instance(self, instance_name: str) -> int: return self._receipts_id_gen.get_current_token_for_writer(instance_name) diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 38d1e650ad..320d29e474 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2093,6 +2093,58 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): "replace_refresh_token", _replace_refresh_token_txn ) + async def set_device_for_refresh_token( + self, user_id: str, old_device_id: str, device_id: str + ) -> None: + """Moves refresh tokens from old device to current device + + Args: + user_id: The user of the devices. + old_device_id: The old device. + device_id: The new device ID. + Returns: + None + """ + + await self.db_pool.simple_update( + "refresh_tokens", + keyvalues={"user_id": user_id, "device_id": old_device_id}, + updatevalues={"device_id": device_id}, + desc="set_device_for_refresh_token", + ) + + def _set_device_for_access_token_txn( + self, txn: LoggingTransaction, token: str, device_id: str + ) -> str: + old_device_id = self.db_pool.simple_select_one_onecol_txn( + txn, "access_tokens", {"token": token}, "device_id" + ) + + self.db_pool.simple_update_txn( + txn, "access_tokens", {"token": token}, {"device_id": device_id} + ) + + self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) + + return old_device_id + + async def set_device_for_access_token(self, token: str, device_id: str) -> str: + """Sets the device ID associated with an access token. + + Args: + token: The access token to modify. + device_id: The new device ID. + Returns: + The old device ID associated with the access token. + """ + + return await self.db_pool.runInteraction( + "set_device_for_access_token", + self._set_device_for_access_token_txn, + token, + device_id, + ) + async def add_login_token_to_user( self, user_id: str, @@ -2396,6 +2448,154 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) + async def user_delete_access_tokens( + self, + user_id: str, + except_token_id: Optional[int] = None, + device_id: Optional[str] = None, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + except_token_id: access_tokens ID which should *not* be deleted + device_id: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: + keyvalues = {"user_id": user_id} + if device_id is not None: + keyvalues["device_id"] = device_id + + items = keyvalues.items() + where_clause = " AND ".join(k + " = ?" for k, _ in items) + values: List[Union[str, int]] = [v for _, v in items] + # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat + # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where + # clause and values before we handle that. This seems to be only used in the "set password" handler. + refresh_where_clause = where_clause + refresh_values = values.copy() + if except_token_id: + # TODO: support that for refresh tokens + where_clause += " AND id != ?" + values.append(except_token_id) + + txn.execute( + "SELECT token, id, device_id FROM access_tokens WHERE %s" + % where_clause, + values, + ) + tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(token,) for token, _, _ in tokens_and_devices], + ) + + txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) + + txn.execute( + "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, + refresh_values, + ) + + return tokens_and_devices + + return await self.db_pool.runInteraction("user_delete_access_tokens", f) + + async def user_delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + device_ids: The devices to delete tokens for. + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def user_delete_access_tokens_for_devices_txn( + txn: LoggingTransaction, batch_device_ids: StrCollection + ) -> List[Tuple[str, int, Optional[str]]]: + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + clause, args = make_in_list_sql_clause( + txn.database_engine, "device_id", batch_device_ids + ) + args.append(user_id) + + if self.database_engine.supports_returning: + sql = f""" + DELETE FROM access_tokens + WHERE {clause} AND user_id = ? + RETURNING token, id, device_id + """ + txn.execute(sql, args) + tokens_and_devices = txn.fetchall() + else: + tokens_and_devices = self.db_pool.simple_select_many_txn( + txn, + table="access_tokens", + column="device_id", + iterable=batch_device_ids, + keyvalues={"user_id": user_id}, + retcols=("token", "id", "device_id"), + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="access_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(t[0],) for t in tokens_and_devices], + ) + return tokens_and_devices + + results = [] + for batch_device_ids in batch_iter(device_ids, 1000): + tokens_and_devices = await self.db_pool.runInteraction( + "user_delete_access_tokens_for_devices", + user_delete_access_tokens_for_devices_txn, + batch_device_ids, + ) + results.extend(tokens_and_devices) + + return results + + async def delete_access_token(self, access_token: str) -> None: + def f(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_one_txn( + txn, table="access_tokens", keyvalues={"token": access_token} + ) + + self._invalidate_cache_and_stream( + txn, self.get_user_by_access_token, (access_token,) + ) + + await self.db_pool.runInteraction("delete_access_token", f) + class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): def __init__( @@ -2620,58 +2820,6 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): return next_id - async def set_device_for_refresh_token( - self, user_id: str, old_device_id: str, device_id: str - ) -> None: - """Moves refresh tokens from old device to current device - - Args: - user_id: The user of the devices. - old_device_id: The old device. - device_id: The new device ID. - Returns: - None - """ - - await self.db_pool.simple_update( - "refresh_tokens", - keyvalues={"user_id": user_id, "device_id": old_device_id}, - updatevalues={"device_id": device_id}, - desc="set_device_for_refresh_token", - ) - - def _set_device_for_access_token_txn( - self, txn: LoggingTransaction, token: str, device_id: str - ) -> str: - old_device_id = self.db_pool.simple_select_one_onecol_txn( - txn, "access_tokens", {"token": token}, "device_id" - ) - - self.db_pool.simple_update_txn( - txn, "access_tokens", {"token": token}, {"device_id": device_id} - ) - - self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) - - return old_device_id - - async def set_device_for_access_token(self, token: str, device_id: str) -> str: - """Sets the device ID associated with an access token. - - Args: - token: The access token to modify. - device_id: The new device ID. - Returns: - The old device ID associated with the access token. - """ - - return await self.db_pool.runInteraction( - "set_device_for_access_token", - self._set_device_for_access_token_txn, - token, - device_id, - ) - async def user_set_password_hash( self, user_id: str, password_hash: Optional[str] ) -> None: @@ -2743,162 +2891,6 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): await self.db_pool.runInteraction("user_set_consent_server_notice_sent", f) - async def user_delete_access_tokens( - self, - user_id: str, - except_token_id: Optional[int] = None, - device_id: Optional[str] = None, - ) -> List[Tuple[str, int, Optional[str]]]: - """ - Invalidate access and refresh tokens belonging to a user - - Args: - user_id: ID of user the tokens belong to - except_token_id: access_tokens ID which should *not* be deleted - device_id: ID of device the tokens are associated with. - If None, tokens associated with any device (or no device) will - be deleted - Returns: - A tuple of (token, token id, device id) for each of the deleted tokens - """ - - def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: - keyvalues = {"user_id": user_id} - if device_id is not None: - keyvalues["device_id"] = device_id - - items = keyvalues.items() - where_clause = " AND ".join(k + " = ?" for k, _ in items) - values: List[Union[str, int]] = [v for _, v in items] - # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat - # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where - # clause and values before we handle that. This seems to be only used in the "set password" handler. - refresh_where_clause = where_clause - refresh_values = values.copy() - if except_token_id: - # TODO: support that for refresh tokens - where_clause += " AND id != ?" - values.append(except_token_id) - - txn.execute( - "SELECT token, id, device_id FROM access_tokens WHERE %s" - % where_clause, - values, - ) - tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] - - self._invalidate_cache_and_stream_bulk( - txn, - self.get_user_by_access_token, - [(token,) for token, _, _ in tokens_and_devices], - ) - - txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) - - txn.execute( - "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, - refresh_values, - ) - - return tokens_and_devices - - return await self.db_pool.runInteraction("user_delete_access_tokens", f) - - async def user_delete_access_tokens_for_devices( - self, - user_id: str, - device_ids: StrCollection, - ) -> List[Tuple[str, int, Optional[str]]]: - """ - Invalidate access and refresh tokens belonging to a user - - Args: - user_id: ID of user the tokens belong to - device_ids: The devices to delete tokens for. - Returns: - A tuple of (token, token id, device id) for each of the deleted tokens - """ - - def user_delete_access_tokens_for_devices_txn( - txn: LoggingTransaction, batch_device_ids: StrCollection - ) -> List[Tuple[str, int, Optional[str]]]: - self.db_pool.simple_delete_many_txn( - txn, - table="refresh_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - - clause, args = make_in_list_sql_clause( - txn.database_engine, "device_id", batch_device_ids - ) - args.append(user_id) - - if self.database_engine.supports_returning: - sql = f""" - DELETE FROM access_tokens - WHERE {clause} AND user_id = ? - RETURNING token, id, device_id - """ - txn.execute(sql, args) - tokens_and_devices = txn.fetchall() - else: - tokens_and_devices = self.db_pool.simple_select_many_txn( - txn, - table="access_tokens", - column="device_id", - iterable=batch_device_ids, - keyvalues={"user_id": user_id}, - retcols=("token", "id", "device_id"), - ) - - self.db_pool.simple_delete_many_txn( - txn, - table="access_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - - self._invalidate_cache_and_stream_bulk( - txn, - self.get_user_by_access_token, - [(t[0],) for t in tokens_and_devices], - ) - return tokens_and_devices - - results = [] - for batch_device_ids in batch_iter(device_ids, 1000): - tokens_and_devices = await self.db_pool.runInteraction( - "user_delete_access_tokens_for_devices", - user_delete_access_tokens_for_devices_txn, - batch_device_ids, - ) - results.extend(tokens_and_devices) - - return results - - async def delete_access_token(self, access_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="access_tokens", keyvalues={"token": access_token} - ) - - self._invalidate_cache_and_stream( - txn, self.get_user_by_access_token, (access_token,) - ) - - await self.db_pool.runInteraction("delete_access_token", f) - - async def delete_refresh_token(self, refresh_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="refresh_tokens", keyvalues={"token": refresh_token} - ) - - await self.db_pool.runInteraction("delete_refresh_token", f) - async def add_user_pending_deactivation(self, user_id: str) -> None: """ Adds a user to the table of users who need to be parted from all the rooms they're diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 29a001ff92..5edac56ec3 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -324,7 +324,7 @@ class RelationsWorkerStore(SQLBaseStore): account_data_key=0, push_rules_key=0, to_device_key=0, - device_list_key=0, + device_list_key=MultiWriterStreamToken(stream=0), groups_key=0, un_partial_stated_rooms_key=0, ) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 153bdb3510..604365badf 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1574,6 +1574,11 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): """Get the event ID of the initial join that started the partial join, and the device list stream ID at the point we started the partial join. + + This only returns the minimum device list stream ID at the time of + joining, not the full device list stream token. The only impact of this + is that we may be sending again device list updates that we've already + sent to some destinations, which is harmless. """ return cast( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index b6c6b69b22..66280f2f9a 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -61,7 +61,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from typing_extensions import assert_never from twisted.internet import defer @@ -657,23 +656,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): component. """ - min_pos = self._stream_id_gen.get_current_token() - - positions = {} - if isinstance(self._stream_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._stream_id_gen.get_positions().items() - if p > min_pos - } - - return RoomStreamToken(stream=min_pos, instance_map=immutabledict(positions)) + return RoomStreamToken.from_generator(self._stream_id_gen) def get_events_stream_id_generator(self) -> MultiWriterIdGenerator: return self._stream_id_gen diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 856f646795..4534068e7c 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -203,7 +203,7 @@ class EventSources: account_data_key=0, push_rules_key=0, to_device_key=0, - device_list_key=0, + device_list_key=MultiWriterStreamToken(stream=0), groups_key=0, un_partial_stated_rooms_key=0, ) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 5549f3c9f8..d09fd30e81 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: from synapse.appservice.api import ApplicationService from synapse.storage.databases.main import DataStore, PurgeEventsStore from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore + from synapse.storage.util.id_generators import MultiWriterIdGenerator logger = logging.getLogger(__name__) @@ -570,6 +571,25 @@ class AbstractMultiWriterStreamToken(metaclass=abc.ABCMeta): ), ) + @classmethod + def from_generator(cls, generator: "MultiWriterIdGenerator") -> Self: + """Get the current token out of a MultiWriterIdGenerator""" + + # The `min_pos` is the minimum position that we know all instances + # have finished persisting to, so we only care about instances whose + # positions are ahead of that. (Instance positions can be behind the + # min position as there are times we can work out that the minimum + # position is ahead of the naive minimum across all current + # positions. See MultiWriterIdGenerator for details) + min_pos = generator.get_current_token() + positions = { + instance: position + for instance, position in generator.get_positions().items() + if position > min_pos + } + + return cls(stream=min_pos, instance_map=immutabledict(positions)) + @attr.s(frozen=True, slots=True, order=False) class RoomStreamToken(AbstractMultiWriterStreamToken): @@ -980,7 +1000,9 @@ class StreamToken: account_data_key: int push_rules_key: int to_device_key: int - device_list_key: int + device_list_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) # Note that the groups key is no longer used and may have bogus values. groups_key: int un_partial_stated_rooms_key: int @@ -1021,7 +1043,9 @@ class StreamToken: account_data_key=int(account_data_key), push_rules_key=int(push_rules_key), to_device_key=int(to_device_key), - device_list_key=int(device_list_key), + device_list_key=await MultiWriterStreamToken.parse( + store, device_list_key + ), groups_key=int(groups_key), un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), ) @@ -1040,7 +1064,7 @@ class StreamToken: str(self.account_data_key), str(self.push_rules_key), str(self.to_device_key), - str(self.device_list_key), + await self.device_list_key.to_string(store), # Note that the groups key is no longer used, but it is still # serialized so that there will not be confusion in the future # if additional tokens are added. @@ -1069,6 +1093,12 @@ class StreamToken: StreamKeyType.RECEIPT, self.receipt_key.copy_and_advance(new_value) ) return new_token + elif key == StreamKeyType.DEVICE_LIST: + new_token = self.copy_and_replace( + StreamKeyType.DEVICE_LIST, + self.device_list_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1087,7 +1117,11 @@ class StreamToken: @overload def get_field( - self, key: Literal[StreamKeyType.RECEIPT] + self, + key: Literal[ + StreamKeyType.RECEIPT, + StreamKeyType.DEVICE_LIST, + ], ) -> MultiWriterStreamToken: ... @overload @@ -1095,7 +1129,6 @@ class StreamToken: self, key: Literal[ StreamKeyType.ACCOUNT_DATA, - StreamKeyType.DEVICE_LIST, StreamKeyType.PRESENCE, StreamKeyType.PUSH_RULES, StreamKeyType.TO_DEVICE, @@ -1161,7 +1194,16 @@ class StreamToken: StreamToken.START = StreamToken( - RoomStreamToken(stream=0), 0, 0, MultiWriterStreamToken(stream=0), 0, 0, 0, 0, 0, 0 + room_key=RoomStreamToken(stream=0), + presence_key=0, + typing_key=0, + receipt_key=MultiWriterStreamToken(stream=0), + account_data_key=0, + push_rules_key=0, + to_device_key=0, + device_list_key=MultiWriterStreamToken(stream=0), + groups_key=0, + un_partial_stated_rooms_key=0, ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 64e8c12817..267ea0b06e 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -30,7 +30,7 @@ from synapse.api.constants import EduTypes, RoomEncryptionAlgorithms from synapse.api.presence import UserPresenceState from synapse.federation.sender.per_destination_queue import MAX_PRESENCE_STATES_PER_EDU from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceListUpdater, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import login from synapse.server import HomeServer @@ -500,7 +500,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): hs.get_datastores().main.get_current_hosts_in_room = get_current_hosts_in_room # type: ignore[assignment] device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.device_handler = device_handler # whenever send_transaction is called, record the edu data @@ -554,6 +554,8 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): "devices": [{"device_id": "D1"}], } + assert isinstance(self.device_handler.device_list_updater, DeviceListUpdater) + self.get_success( self.device_handler.device_list_updater.incoming_device_list_update( "host2", diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 99a0c50211..1e989ca528 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -29,7 +29,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceHandler +from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer @@ -53,7 +53,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): application_service_api=self.appservice_api, ) handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.store = hs.get_datastores().main self.device_message_handler = hs.get_device_message_handler() @@ -229,7 +229,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): # queue a bunch of messages in the inbox requester = create_requester(sender, device_id=DEVICE_ID) - for i in range(DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): + for i in range(DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): self.get_success( self.device_message_handler.send_device_message( requester, "message_type", {receiver: {"*": {"val": i}}} @@ -462,7 +462,7 @@ class DehydrationTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: hs = self.setup_test_homeserver("server") handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.message_handler = hs.get_device_message_handler() self.registration = hs.get_registration_handler() diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 323950c2f4..182f9dab5d 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -31,7 +31,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import Codes, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex from synapse.types import JsonDict, UserID @@ -856,7 +856,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys1)) device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) e = self.get_failure( device_handler.check_device_registered( user_id=local_user, diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index b6ba472d7d..9972af3aa3 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -28,7 +28,7 @@ from synapse.api.constants import EduTypes, EventTypes from synapse.api.errors import NotFoundError from synapse.events import EventBase from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.handlers.presence import UserPresenceState from synapse.handlers.push_rules import InvalidRuleException from synapse.module_api import ModuleApi @@ -819,7 +819,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Delete the device. device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.get_success(device_handler.delete_devices(user_id, [device_id])) # Check that the callback was called and the pushers still existed. diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index 531162a6e9..660fa465bd 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -26,7 +26,7 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.rest.client import devices, login from synapse.server import HomeServer from synapse.util import Clock @@ -42,7 +42,7 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.admin_user = self.register_user("admin", "pass", admin=True) From 6127aa0d5022c662e81d7d6d5d823c600774d505 Mon Sep 17 00:00:00 2001 From: Alex Durham Date: Fri, 18 Jul 2025 15:23:28 +0200 Subject: [PATCH 491/691] Don't allow tagnames longer than 255 bytes (#18660) --- changelog.d/18660.bugfix | 1 + synapse/rest/client/tags.py | 15 +++++++- tests/rest/client/test_tags.py | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18660.bugfix diff --git a/changelog.d/18660.bugfix b/changelog.d/18660.bugfix new file mode 100644 index 0000000000..fd65b43a1b --- /dev/null +++ b/changelog.d/18660.bugfix @@ -0,0 +1 @@ +Don't allow creation of tags with names longer than 255 bytes, as per the spec. \ No newline at end of file diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index b6648f3499..fb59efb11f 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -20,9 +20,10 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Tuple -from synapse.api.errors import AuthError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -35,6 +36,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +MAX_TAG_LENGTH = 255 + class TagListServlet(RestServlet): """ @@ -86,6 +89,16 @@ class TagServlet(RestServlet): requester = await self.auth.get_user_by_req(request) if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") + + # check if the tag exceeds the length allowed by the matrix-specification + # as defined in: https://spec.matrix.org/v1.15/client-server-api/#events-14 + if len(tag.encode("utf-8")) > MAX_TAG_LENGTH: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "tag parameter's length is over 255 bytes", + errcode=Codes.INVALID_PARAM, + ) + # Check if the user has any membership in the room and raise error if not. # Although it's not harmful for users to tag random rooms, it's just superfluous # data we don't need to track or allow. diff --git a/tests/rest/client/test_tags.py b/tests/rest/client/test_tags.py index 5d596409e1..aee2f6affb 100644 --- a/tests/rest/client/test_tags.py +++ b/tests/rest/client/test_tags.py @@ -15,6 +15,7 @@ """Tests REST events for /tags paths.""" from http import HTTPStatus +from urllib import parse as urlparse import synapse.rest.admin from synapse.rest.client import login, room, tags @@ -93,3 +94,68 @@ class RoomTaggingTestCase(unittest.HomeserverTestCase): ) # Check that the request failed with the correct error self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.result) + + def test_put_tag_fails_if_tag_is_too_long(self) -> None: + """ + Test that a user cannot add a tag to a room that is longer than the 255 bytes + allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes + tag = "X" * 300 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_fails_if_tag_is_too_long_with_graphemes(self) -> None: + """ + Test that a user cannot add a tag to a room that contains graphemes which are in total + longer than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes (275) + tag = "👩‍🚒" * 25 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_succeeds_with_graphemes(self) -> None: + """ + Test that a user can add a tag to a room that contains graphemes which are in total + less than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string of acceptable length (220 bytes) + tag = "👩‍🚒" * 20 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request succeeded + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) From c58d7ade38ef5070bfacfd2911a43a92b08f1ac7 Mon Sep 17 00:00:00 2001 From: Strac Consulting Engineers Pty Ltd Date: Sat, 19 Jul 2025 00:37:17 +1000 Subject: [PATCH 492/691] Update msc3861_delegated.py spelling correction (#18697) --- synapse/api/auth/msc3861_delegated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 581c9c1e74..a584ef9ab3 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -480,7 +480,7 @@ class MSC3861DelegatedAuth(BaseAuth): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. - logger.info("Admin toked used") + logger.info("Admin token used") # XXX: that user doesn't exist and won't be provisioned. # This is mostly fine for admin calls, but we should also think about doing # requesters without a user_id. From 797fa5728d137e29c034731c11d80305c39dad04 Mon Sep 17 00:00:00 2001 From: Strac Consulting Engineers Pty Ltd Date: Sat, 19 Jul 2025 00:37:34 +1000 Subject: [PATCH 493/691] 18697.misc (#18698) --- changelog.d/18697.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/18697.misc diff --git a/changelog.d/18697.misc b/changelog.d/18697.misc new file mode 100644 index 0000000000..1289e56fd5 --- /dev/null +++ b/changelog.d/18697.misc @@ -0,0 +1 @@ +Correct spelling of 'Admin token used' log line. From 56f5097d1c8de968b845eda9e2abd5207ee76825 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 18 Jul 2025 10:28:10 -0500 Subject: [PATCH 494/691] Prevent dirty `Cargo.lock` changes from install (#18693) Spawning from https://github.com/element-hq/synapse/pull/18689 Example CI failure that will stop people from leaving stray `Cargo.lock` changes behind, ``` Error: Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes. ``` --- .github/workflows/tests.yml | 42 ++++++++++++++++++++++++++++++++++++- .rustfmt.toml | 5 +++++ changelog.d/18693.misc | 1 + 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18693.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6250c8ed2..4e32349432 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -263,6 +263,43 @@ jobs: - run: cargo clippy --all-features -- -D warnings + lint-rust: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + # Install like a normal project from source with all optional dependencies + extras: all + install-project: "true" + poetry-version: "2.1.1" + + - name: Ensure `Cargo.lock` is up to date (no stray changes after install) + # The `::error::` syntax is using GitHub Actions' error annotations, see + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions + run: | + if git diff --quiet Cargo.lock; then + echo "Cargo.lock is up to date" + else + echo "::error::Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes." + git diff --exit-code Cargo.lock + exit 1 + fi + + # This job is split from `lint-rust` because it requires a nightly Rust toolchain + # for some of the unstable options we use in `.rustfmt.toml`. lint-rustfmt: runs-on: ubuntu-latest needs: changes @@ -274,7 +311,8 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: - # We use nightly so that it correctly groups together imports + # We use nightly so that we can use some unstable options that we use in + # `.rustfmt.toml`. toolchain: nightly-2025-04-23 components: rustfmt - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 @@ -309,6 +347,7 @@ jobs: - check-lockfile - lint-clippy - lint-clippy-nightly + - lint-rust - lint-rustfmt - lint-readme runs-on: ubuntu-latest @@ -327,6 +366,7 @@ jobs: lint-pydantic lint-clippy lint-clippy-nightly + lint-rust lint-rustfmt lint-readme diff --git a/.rustfmt.toml b/.rustfmt.toml index bf96e7743d..29e67033cc 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1,6 @@ +# Unstable options are only available on a nightly toolchain and must be opted into +unstable_features = true + +# `group_imports` is an unstable option that requires nightly Rust toolchain. Tracked by +# https://github.com/rust-lang/rustfmt/issues/5083 group_imports = "StdExternalCrate" diff --git a/changelog.d/18693.misc b/changelog.d/18693.misc new file mode 100644 index 0000000000..e2fdaed6b2 --- /dev/null +++ b/changelog.d/18693.misc @@ -0,0 +1 @@ +Prevent dirty `Cargo.lock` changes from install. From 875269eb5359b4e5d8e6b78533757009bc4329ad Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 21 Jul 2025 14:54:28 +0000 Subject: [PATCH 495/691] Add experimental and incomplete support for MSC4306: Thread Subscriptions. (#18674) Implements: [MSC4306](https://github.com/matrix-org/matrix-spec-proposals/blob/rei/msc_thread_subscriptions/proposals/4306-thread-subscriptions.md) (partially) What's missing: - Changes to push rules Signed-off-by: Olivier 'reivilibre --- changelog.d/18674.feature | 1 + docker/configure_workers_and_start.py | 10 + synapse/_scripts/synapse_port_db.py | 1 + synapse/app/generic_worker.py | 4 + synapse/config/experimental.py | 4 + synapse/config/workers.py | 4 + synapse/handlers/deactivate_account.py | 3 + synapse/handlers/thread_subscriptions.py | 126 ++++++ synapse/replication/tcp/handler.py | 14 +- synapse/replication/tcp/streams/__init__.py | 3 + synapse/replication/tcp/streams/_base.py | 44 ++ synapse/rest/client/thread_subscriptions.py | 98 +++++ synapse/server.py | 5 + synapse/storage/databases/main/__init__.py | 5 +- synapse/storage/databases/main/events.py | 4 + .../databases/main/thread_subscriptions.py | 382 ++++++++++++++++++ .../main/delta/92/04_thread_subscriptions.sql | 59 +++ .../04_thread_subscriptions_seq.sql.postgres | 19 + ...thread_subscriptions_comments.sql.postgres | 18 + ...sent_stream_ordering_comments.sql.postgres | 24 ++ synapse/storage/util/id_generators.py | 9 + synapse/types/__init__.py | 3 +- .../tcp/streams/test_thread_subscriptions.py | 157 +++++++ .../rest/client/test_thread_subscriptions.py | 256 ++++++++++++ tests/storage/test_thread_subscriptions.py | 272 +++++++++++++ 25 files changed, 1522 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18674.feature create mode 100644 synapse/handlers/thread_subscriptions.py create mode 100644 synapse/rest/client/thread_subscriptions.py create mode 100644 synapse/storage/databases/main/thread_subscriptions.py create mode 100644 synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql create mode 100644 synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres create mode 100644 synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres create mode 100644 synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres create mode 100644 tests/replication/tcp/streams/test_thread_subscriptions.py create mode 100644 tests/rest/client/test_thread_subscriptions.py create mode 100644 tests/storage/test_thread_subscriptions.py diff --git a/changelog.d/18674.feature b/changelog.d/18674.feature new file mode 100644 index 0000000000..b1a1aa11f1 --- /dev/null +++ b/changelog.d/18674.feature @@ -0,0 +1 @@ +Add experimental and incomplete support for [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-spec-proposals/blob/rei/msc_thread_subscriptions/proposals/4306-thread-subscriptions.md). \ No newline at end of file diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 7909b9d932..6212a94042 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -327,6 +327,15 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "thread_subscriptions": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/unstable/io.element.msc4306/.*", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files @@ -427,6 +436,7 @@ def add_worker_roles_to_shared_config( "to_device", "typing", "push_rules", + "thread_subscriptions", } # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 6c3e380355..9a0b459e65 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -136,6 +136,7 @@ BOOLEAN_COLUMNS = { "has_known_state", "is_encrypted", ], + "thread_subscriptions": ["subscribed", "automatic"], "users": ["shadow_banned", "approved", "locked", "suspended"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index d0924c413b..4f5bea6bd6 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -104,6 +104,9 @@ from synapse.storage.databases.main.stats import StatsStore from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.storage.databases.main.tags import TagsWorkerStore from synapse.storage.databases.main.task_scheduler import TaskSchedulerWorkerStore +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.databases.main.transactions import TransactionWorkerStore from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore from synapse.storage.databases.main.user_directory import UserDirectoryStore @@ -132,6 +135,7 @@ class GenericWorkerStore( KeyStore, RoomWorkerStore, DirectoryWorkerStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, ApplicationServiceTransactionWorkerStore, ApplicationServiceWorkerStore, diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index b14bc97ae7..1b7474034f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -581,3 +581,7 @@ class ExperimentalConfig(Config): # MSC4155: Invite filtering self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) + + # MSC4306: Thread Subscriptions + # (and MSC4308: sliding sync extension for thread subscriptions) + self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index c0c8a13861..825ba78482 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -174,6 +174,10 @@ class WriterLocations: default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) + thread_subscriptions: List[str] = attr.ib( + default=["master"], + converter=_instance_to_list_converter, + ) @attr.s(auto_attribs=True) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 8d4d84bed1..305363892f 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -187,6 +187,9 @@ class DeactivateAccountHandler: # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Remove thread subscriptions for the user + await self.store.purge_thread_subscription_settings_for_user(user_id) + # Delete any server-side backup keys await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) diff --git a/synapse/handlers/thread_subscriptions.py b/synapse/handlers/thread_subscriptions.py new file mode 100644 index 0000000000..79e4d6040d --- /dev/null +++ b/synapse/handlers/thread_subscriptions.py @@ -0,0 +1,126 @@ +import logging +from typing import TYPE_CHECKING, Optional + +from synapse.api.errors import AuthError, NotFoundError +from synapse.storage.databases.main.thread_subscriptions import ThreadSubscription +from synapse.types import UserID + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ThreadSubscriptionsHandler: + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.event_handler = hs.get_event_handler() + self.auth = hs.get_auth() + + async def get_thread_subscription_settings( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + ) -> Optional[ThreadSubscription]: + """Get thread subscription settings for a specific thread and user. + Checks that the thread root is both a real event and also that it is visible + to the user. + + Args: + user_id: The ID of the user + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` containing the active subscription settings or None if not set + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + raise NotFoundError("No such thread root") + + return await self.store.get_subscription_for_thread( + user_id.to_string(), event.room_id, thread_root_event_id + ) + + async def subscribe_user_to_thread( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + *, + automatic: bool, + ) -> Optional[int]: + """Sets or updates a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + automatic: whether the user was subscribed by an automatic decision by + their client. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + return await self.store.subscribe_user_to_thread( + user_id.to_string(), + event.room_id, + thread_root_event_id, + automatic=automatic, + ) + + async def unsubscribe_user_from_thread( + self, user_id: UserID, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Clears a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + return await self.store.unsubscribe_user_from_thread( + user_id.to_string(), + event.room_id, + thread_root_event_id, + ) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index e434bed3e5..3611c678c2 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -72,7 +72,10 @@ from synapse.replication.tcp.streams import ( ToDeviceStream, TypingStream, ) -from synapse.replication.tcp.streams._base import DeviceListsStream +from synapse.replication.tcp.streams._base import ( + DeviceListsStream, + ThreadSubscriptionsStream, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -186,6 +189,15 @@ class ReplicationCommandHandler: continue + if isinstance(stream, ThreadSubscriptionsStream): + if ( + hs.get_instance_name() + in hs.config.worker.writers.thread_subscriptions + ): + self._streams_to_replicate.append(stream) + + continue + if isinstance(stream, DeviceListsStream): if hs.get_instance_name() in hs.config.worker.writers.device_lists: self._streams_to_replicate.append(stream) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 677dcb7b40..25c15e5d48 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -41,6 +41,7 @@ from synapse.replication.tcp.streams._base import ( PushRulesStream, ReceiptsStream, Stream, + ThreadSubscriptionsStream, ToDeviceStream, TypingStream, ) @@ -67,6 +68,7 @@ STREAMS_MAP = { ToDeviceStream, FederationStream, AccountDataStream, + ThreadSubscriptionsStream, UnPartialStatedRoomStream, UnPartialStatedEventStream, ) @@ -86,6 +88,7 @@ __all__ = [ "DeviceListsStream", "ToDeviceStream", "AccountDataStream", + "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", "UnPartialStatedEventStream", ] diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index ebf5964d29..3ef86486e6 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -723,3 +723,47 @@ class AccountDataStream(_StreamFromIdGen): heapq.merge(room_rows, global_rows, tag_rows, key=lambda row: row[0]) ) return updates, to_token, limited + + +class ThreadSubscriptionsStream(_StreamFromIdGen): + """A thread subscription was changed.""" + + @attr.s(slots=True, auto_attribs=True) + class ThreadSubscriptionsStreamRow: + """Stream to inform workers about changes to thread subscriptions.""" + + user_id: str + room_id: str + event_id: str # The event ID of the thread root + + NAME = "thread_subscriptions" + ROW_TYPE = ThreadSubscriptionsStreamRow + + def __init__(self, hs: Any): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._thread_subscriptions_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_thread_subscriptions( + from_token, to_token, limit + ) + rows = [ + ( + stream_id, + # These are the args to `ThreadSubscriptionsStreamRow` + (user_id, room_id, event_id), + ) + for stream_id, user_id, room_id, event_id in updates + ] + + logger.error("TS %d->%d %r", from_token, to_token, rows) + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit diff --git a/synapse/rest/client/thread_subscriptions.py b/synapse/rest/client/thread_subscriptions.py new file mode 100644 index 0000000000..5307132ec3 --- /dev/null +++ b/synapse/rest/client/thread_subscriptions.py @@ -0,0 +1,98 @@ +from http import HTTPStatus +from typing import Tuple + +from synapse._pydantic_compat import StrictBool +from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.http.server import HttpServer +from synapse.http.servlet import ( + RestServlet, + parse_and_validate_json_object_from_request, +) +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.server import HomeServer +from synapse.types import JsonDict, RoomID +from synapse.types.rest import RequestBodyModel + + +class ThreadSubscriptionsRestServlet(RestServlet): + PATTERNS = client_patterns( + "/io.element.msc4306/rooms/(?P[^/]*)/thread/(?P[^/]*)/subscription$", + unstable=True, + releases=(), + ) + CATEGORY = "Thread Subscriptions requests (unstable)" + + def __init__(self, hs: "HomeServer"): + self.auth = hs.get_auth() + self.is_mine = hs.is_mine + self.store = hs.get_datastores().main + self.handler = hs.get_thread_subscriptions_handler() + + class PutBody(RequestBodyModel): + automatic: StrictBool + + async def on_GET( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + subscription = await self.handler.get_thread_subscription_settings( + requester.user, + room_id, + thread_root_id, + ) + + if subscription is None: + raise NotFoundError("Not subscribed.") + + return HTTPStatus.OK, {"automatic": subscription.automatic} + + async def on_PUT( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + body = parse_and_validate_json_object_from_request(request, self.PutBody) + + await self.handler.subscribe_user_to_thread( + requester.user, + room_id, + thread_root_id, + automatic=body.automatic, + ) + + return HTTPStatus.OK, {} + + async def on_DELETE( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + await self.handler.unsubscribe_user_from_thread( + requester.user, + room_id, + thread_root_id, + ) + + return HTTPStatus.OK, {} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4306_enabled: + ThreadSubscriptionsRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index 5270f7792d..231bd14907 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -117,6 +117,7 @@ from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler from synapse.handlers.sync import SyncHandler +from synapse.handlers.thread_subscriptions import ThreadSubscriptionsHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler from synapse.handlers.worker_lock import WorkerLocksHandler @@ -789,6 +790,10 @@ class HomeServer(metaclass=abc.ABCMeta): def get_timestamp_lookup_handler(self) -> TimestampLookupHandler: return TimestampLookupHandler(self) + @cache_in_self + def get_thread_subscriptions_handler(self) -> ThreadSubscriptionsHandler: + return ThreadSubscriptionsHandler(self) + @cache_in_self def get_registration_handler(self) -> RegistrationHandler: return RegistrationHandler(self) diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 86431f6e40..de55c452ae 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -19,7 +19,6 @@ # [This file includes modifications made by New Vector Limited] # # - import logging from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast @@ -35,6 +34,9 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.sliding_sync import SlidingSyncStore from synapse.storage.databases.main.stats import UserSortOrder +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.engines import BaseDatabaseEngine from synapse.storage.types import Cursor from synapse.types import get_domain_from_id @@ -141,6 +143,7 @@ class DataStore( SearchStore, TagsStore, AccountDataStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, StreamWorkerStore, OpenIdStore, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index b7fbfdc0ca..741146417f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2986,6 +2986,10 @@ class PersistEventsStore: # Upsert into the threads table, but only overwrite the value if the # new event is of a later topological order OR if the topological # ordering is equal, but the stream ordering is later. + # (Note by definition that the stream ordering will always be later + # unless this is a backfilled event [= negative stream ordering] + # because we are only persisting this event now and stream_orderings + # are strictly monotonically increasing) sql = """ INSERT INTO threads (room_id, thread_id, latest_event_id, topological_ordering, stream_ordering) VALUES (?, ?, ?, ?, ?) diff --git a/synapse/storage/databases/main/thread_subscriptions.py b/synapse/storage/databases/main/thread_subscriptions.py new file mode 100644 index 0000000000..e04e692e6a --- /dev/null +++ b/synapse/storage/databases/main/thread_subscriptions.py @@ -0,0 +1,382 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +import logging +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) + +import attr + +from synapse.replication.tcp.streams._base import ThreadSubscriptionsStream +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore +from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.util.caches.descriptors import cached + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ThreadSubscription: + automatic: bool + """ + whether the subscription was made automatically (as opposed to by manual + action from the user) + """ + + +class ThreadSubscriptionsWorkerStore(CacheInvalidationWorkerStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._can_write_to_thread_subscriptions = ( + self._instance_name in hs.config.worker.writers.thread_subscriptions + ) + + self._thread_subscriptions_id_gen: MultiWriterIdGenerator = ( + MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="thread_subscriptions", + instance_name=self._instance_name, + tables=[ + ("thread_subscriptions", "instance_name", "stream_id"), + ], + sequence_name="thread_subscriptions_sequence", + writers=hs.config.worker.writers.thread_subscriptions, + ) + ) + + def process_replication_rows( + self, + stream_name: str, + instance_name: str, + token: int, + rows: Iterable[Any], + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + for row in rows: + self.get_subscription_for_thread.invalidate( + (row.user_id, row.room_id, row.event_id) + ) + + super().process_replication_rows(stream_name, instance_name, token, rows) + + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + self._thread_subscriptions_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + + async def subscribe_user_to_thread( + self, user_id: str, room_id: str, thread_root_event_id: str, *, automatic: bool + ) -> Optional[int]: + """Updates a user's subscription settings for a specific thread root. + + If no change would be made to the subscription, does not produce any database change. + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + automatic: Whether the subscription was performed automatically by the user's client. + Only `False` will overwrite an existing value of automatic for a subscription row. + + Returns: + The stream ID for this update, if the update isn't no-opped. + """ + assert self._can_write_to_thread_subscriptions + + def _subscribe_user_to_thread_txn(txn: LoggingTransaction) -> Optional[int]: + already_automatic = self.db_pool.simple_select_one_onecol_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + }, + retcol="automatic", + allow_none=True, + ) + + if already_automatic is None: + already_subscribed = False + already_automatic = True + else: + already_subscribed = True + # convert int (SQLite bool) to Python bool + already_automatic = bool(already_automatic) + + if already_subscribed and already_automatic == automatic: + # there is nothing we need to do here + return None + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + + values: Dict[str, Optional[Union[bool, int, str]]] = { + "subscribed": True, + "stream_id": stream_id, + "instance_name": self._instance_name, + "automatic": already_automatic and automatic, + } + + self.db_pool.simple_upsert_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + values=values, + ) + + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + + return stream_id + + return await self.db_pool.runInteraction( + "subscribe_user_to_thread", _subscribe_user_to_thread_txn + ) + + async def unsubscribe_user_from_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Unsubscribes a user from a thread. + + If no change would be made to the subscription, does not produce any database change. + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + """ + + assert self._can_write_to_thread_subscriptions + + def _unsubscribe_user_from_thread_txn(txn: LoggingTransaction) -> Optional[int]: + already_subscribed = self.db_pool.simple_select_one_onecol_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + retcol="subscribed", + allow_none=True, + ) + + if already_subscribed is None or already_subscribed is False: + # there is nothing we need to do here + return None + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + + self.db_pool.simple_update_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + }, + updatevalues={ + "subscribed": False, + "stream_id": stream_id, + "instance_name": self._instance_name, + }, + ) + + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + + return stream_id + + return await self.db_pool.runInteraction( + "unsubscribe_user_from_thread", _unsubscribe_user_from_thread_txn + ) + + async def purge_thread_subscription_settings_for_user(self, user_id: str) -> None: + """ + Purge all subscriptions for the user. + The fact that subscriptions have been purged will not be streamed; + all stream rows for the user will in fact be removed. + This is intended only for dealing with user deactivation. + """ + + def _purge_thread_subscription_settings_for_user_txn( + txn: LoggingTransaction, + ) -> None: + self.db_pool.simple_delete_txn( + txn, + table="thread_subscriptions", + keyvalues={"user_id": user_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_subscription_for_thread, (user_id,) + ) + + await self.db_pool.runInteraction( + desc="purge_thread_subscription_settings_for_user", + func=_purge_thread_subscription_settings_for_user_txn, + ) + + @cached(tree=True) + async def get_subscription_for_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[ThreadSubscription]: + """Get the thread subscription for a specific thread and user. + + Args: + user_id: The ID of the user + room_id: The ID of the room + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` dataclass if there is a subscription, + or `None` if there is no subscription. + + If there is a row in the table but `subscribed` is `False`, + behaves the same as if there was no row at all and returns `None`. + """ + row = await self.db_pool.simple_select_one( + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "event_id": thread_root_event_id, + "subscribed": True, + }, + retcols=("automatic",), + allow_none=True, + desc="get_subscription_for_thread", + ) + + if row is None: + return None + + (automatic_rawbool,) = row + + # convert SQLite integer booleans into real booleans + automatic = bool(automatic_rawbool) + + return ThreadSubscription(automatic=automatic) + + def get_max_thread_subscriptions_stream_id(self) -> int: + """Get the current maximum stream_id for thread subscriptions. + + Returns: + The maximum stream_id + """ + return self._thread_subscriptions_id_gen.get_current_token() + + async def get_updated_thread_subscriptions( + self, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str, str]]: + """Get updates to thread subscriptions between two stream IDs. + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of (stream_id, user_id, room_id, thread_root_id) tuples + """ + + def get_updated_thread_subscriptions_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str, str]]: + sql = """ + SELECT stream_id, user_id, room_id, event_id + FROM thread_subscriptions + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """ + + txn.execute(sql, (from_id, to_id, limit)) + return cast(List[Tuple[int, str, str, str]], txn.fetchall()) + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions", + get_updated_thread_subscriptions_txn, + ) + + async def get_updated_thread_subscriptions_for_user( + self, user_id: str, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str]]: + """Get updates to thread subscriptions for a specific user. + + Args: + user_id: The ID of the user + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + A list of (stream_id, room_id, thread_root_event_id) tuples. + """ + + def get_updated_thread_subscriptions_for_user_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str]]: + sql = """ + SELECT stream_id, room_id, event_id + FROM thread_subscriptions + WHERE user_id = ? AND ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """ + + txn.execute(sql, (user_id, from_id, to_id, limit)) + return [(row[0], row[1], row[2]) for row in txn] + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions_for_user", + get_updated_thread_subscriptions_for_user_txn, + ) diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql new file mode 100644 index 0000000000..d19dd7a46d --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql @@ -0,0 +1,59 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Introduce a table for tracking users' subscriptions to threads. +CREATE TABLE thread_subscriptions ( + stream_id INTEGER NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + + subscribed BOOLEAN NOT NULL, + automatic BOOLEAN NOT NULL, + + CONSTRAINT thread_subscriptions_fk_users + FOREIGN KEY (user_id) + REFERENCES users(name), + + CONSTRAINT thread_subscriptions_fk_rooms + FOREIGN KEY (room_id) + -- When we delete a room, we should already have deleted all the events in that room + -- and so there shouldn't be any subscriptions left in that room. + -- So the `ON DELETE CASCADE` should be optional, but included anyway for good measure. + REFERENCES rooms(room_id) ON DELETE CASCADE, + + CONSTRAINT thread_subscriptions_fk_events + FOREIGN KEY (event_id) + REFERENCES events(event_id) ON DELETE CASCADE, + + -- This order provides a useful index for: + -- 1. foreign key constraint on (room_id) + -- 2. foreign key constraint on (room_id, event_id) + -- 3. finding the user's settings for a specific thread (as well as enforcing uniqueness) + UNIQUE (room_id, event_id, user_id) +); + +-- this provides a useful index for finding a user's own rules, +-- potentially scoped to a single room +CREATE INDEX thread_subscriptions_user_room ON thread_subscriptions (user_id, room_id); + +-- this provides a useful way for clients to efficiently find new changes to +-- their subscriptions. +-- (This is necessary to sync subscriptions between multiple devices.) +CREATE INDEX thread_subscriptions_by_user ON thread_subscriptions (user_id, stream_id); + +-- this provides a useful index for deleting the subscriptions when the underlying +-- events are removed. This also covers the foreign key constraint on `events`. +CREATE INDEX thread_subscriptions_by_event ON thread_subscriptions (event_id); diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres new file mode 100644 index 0000000000..8d53691747 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres @@ -0,0 +1,19 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE thread_subscriptions_sequence + -- Synapse streams start at 2, because the default position is 1 + -- so any item inserted at position 1 is ignored. + -- This is also what existing streams do, except they use `setval(..., 1)` + -- which is semantically the same except less obvious. + START WITH 2; diff --git a/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres new file mode 100644 index 0000000000..b0729894c0 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON TABLE thread_subscriptions IS 'Tracks local users that subscribe to threads'; + +COMMENT ON COLUMN thread_subscriptions.subscribed IS 'Whether the user is subscribed to the thread or not. We track unsubscribed threads because we need to stream the subscription change to the client.'; + +COMMENT ON COLUMN thread_subscriptions.automatic IS 'True if the user was subscribed to the thread automatically by their client, or false if the client manually requested the subscription.'; diff --git a/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres new file mode 100644 index 0000000000..3fc7e4b11e --- /dev/null +++ b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres @@ -0,0 +1,24 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON COLUMN threads.latest_event_id IS + 'the ID of the event that is latest, ordered by (topological_ordering, stream_ordering)'; + +COMMENT ON COLUMN threads.topological_ordering IS + $$the topological ordering of the thread''s LATEST event. +Used as the primary way of ordering threads by recency in a room.$$; + +COMMENT ON COLUMN threads.stream_ordering IS + $$the stream ordering of the thread's LATEST event. +Used as a tie-breaker for ordering threads by recency in a room, when the topological order is a tie. +Also used for recency ordering in sliding sync.$$; diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index fe6b6579e6..026a0517d2 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -184,6 +184,12 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): Note: Only works with Postgres. + Warning: Streams using this generator start at ID 2, because ID 1 is always assumed + to have been 'seen as persisted'. + Unclear if this extant behaviour is desirable for some reason. + When creating a new sequence for a new stream, + it will be necessary to use `START WITH 2`. + Args: db_conn db @@ -269,6 +275,9 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._known_persisted_positions: List[int] = [] # The maximum stream ID that we have seen been allocated across any writer. + # Since this defaults to 1, this means that ID 1 is assumed to have already + # been 'seen'. In other words, multi-writer streams start at 2. + # Unclear if this is desirable behaviour. self._max_seen_allocated_stream_id = 1 # The maximum position of the local instance. This can be higher than diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index d09fd30e81..3b516fce3d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -362,7 +362,8 @@ class RoomID(DomainSpecificString): @attr.s(slots=True, frozen=True, repr=False) class EventID(DomainSpecificString): - """Structure representing an event id.""" + """Structure representing an event ID which is namespaced to a homeserver. + Room versions 3 and above are not supported by this grammar.""" SIGIL = "$" diff --git a/tests/replication/tcp/streams/test_thread_subscriptions.py b/tests/replication/tcp/streams/test_thread_subscriptions.py new file mode 100644 index 0000000000..30c3415ad4 --- /dev/null +++ b/tests/replication/tcp/streams/test_thread_subscriptions.py @@ -0,0 +1,157 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.replication.tcp.streams._base import ( + _STREAM_UPDATE_TARGET_ROW_COUNT, + ThreadSubscriptionsStream, +) +from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction +from synapse.util import Clock + +from tests.replication._base import BaseStreamTestCase + + +class ThreadSubscriptionsStreamTestCase(BaseStreamTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Postgres + def f(txn: LoggingTransaction) -> None: + txn.execute( + """ + ALTER TABLE thread_subscriptions + DROP CONSTRAINT thread_subscriptions_fk_users, + DROP CONSTRAINT thread_subscriptions_fk_rooms, + DROP CONSTRAINT thread_subscriptions_fk_events; + """, + ) + + self.get_success( + self.hs.get_datastores().main.db_pool.runInteraction( + "disable_foreign_keys", f + ) + ) + + def test_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates""" + store = self.hs.get_datastores().main + + # Create thread subscription updates + updates = [] + room_id = "!test_room:example.com" + + # Generate several thread subscription updates + for i in range(_STREAM_UPDATE_TARGET_ROW_COUNT + 5): + thread_root_id = f"$thread_{i}:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + room_id, + thread_root_id, + automatic=True, + ) + ) + updates.append(thread_root_id) + + # Also add one in a different room + other_room_id = "!other_room:example.com" + other_thread_root_id = "$other_thread:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + other_room_id, + other_thread_root_id, + automatic=False, + ) + ) + + # Not yet connected: no rows should yet have been received + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Now reconnect to pull the updates + self.reconnect() + self.replicate() + + # We should have received all the expected rows in the right order + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Verify all the thread subscription updates + for thread_id in updates: + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_id) + + # Verify the last update in the different room + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, other_room_id) + self.assertEqual(row.event_id, other_thread_root_id) + + self.assertEqual([], received_rows) + + def test_multiple_users_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates for multiple users""" + store = self.hs.get_datastores().main + room_id = "!test_room:example.com" + thread_root_id = "$thread_root:example.com" + + # Create updates for multiple users + users = ["@user1:example.com", "@user2:example.com", "@user3:example.com"] + for user_id in users: + self.get_success( + store.subscribe_user_to_thread( + user_id, room_id, thread_root_id, automatic=True + ) + ) + + # Check no rows have been received yet + self.replicate() + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Not yet connected: no rows should yet have been received + self.reconnect() + self.replicate() + + # We should have received all the expected rows + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Should have one update per user + self.assertEqual(len(received_rows), len(users)) + + # Verify all updates + for i, user_id in enumerate(users): + (stream_name, token, row) = received_rows[i] + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, user_id) + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_root_id) diff --git a/tests/rest/client/test_thread_subscriptions.py b/tests/rest/client/test_thread_subscriptions.py new file mode 100644 index 0000000000..a5c38753cb --- /dev/null +++ b/tests/rest/client/test_thread_subscriptions.py @@ -0,0 +1,256 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from http import HTTPStatus + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, profile, room, thread_subscriptions +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest + +PREFIX = "/_matrix/client/unstable/io.element.msc4306/rooms" + + +class ThreadSubscriptionsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + thread_subscriptions.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc4306_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("user", "password") + self.token = self.login("user", "password") + self.other_user_id = self.register_user("other_user", "password") + self.other_token = self.login("other_user", "password") + + # Create a room and send a message to use as a thread root + self.room_id = self.helper.create_room_as(self.user_id, tok=self.token) + self.helper.join(self.room_id, self.other_user_id, tok=self.other_token) + response = self.helper.send(self.room_id, body="Root message", tok=self.token) + self.root_event_id = response["event_id"] + + # Send a message in the thread + self.helper.send_event( + room_id=self.room_id, + type="m.room.message", + content={ + "body": "Thread message", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": "m.thread", + "event_id": self.root_event_id, + }, + }, + tok=self.token, + ) + + def test_get_thread_subscription_unsubscribed(self) -> None: + """Test retrieving thread subscription when not subscribed.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_nonexistent_thread(self) -> None: + """Test retrieving subscription settings for a nonexistent thread.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_no_access(self) -> None: + """Test that a user can't get thread subscription for a thread they can't access.""" + self.register_user("no_access", "password") + no_access_token = self.login("no_access", "password") + + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_subscribe_manual_then_automatic(self) -> None: + """Test subscribing to a thread, first a manual subscription then an automatic subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": False, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + # Now also register an automatic subscription; it should not + # override the manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": True}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_subscribe_automatic_then_manual(self) -> None: + """Test subscribing to a thread, first an automatic subscription then a manual subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": True, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + # Now also register a manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": False}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_unsubscribe(self) -> None: + """Test subscribing to a thread, then unsubscribing.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": True, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + # Now also register a manual subscription + channel = self.make_request( + "DELETE", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_nonexistent_thread(self) -> None: + """Test setting subscription settings for a nonexistent thread.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + {"automatic": True}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_no_access(self) -> None: + """Test that a user can't set thread subscription for a thread they can't access.""" + self.register_user("no_access2", "password") + no_access_token = self.login("no_access2", "password") + + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": True}, + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_invalid_body(self) -> None: + """Test that sending invalid subscription settings is rejected.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + # non-boolean `automatic` + {"automatic": "true"}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) diff --git a/tests/storage/test_thread_subscriptions.py b/tests/storage/test_thread_subscriptions.py new file mode 100644 index 0000000000..dd0b804f1f --- /dev/null +++ b/tests/storage/test_thread_subscriptions.py @@ -0,0 +1,272 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import Optional + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction +from synapse.storage.engines.sqlite import Sqlite3Engine +from synapse.util import Clock + +from tests import unittest + + +class ThreadSubscriptionsTestCase(unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.user_id = "@user:test" + self.room_id = "!room:test" + self.thread_root_id = "$thread_root:test" + self.other_thread_root_id = "$other_thread_root:test" + + # Disable foreign key checks for testing + # This allows us to insert test data without having to create actual events + db_pool = self.store.db_pool + if isinstance(db_pool.engine, Sqlite3Engine): + self.get_success( + db_pool.execute("disable_foreign_keys", "PRAGMA foreign_keys = OFF;") + ) + else: + # Postgres + def f(txn: LoggingTransaction) -> None: + txn.execute( + """ + ALTER TABLE thread_subscriptions + DROP CONSTRAINT thread_subscriptions_fk_users, + DROP CONSTRAINT thread_subscriptions_fk_rooms, + DROP CONSTRAINT thread_subscriptions_fk_events; + """, + ) + + self.get_success(db_pool.runInteraction("disable_foreign_keys", f)) + + # Create rooms and events in the db to satisfy foreign key constraints + self.get_success(db_pool.simple_insert("rooms", {"room_id": self.room_id})) + + self.get_success( + db_pool.simple_insert( + "events", + { + "event_id": self.thread_root_id, + "room_id": self.room_id, + "topological_ordering": 1, + "stream_ordering": 1, + "type": "m.room.message", + "depth": 1, + "processed": True, + "outlier": False, + }, + ) + ) + + self.get_success( + db_pool.simple_insert( + "events", + { + "event_id": self.other_thread_root_id, + "room_id": self.room_id, + "topological_ordering": 2, + "stream_ordering": 2, + "type": "m.room.message", + "depth": 2, + "processed": True, + "outlier": False, + }, + ) + ) + + # Create the user + self.get_success( + db_pool.simple_insert("users", {"name": self.user_id, "is_guest": 0}) + ) + + def _subscribe( + self, + thread_root_id: str, + *, + automatic: bool, + room_id: Optional[str] = None, + user_id: Optional[str] = None, + ) -> Optional[int]: + if user_id is None: + user_id = self.user_id + + if room_id is None: + room_id = self.room_id + + return self.get_success( + self.store.subscribe_user_to_thread( + user_id, + room_id, + thread_root_id, + automatic=automatic, + ) + ) + + def _unsubscribe( + self, + thread_root_id: str, + room_id: Optional[str] = None, + user_id: Optional[str] = None, + ) -> Optional[int]: + if user_id is None: + user_id = self.user_id + + if room_id is None: + room_id = self.room_id + + return self.get_success( + self.store.unsubscribe_user_from_thread( + user_id, + room_id, + thread_root_id, + ) + ) + + def test_set_and_get_thread_subscription(self) -> None: + """Test basic setting and getting of thread subscriptions.""" + # Initial state: no subscription + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertIsNone(subscription) + + # Subscribe + self._subscribe( + self.thread_root_id, + automatic=True, + ) + + # Assert subscription went through + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertIsNotNone(subscription) + self.assertTrue(subscription.automatic) # type: ignore + + # Now make it a manual subscription + self._subscribe( + self.thread_root_id, + automatic=False, + ) + + # Assert the manual subscription overrode the automatic one + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertFalse(subscription.automatic) # type: ignore + + def test_purge_thread_subscriptions_for_user(self) -> None: + """Test purging all thread subscription settings for a user.""" + # Set subscription settings for multiple threads + self._subscribe(self.thread_root_id, automatic=True) + self._subscribe(self.other_thread_root_id, automatic=False) + + subscriptions = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, + from_id=0, + to_id=50, + limit=50, + ) + ) + min_id = min(id for (id, _, _) in subscriptions) + self.assertEqual( + subscriptions, + [ + (min_id, self.room_id, self.thread_root_id), + (min_id + 1, self.room_id, self.other_thread_root_id), + ], + ) + + # Purge all settings for the user + self.get_success( + self.store.purge_thread_subscription_settings_for_user(self.user_id) + ) + + # Check user has no subscriptions + subscriptions = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, + from_id=0, + to_id=50, + limit=50, + ) + ) + self.assertEqual(subscriptions, []) + + def test_get_updated_thread_subscriptions(self) -> None: + """Test getting updated thread subscriptions since a stream ID.""" + + stream_id1 = self._subscribe(self.thread_root_id, automatic=False) + stream_id2 = self._subscribe(self.other_thread_root_id, automatic=True) + assert stream_id1 is not None + assert stream_id2 is not None + + # Get updates since initial ID (should include both changes) + updates = self.get_success( + self.store.get_updated_thread_subscriptions(0, stream_id2, 10) + ) + self.assertEqual(len(updates), 2) + + # Get updates since first change (should include only the second change) + updates = self.get_success( + self.store.get_updated_thread_subscriptions(stream_id1, stream_id2, 10) + ) + self.assertEqual( + updates, + [(stream_id2, self.user_id, self.room_id, self.other_thread_root_id)], + ) + + def test_get_updated_thread_subscriptions_for_user(self) -> None: + """Test getting updated thread subscriptions for a specific user.""" + other_user_id = "@other_user:test" + + # Set thread subscription for main user + stream_id1 = self._subscribe(self.thread_root_id, automatic=True) + assert stream_id1 is not None + + # Set thread subscription for other user + stream_id2 = self._subscribe( + self.other_thread_root_id, + automatic=True, + user_id=other_user_id, + ) + assert stream_id2 is not None + + # Get updates for main user + updates = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, 0, stream_id2, 10 + ) + ) + self.assertEqual(updates, [(stream_id1, self.room_id, self.thread_root_id)]) + + # Get updates for other user + updates = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + other_user_id, 0, max(stream_id1, stream_id2), 10 + ) + ) + self.assertEqual( + updates, [(stream_id2, self.room_id, self.other_thread_root_id)] + ) From 8a4e2e826de553669b10245871524f40a04adf08 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Mon, 21 Jul 2025 18:17:43 +0200 Subject: [PATCH 496/691] Dedicated MAS API (#18520) This introduces a dedicated API for MAS to consume. Companion PR on the MAS side: element-hq/matrix-authentication-service#4801 This has a few advantages over the previous admin API: - it works on workers (this will be documented once we stabilise MSC3861 as a whole) - it is more efficient because more focused - it propagates trace contexts from MAS - it is only accessible to MAS (through the shared secret) and will let us remove the weird hack that made this token 'admin' with a ghost '@__oidc_admin:' user The next MAS version should support it, but will be opt-in. The version after that should use this new API by default --------- Co-authored-by: Eric Eastwood --- changelog.d/18520.misc | 1 + synapse/_pydantic_compat.py | 3 + synapse/api/auth/msc3861_delegated.py | 24 +- synapse/rest/synapse/client/__init__.py | 2 + synapse/rest/synapse/mas/__init__.py | 71 ++ synapse/rest/synapse/mas/_base.py | 47 + synapse/rest/synapse/mas/devices.py | 238 ++++ synapse/rest/synapse/mas/users.py | 467 ++++++++ tests/rest/synapse/mas/__init__.py | 12 + tests/rest/synapse/mas/_base.py | 43 + tests/rest/synapse/mas/test_devices.py | 693 +++++++++++ tests/rest/synapse/mas/test_users.py | 1399 +++++++++++++++++++++++ 12 files changed, 2997 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18520.misc create mode 100644 synapse/rest/synapse/mas/__init__.py create mode 100644 synapse/rest/synapse/mas/_base.py create mode 100644 synapse/rest/synapse/mas/devices.py create mode 100644 synapse/rest/synapse/mas/users.py create mode 100644 tests/rest/synapse/mas/__init__.py create mode 100644 tests/rest/synapse/mas/_base.py create mode 100644 tests/rest/synapse/mas/test_devices.py create mode 100644 tests/rest/synapse/mas/test_users.py diff --git a/changelog.d/18520.misc b/changelog.d/18520.misc new file mode 100644 index 0000000000..d005d3b7f7 --- /dev/null +++ b/changelog.d/18520.misc @@ -0,0 +1 @@ +Dedicated internal API for Matrix Authentication Service to Synapse communication. diff --git a/synapse/_pydantic_compat.py b/synapse/_pydantic_compat.py index f0eedf5c6d..e9b43aebe3 100644 --- a/synapse/_pydantic_compat.py +++ b/synapse/_pydantic_compat.py @@ -48,6 +48,7 @@ if TYPE_CHECKING or HAS_PYDANTIC_V2: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.v1.error_wrappers import ErrorWrapper @@ -68,6 +69,7 @@ else: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.error_wrappers import ErrorWrapper @@ -92,4 +94,5 @@ __all__ = ( "StrictStr", "ValidationError", "validator", + "root_validator", ) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index a584ef9ab3..567f2e834c 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -369,6 +369,12 @@ class MSC3861DelegatedAuth(BaseAuth): async def is_server_admin(self, requester: Requester) -> bool: return "urn:synapse:admin:*" in requester.scope + def _is_access_token_the_admin_token(self, token: str) -> bool: + admin_token = self._admin_token() + if admin_token is None: + return False + return token == admin_token + async def get_user_by_req( self, request: SynapseRequest, @@ -434,7 +440,7 @@ class MSC3861DelegatedAuth(BaseAuth): requester = await self.get_user_by_access_token(access_token, allow_expired) # Do not record requests from MAS using the virtual `__oidc_admin` user. - if access_token != self._admin_token(): + if not self._is_access_token_the_admin_token(access_token): await self._record_request(request, requester) if not allow_guest and requester.is_guest: @@ -470,13 +476,25 @@ class MSC3861DelegatedAuth(BaseAuth): raise UnrecognizedRequestError(code=404) + def is_request_using_the_admin_token(self, request: SynapseRequest) -> bool: + """ + Check if the request is using the admin token. + + Args: + request: The request to check. + + Returns: + True if the request is using the admin token, False otherwise. + """ + access_token = self.get_access_token_from_request(request) + return self._is_access_token_the_admin_token(access_token) + async def get_user_by_access_token( self, token: str, allow_expired: bool = False, ) -> Requester: - admin_token = self._admin_token() - if admin_token is not None and token == admin_token: + if self._is_access_token_the_admin_token(token): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 7b5bfc0421..043c508379 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -30,6 +30,7 @@ from synapse.rest.synapse.client.pick_username import pick_username_resource from synapse.rest.synapse.client.rendezvous import MSC4108RendezvousSessionResource from synapse.rest.synapse.client.sso_register import SsoRegisterResource from synapse.rest.synapse.client.unsubscribe import UnsubscribeResource +from synapse.rest.synapse.mas import MasResource if TYPE_CHECKING: from synapse.server import HomeServer @@ -60,6 +61,7 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc from synapse.rest.synapse.client.jwks import JwksResource resources["/_synapse/jwks"] = JwksResource(hs) + resources["/_synapse/mas"] = MasResource(hs) # provider-specific SSO bits. Only load these if they are enabled, since they # rely on optional dependencies. diff --git a/synapse/rest/synapse/mas/__init__.py b/synapse/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..8115c563d2 --- /dev/null +++ b/synapse/rest/synapse/mas/__init__.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.resource import Resource + +from synapse.rest.synapse.mas.devices import ( + MasDeleteDeviceResource, + MasSyncDevicesResource, + MasUpdateDeviceDisplayNameResource, + MasUpsertDeviceResource, +) +from synapse.rest.synapse.mas.users import ( + MasAllowCrossSigningResetResource, + MasDeleteUserResource, + MasIsLocalpartAvailableResource, + MasProvisionUserResource, + MasQueryUserResource, + MasReactivateUserResource, + MasSetDisplayNameResource, + MasUnsetDisplayNameResource, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +logger = logging.getLogger(__name__) + + +class MasResource(Resource): + """ + Provides endpoints for MAS to manage user accounts and devices. + + All endpoints are mounted under the path `/_synapse/mas/` and only work + using the MAS admin token. + """ + + def __init__(self, hs: "HomeServer"): + Resource.__init__(self) + self.putChild(b"query_user", MasQueryUserResource(hs)) + self.putChild(b"provision_user", MasProvisionUserResource(hs)) + self.putChild(b"is_localpart_available", MasIsLocalpartAvailableResource(hs)) + self.putChild(b"delete_user", MasDeleteUserResource(hs)) + self.putChild(b"upsert_device", MasUpsertDeviceResource(hs)) + self.putChild(b"delete_device", MasDeleteDeviceResource(hs)) + self.putChild( + b"update_device_display_name", MasUpdateDeviceDisplayNameResource(hs) + ) + self.putChild(b"sync_devices", MasSyncDevicesResource(hs)) + self.putChild(b"reactivate_user", MasReactivateUserResource(hs)) + self.putChild(b"set_displayname", MasSetDisplayNameResource(hs)) + self.putChild(b"unset_displayname", MasUnsetDisplayNameResource(hs)) + self.putChild( + b"allow_cross_signing_reset", MasAllowCrossSigningResetResource(hs) + ) diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..caf392fc3a --- /dev/null +++ b/synapse/rest/synapse/mas/_base.py @@ -0,0 +1,47 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +from typing import TYPE_CHECKING, cast + +from synapse.api.errors import SynapseError +from synapse.http.server import DirectServeJsonResource + +if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +class MasBaseResource(DirectServeJsonResource): + def __init__(self, hs: "HomeServer"): + # Importing this module requires authlib, which is an optional + # dependency but required if msc3861 is enabled + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + DirectServeJsonResource.__init__(self, extract_context=True) + auth = hs.get_auth() + assert isinstance(auth, MSC3861DelegatedAuth) + self.msc3861_auth = auth + self.store = cast("GenericWorkerStore", hs.get_datastores().main) + self.hostname = hs.hostname + + def assert_request_is_from_mas(self, request: "SynapseRequest") -> None: + """Assert that the request is coming from MAS itself, not a regular user. + + Throws a 403 if the request is not coming from MAS. + """ + if not self.msc3861_auth.is_request_using_the_admin_token(request): + raise SynapseError(403, "This endpoint must only be called by MAS") diff --git a/synapse/rest/synapse/mas/devices.py b/synapse/rest/synapse/mas/devices.py new file mode 100644 index 0000000000..6cc1153590 --- /dev/null +++ b/synapse/rest/synapse/mas/devices.py @@ -0,0 +1,238 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional, Tuple + +from synapse._pydantic_compat import StrictStr +from synapse.api.errors import NotFoundError +from synapse.http.servlet import parse_and_validate_json_object_from_request +from synapse.types import JsonDict, UserID +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasUpsertDeviceResource(MasBaseResource): + """ + Endpoint for MAS to create or update user devices. + + Takes a localpart, device ID, and optional display name to create new devices + or update existing ones. + + POST /_synapse/mas/upsert_device + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: Optional[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + inserted = await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=body.device_id, + display_name=body.display_name, + ) + + return HTTPStatus.CREATED if inserted else HTTPStatus.OK, {} + + +class MasDeleteDeviceResource(MasBaseResource): + """ + Endpoint for MAS to delete user devices. + + Takes a localpart and device ID to remove the specified device from the user's account. + + POST /_synapse/mas/delete_device + {"localpart": "alice", "device_id": "DEVICE123"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.delete_devices( + user_id=str(user_id), + device_ids=[body.device_id], + ) + + return HTTPStatus.NO_CONTENT, {} + + +class MasUpdateDeviceDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to update a device's display name. + + Takes a localpart, device ID, and new display name to update the device's name. + + POST /_synapse/mas/update_device_display_name + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's New Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.update_device( + user_id=str(user_id), + device_id=body.device_id, + content={"display_name": body.display_name}, + ) + + return HTTPStatus.OK, {} + + +class MasSyncDevicesResource(MasBaseResource): + """ + Endpoint for MAS to synchronize a user's complete device list. + + Takes a localpart and a set of device IDs to ensure the user's device list + matches the provided set by adding missing devices and removing extra ones. + + POST /_synapse/mas/sync_devices + {"localpart": "alice", "devices": ["DEVICE123", "DEVICE456"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + devices: set[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + current_devices = await self.store.get_devices_by_user(user_id=str(user_id)) + current_devices_list = set(current_devices.keys()) + target_device_list = set(body.devices) + + to_add = target_device_list - current_devices_list + to_delete = current_devices_list - target_device_list + + # Log what we're about to do to make it easier to debug if it stops + # mid-way, as this can be a long operation if there are a lot of devices + # to delete or to add. + if to_add and to_delete: + logger.info( + "Syncing %d devices for user %s will add %d devices and delete %d devices", + len(target_device_list), + user_id, + len(to_add), + len(to_delete), + ) + elif to_add: + logger.info( + "Syncing %d devices for user %s will add %d devices", + len(target_device_list), + user_id, + len(to_add), + ) + elif to_delete: + logger.info( + "Syncing %d devices for user %s will delete %d devices", + len(target_device_list), + user_id, + len(to_delete), + ) + + if to_delete: + await self.device_handler.delete_devices( + user_id=str(user_id), device_ids=to_delete + ) + + for device_id in to_add: + await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=device_id, + ) + + return 200, {} diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py new file mode 100644 index 0000000000..09aa13bebb --- /dev/null +++ b/synapse/rest/synapse/mas/users.py @@ -0,0 +1,467 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Optional, Tuple, TypedDict + +from synapse._pydantic_compat import StrictBool, StrictStr, root_validator +from synapse.api.errors import NotFoundError, SynapseError +from synapse.http.servlet import ( + parse_and_validate_json_object_from_request, + parse_string, +) +from synapse.types import JsonDict, UserID, UserInfo, create_requester +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasQueryUserResource(MasBaseResource): + """ + Endpoint for MAS to query user information by localpart. + + Takes a localpart parameter and returns user profile data including display name, + avatar URL, and account status (suspended/deactivated). + + GET /_synapse/mas/query_user?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + class Response(TypedDict): + user_id: str + display_name: Optional[str] + avatar_url: Optional[str] + is_suspended: bool + is_deactivated: bool + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, Response]: + self.assert_request_is_from_mas(request) + + localpart = parse_string(request, "localpart", required=True) + user_id = UserID(localpart, self.hostname) + + user: Optional[UserInfo] = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + profile = await self.store.get_profileinfo(user_id=user_id) + + return HTTPStatus.OK, self.Response( + user_id=user_id.to_string(), + display_name=profile.display_name, + avatar_url=profile.avatar_url, + is_suspended=user.suspended, + is_deactivated=user.is_deactivated, + ) + + +class MasProvisionUserResource(MasBaseResource): + """ + Endpoint for MAS to create or update user accounts and their profile data. + + Takes a localpart and optional profile fields (display name, avatar URL, email addresses). + Can create new users or update existing ones by setting or unsetting profile fields. + + POST /_synapse/mas/provision_user + {"localpart": "alice", "set_displayname": "Alice", "set_emails": ["alice@example.com"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + self.registration_handler = hs.get_registration_handler() + self.identity_handler = hs.get_identity_handler() + self.auth_handler = hs.get_auth_handler() + self.profile_handler = hs.get_profile_handler() + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + unset_displayname: StrictBool = False + set_displayname: Optional[StrictStr] = None + + unset_avatar_url: StrictBool = False + set_avatar_url: Optional[StrictStr] = None + + unset_emails: StrictBool = False + set_emails: Optional[list[StrictStr]] = None + + @root_validator(pre=True) + def validate_exclusive(cls, values: Any) -> Any: + if "unset_displayname" in values and "set_displayname" in values: + raise ValueError( + "Cannot specify both unset_displayname and set_displayname" + ) + if "unset_avatar_url" in values and "set_avatar_url" in values: + raise ValueError( + "Cannot specify both unset_avatar_url and set_avatar_url" + ) + if "unset_emails" in values and "set_emails" in values: + raise ValueError("Cannot specify both unset_emails and set_emails") + + return values + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + + localpart = body.localpart + user_id = UserID(localpart, self.hostname) + + requester = create_requester(user_id=user_id) + existing_user = await self.store.get_user_by_id(user_id=str(user_id)) + if existing_user is None: + created = True + await self.registration_handler.register_user( + localpart=localpart, + default_display_name=body.set_displayname, + bind_emails=body.set_emails, + by_admin=True, + ) + else: + created = False + if body.unset_displayname: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname="", + by_admin=True, + ) + elif body.set_displayname is not None: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname=body.set_displayname, + by_admin=True, + ) + + new_email_list: Optional[set[str]] = None + if body.unset_emails: + new_email_list = set() + elif body.set_emails is not None: + new_email_list = set(body.set_emails) + + if new_email_list is not None: + medium = "email" + current_threepid_list = await self.store.user_get_threepids( + user_id=user_id.to_string() + ) + current_email_list = { + t.address for t in current_threepid_list if t.medium == medium + } + + to_delete = current_email_list - new_email_list + to_add = new_email_list - current_email_list + + for address in to_delete: + await self.identity_handler.try_unbind_threepid( + mxid=user_id.to_string(), + medium=medium, + address=address, + id_server=None, + ) + + await self.auth_handler.delete_local_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + ) + + current_time = self.clock.time_msec() + for address in to_add: + await self.auth_handler.add_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + validated_at=current_time, + ) + + if body.unset_avatar_url: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url="", + by_admin=True, + ) + elif body.set_avatar_url is not None: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url=body.set_avatar_url, + by_admin=True, + ) + + return HTTPStatus.CREATED if created else HTTPStatus.OK, {} + + +class MasIsLocalpartAvailableResource(MasBaseResource): + """ + Endpoint for MAS to check if a localpart is available for user registration. + + Takes a localpart parameter and validates its format and availability, + checking for conflicts with existing users or application service namespaces. + + GET /_synapse/mas/is_localpart_available?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.registration_handler = hs.get_registration_handler() + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + localpart = parse_string(request, "localpart") + if localpart is None: + raise SynapseError(400, "Missing localpart") + + await self.registration_handler.check_username(localpart) + + return HTTPStatus.OK, {} + + +class MasDeleteUserResource(MasBaseResource): + """ + Endpoint for MAS to delete/deactivate user accounts. + + Takes a localpart and an erase flag to determine whether to deactivate + the account and optionally erase user data for compliance purposes. + + POST /_synapse/mas/delete_user + {"localpart": "alice", "erase": true} + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + erase: StrictBool + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.deactivate_account( + user_id=user_id.to_string(), + erase_data=body.erase, + requester=create_requester(user_id=user_id), + ) + + return HTTPStatus.OK, {} + + +class MasReactivateUserResource(MasBaseResource): + """ + Endpoint for MAS to reactivate previously deactivated user accounts. + + Takes a localpart parameter to restore access to deactivated accounts. + + POST /_synapse/mas/reactivate_user + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.activate_account(user_id=str(user_id)) + + return HTTPStatus.OK, {} + + +class MasSetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to set a user's display name. + + Takes a localpart and display name to update the user's profile. + + POST /_synapse/mas/set_displayname + {"localpart": "alice", "displayname": "Alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + displayname: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname=body.displayname, + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasUnsetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to clear a user's display name. + + Takes a localpart parameter to remove the display name for the specified user. + + POST /_synapse/mas/unset_displayname + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname="", + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasAllowCrossSigningResetResource(MasBaseResource): + """ + Endpoint for MAS to allow cross-signing key reset without user interaction. + + Takes a localpart parameter to temporarily allow cross-signing key replacement + without requiring User-Interactive Authentication (UIA). + + POST /_synapse/mas/allow_cross_signing_reset + {"localpart": "alice"} + """ + + REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + timestamp = ( + await self.store.allow_master_cross_signing_key_replacement_without_uia( + user_id=str(user_id), + duration_ms=self.REPLACEMENT_PERIOD_MS, + ) + ) + + if timestamp is None: + # If there are no cross-signing keys, this is a no-op, but we should log + logger.warning( + "User %s has no master cross-signing key", user_id.to_string() + ) + + return HTTPStatus.OK, {} diff --git a/tests/rest/synapse/mas/__init__.py b/tests/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..db2cfe109f --- /dev/null +++ b/tests/rest/synapse/mas/__init__.py @@ -0,0 +1,12 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/tests/rest/synapse/mas/_base.py b/tests/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..19d33807a6 --- /dev/null +++ b/tests/rest/synapse/mas/_base.py @@ -0,0 +1,43 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.web.resource import Resource + +from synapse.rest.synapse.client import build_synapse_client_resource_tree +from synapse.types import JsonDict + +from tests import unittest + + +class BaseTestCase(unittest.HomeserverTestCase): + SHARED_SECRET = "shared_secret" + + def default_config(self) -> JsonDict: + config = super().default_config() + config["enable_registration"] = False + config["experimental_features"] = { + "msc3861": { + "enabled": True, + "issuer": "https://example.com", + "client_id": "dummy", + "client_auth_method": "client_secret_basic", + "client_secret": "dummy", + "admin_token": self.SHARED_SECRET, + } + } + return config + + def create_resource_dict(self) -> dict[str, Resource]: + base = super().create_resource_dict() + base.update(build_synapse_client_resource_tree(self.hs)) + return base diff --git a/tests/rest/synapse/mas/test_devices.py b/tests/rest/synapse/mas/test_devices.py new file mode 100644 index 0000000000..a7cd58d8ff --- /dev/null +++ b/tests/rest/synapse/mas/test_devices.py @@ -0,0 +1,693 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import UserID +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpsertDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_upsert_device(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + # This created a new device, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["device_id"], "DEVICE1") + self.assertIsNone(device["display_name"]) + + def test_update_existing_device(self) -> None: + store = self.hs.get_datastores().main + device_handler = self.hs.get_device_handler() + + # Create an initial device + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + # This updated an existing device, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "New Name") + + def test_upsert_device_with_display_name(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Alice's Phone", + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists with correct display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Alice's Phone") + + def test_upsert_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # We get a 404 here as the user doesn't exist + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Test Device", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_device(self) -> None: + store = self.hs.get_datastores().main + + # Verify device exists before deletion + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 204) + + # Verify the device no longer exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + self.assertIsNone(device) + + def test_delete_nonexistent_device(self) -> None: + # Deleting a non-existent device should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + }, + ) + + self.assertEqual(channel.code, 204) + + def test_delete_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # Should fail on a non-existent user + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpdateDeviceDisplayNameResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_update_device_display_name(self) -> None: + store = self.hs.get_datastores().main + + # Verify initial display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Old Name") + + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Updated Name", + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the display name was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Updated Name") + + def test_update_nonexistent_device(self) -> None: + # Updating a non-existent device should fail + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_update_device_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_display_name(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSyncDevicesResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create some initial devices + device_handler = homeserver.get_device_handler() + for device_id in ["DEVICE1", "DEVICE2", "DEVICE3"]: + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id=device_id, + display_name=f"Device {device_id}", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_sync_devices_no_changes(self) -> None: + # Sync with the same devices that already exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices still exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3"}) + + def test_sync_devices_add_only(self) -> None: + # Sync with additional devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify new devices were added + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual( + set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"} + ) + + def test_sync_devices_delete_only(self) -> None: + # Sync with fewer devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1"}) + + def test_sync_devices_add_and_delete(self) -> None: + # Sync with a mix of additions and deletions + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE4", "DEVICE5"}) + + def test_sync_devices_empty_list(self) -> None: + # Sync with empty device list (delete all devices) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": [], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(devices, {}) + + def test_sync_devices_for_new_user(self) -> None: + # Test syncing devices for a user that doesn't have any devices yet + bob_user_id = UserID("bob", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=bob_user_id.localpart, + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "bob", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were created + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(bob_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2"}) + + def test_sync_devices_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_missing_devices(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_invalid_devices_type(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": "not_a_list", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_nonexistent_user(self) -> None: + # Test syncing devices for a user that doesn't exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_sync_devices_duplicate_device_ids(self) -> None: + # Test syncing with duplicate device IDs (sets should handle this) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist (duplicates should be handled) + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(sorted(devices.keys()), ["DEVICE1", "DEVICE2"]) diff --git a/tests/rest/synapse/mas/test_users.py b/tests/rest/synapse/mas/test_users.py new file mode 100644 index 0000000000..378f29fd4c --- /dev/null +++ b/tests/rest/synapse/mas/test_users.py @@ -0,0 +1,1399 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from urllib.parse import urlencode + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasQueryUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_query_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + store.set_profile_avatar_url( + user_id=alice, + new_avatar_url="mxc://example.com/avatar", + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": False, + "is_deactivated": False, + }, + ) + + self.get_success( + store.set_user_suspended_status(user_id=str(alice), suspended=True) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": True, + "is_deactivated": False, + }, + ) + + # Deactivate the account, it should clear the display name and avatar + # and mark the user as deactivated + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": None, + "avatar_url": None, + "is_suspended": True, + "is_deactivated": True, + }, + ) + + def test_query_unknown_user(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_query_user_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasProvisionUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_provision_user(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "set_emails": ["alice@example.com"], + "set_avatar_url": "mxc://example.com/avatar", + }, + ) + + # This created the user, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + alice = UserID("alice", "test") + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + self.assertEqual(profile.avatar_url, "mxc://example.com/avatar") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].medium, "email") + self.assertEqual(threepids[0].address, "alice@example.com") + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "unset_displayname": True, + "unset_avatar_url": True, + "unset_emails": True, + }, + ) + + # This updated the user, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile and threepids were deleted + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, None) + self.assertEqual(profile.avatar_url, None) + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(threepids, []) + + def test_provision_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_empty_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "", + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_localpart(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "@alice:test", # That's a MXID + "alice@domain.com", + "alice:test", + "alice space", + "alice#hash", + "a" * 1000, # Very long localpart + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": localpart, + "set_displayname": "Alice", + }, + ) + # Should be a validation error + self.assertEqual( + channel.code, 400, f"Should fail for localpart: {localpart}" + ) + + def test_provision_user_multiple_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice.alt@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 2) + email_addresses = {tp.address for tp in threepids} + self.assertEqual( + email_addresses, {"alice@example.com", "alice.alt@example.com"} + ) + + def test_provision_user_duplicate_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + # Should deduplicate + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].address, "alice@example.com") + + def test_provision_user_conflicting_operations(self) -> None: + # Test setting and unsetting the same field + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "unset_displayname": True, + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_json_types(self) -> None: + # Test with wrong data types + invalid_contents: list[JsonDict] = [ + {"localpart": "alice", "set_displayname": 123}, # Number instead of string + { + "localpart": "alice", + "set_emails": "not-an-array", + }, # String instead of array + { + "localpart": "alice", + "unset_displayname": "not-a-bool", + }, # String instead of bool + {"localpart": 123}, # Number instead of string for localpart + ] + + for content in invalid_contents: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content=content, + ) + self.assertEqual(channel.code, 400, f"Should fail for content: {content}") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasIsLocalpartAvailableResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user + store = homeserver.get_datastores().main + self.get_success(store.register_user("@alice:test")) + + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_is_localpart_available(self) -> None: + # "alice" is not available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + # "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_invalid_localparts(self) -> None: + # Numeric-only localparts are not allowed + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=0", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + # A super-long MXID is not allowed by the spec + super_long = "a" * 1000 + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?localpart={super_long}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + def test_is_localpart_available_appservice_exclusive(self) -> None: + # Insert an appservice which has exclusive namespaces + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@as_user_.*:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + self.hs.get_datastores().main.services_cache = [appservice] + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_main", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_user_alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + # Sanity-check that "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_empty_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_invalid_characters(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "alice@domain.com", # Contains @ + "alice:test", # Contains : + "alice space", # Contains space + "alice\\backslash", # Contains backslash + "alice#hash", # Contains hash + "alice$dollar", # Contains $ + "alice%percent", # Contains % + "alice&", # Contains & + "alice?question", # Contains ? + "alice[bracket", # Contains [ + "alice]bracket", # Contains ] + "alice{brace", # Contains { + "alice}brace", # Contains } + "alice|pipe", # Contains | + 'alice"quote', # Contains " + "alice'apostrophe", # Contains ' + "alicegreater", # Contains > + "alice\ttab", # Contains tab + "alice\nnewline", # Contains newline + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?{urlencode({'localpart': localpart})}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + # Should return 400 for invalid characters + self.assertEqual( + channel.code, + 400, + f"Should reject localpart with invalid chars: {localpart}", + ) + self.assertEqual( + channel.json_body["errcode"], "M_INVALID_USERNAME", localpart + ) + + def test_is_localpart_available_case_sensitivity(self) -> None: + # Register a user with an uppercase localpart + self.get_success(self.hs.get_datastores().main.register_user("@BOB:test")) + + # It should report as not available, the search should be case-insensitive + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteUserResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user with a display name + self.get_success( + homeserver.get_registration_handler().register_user( + localpart="alice", + default_display_name="Alice", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_user_no_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # But not erased + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # And erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"erase": False}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_missing_erase(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_invalid_erase_type(self) -> None: + invalid_erase_values = [ + "true", # String instead of bool + 1, # Number instead of bool + "false", # String instead of bool + 0, # Number instead of bool + {}, # Object instead of bool + [], # Array instead of bool + ] + + for erase_value in invalid_erase_values: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": erase_value}, + ) + self.assertEqual( + channel.code, 400, f"Should fail for erase value: {erase_value}" + ) + + def test_delete_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent", "erase": False}, + ) + + self.assertEqual(channel.code, 404) + + def test_delete_already_deleted_user(self) -> None: + # First deletion + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Second deletion should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + def test_delete_user_erase_already_deleted_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # First delete without erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Verify not erased initially + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + # Now delete with erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + self.assertEqual(channel.code, 200) + + # Should now be erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_empty_json(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_extra_fields(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "erase": False, + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasReactivateUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_reactivate_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was reactivated + self.assertFalse( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + + def test_reactivate_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_reactivate_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent"}, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_reactivate_active_user(self) -> None: + # Create an active user + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should be idempotent + self.assertEqual(channel.code, 200, channel.json_body) + + def test_reactivate_erased_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + # Deactivate with erase + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + # Verify user is erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should succeed even for erased users + self.assertEqual(channel.code, 200, channel.json_body) + # Shouldn't be erased anymore + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_reactivate_user_extra_fields(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=False, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSetDisplayNameResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_set_display_name(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile was updated + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Bob") + + def test_set_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_missing_displayname(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_very_long(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + long_name = "A" * 1000 + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": long_name}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_special_characters(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + special_names = [ + "Alice 👋", # Emoji + "Alice & Bob", # HTML entities + "Alice\nNewline", # Newline + "Alice\tTab", # Tab + 'Alice"Quote', # Quote + "Alice'Apostrophe", # Apostrophe + "Alice