diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dc65625c6f..c88312f050 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -102,14 +102,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index bfafe5642a..a607169069 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -48,7 +48,7 @@ jobs: with: ref: master - name: Login to registry - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 1217171b5a..4969ca6723 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -66,7 +66,7 @@ jobs: install: true - name: Set up docker layer caching - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ad171e240f..6b8cb3c585 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | .mypy_cache diff --git a/CHANGES.md b/CHANGES.md index 66de9c915f..e286dc1547 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,27 @@ +# Synapse 1.139.0 (2025-09-30) + +### `/register` requests from old application service implementations may break when using MAS + +If you are using Matrix Authentication Service (MAS), as of this release any +Application Services that do not set `inhibit_login=true` when calling `POST +/_matrix/client/v3/register` will receive the error +`IO.ELEMENT.MSC4190.M_APPSERVICE_LOGIN_UNSUPPORTED` in response. Please see [the +upgrade +notes](https://element-hq.github.io/synapse/develop/upgrade.html#register-requests-from-old-application-service-implementations-may-break-when-using-mas) +for more information. + +No significant changes since 1.139.0rc3. + + +# Synapse 1.139.0rc3 (2025-09-25) + +## Bugfixes + +- Fix a bug introduced in 1.139.0rc1 where `run_coroutine_in_background(...)` incorrectly handled logcontexts, resulting in partially broken logging. ([\#18964](https://github.com/element-hq/synapse/issues/18964)) + + + + # Synapse 1.139.0rc2 (2025-09-23) ## Internal Changes diff --git a/Cargo.lock b/Cargo.lock index 15f99e32f0..321f1c7933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1062,9 +1062,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -1074,9 +1074,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -1250,9 +1250,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -1260,18 +1260,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", diff --git a/README.rst b/README.rst index 92854f631c..d10b662d1a 100644 --- a/README.rst +++ b/README.rst @@ -265,6 +265,8 @@ This software is dual-licensed by New Vector Ltd (Element). It can be used eithe Unless required by applicable law or agreed to in writing, software distributed under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the specific language governing permissions and limitations under the Licenses. +Please contact `licensing@element.io `_ to purchase an Element commercial license for this software. + .. |support| image:: https://img.shields.io/badge/matrix-community%20support-success :alt: (get community support in #synapse:matrix.org) diff --git a/changelog.d/18721.bugfix b/changelog.d/18721.bugfix new file mode 100644 index 0000000000..0aa0b3962d --- /dev/null +++ b/changelog.d/18721.bugfix @@ -0,0 +1 @@ +Fix room upgrade `room_config` argument and documentation for `user_may_create_room` spam-checker callback. diff --git a/changelog.d/18767.misc b/changelog.d/18767.misc new file mode 100644 index 0000000000..5fa32a9f97 --- /dev/null +++ b/changelog.d/18767.misc @@ -0,0 +1 @@ +Update OEmbed providers to use 'X' instead of 'Twitter' in URL previews, following a rebrand. Contributed by @HammyHavoc. diff --git a/changelog.d/18828.feature b/changelog.d/18828.feature new file mode 100644 index 0000000000..e7f3541de4 --- /dev/null +++ b/changelog.d/18828.feature @@ -0,0 +1 @@ +Cleanly shutdown `SynapseHomeServer` object. diff --git a/changelog.d/18868.misc b/changelog.d/18868.misc new file mode 100644 index 0000000000..a9251f9da0 --- /dev/null +++ b/changelog.d/18868.misc @@ -0,0 +1 @@ +Fix `server_name` in logging context for multiple Synapse instances in one process. diff --git a/changelog.d/18903.misc b/changelog.d/18903.misc new file mode 100644 index 0000000000..bafa7dad5c --- /dev/null +++ b/changelog.d/18903.misc @@ -0,0 +1 @@ +Wrap the Rust HTTP client with `make_deferred_yieldable` so it follows Synapse logcontext rules. diff --git a/changelog.d/18914.doc b/changelog.d/18914.doc new file mode 100644 index 0000000000..9d4f03ade7 --- /dev/null +++ b/changelog.d/18914.doc @@ -0,0 +1 @@ +Explain how Deferred callbacks interact with logcontexts. diff --git a/changelog.d/18932.misc b/changelog.d/18932.misc new file mode 100644 index 0000000000..675412ddb9 --- /dev/null +++ b/changelog.d/18932.misc @@ -0,0 +1 @@ +Disconnect background process work from request trace. diff --git a/changelog.d/18934.feature b/changelog.d/18934.feature new file mode 100644 index 0000000000..e24b7a7e34 --- /dev/null +++ b/changelog.d/18934.feature @@ -0,0 +1 @@ +Update [MSC4284: Policy Servers](https://github.com/matrix-org/matrix-spec-proposals/pull/4284) implementation to support signatures when available. \ No newline at end of file diff --git a/changelog.d/18966.misc b/changelog.d/18966.misc new file mode 100644 index 0000000000..42c8782a42 --- /dev/null +++ b/changelog.d/18966.misc @@ -0,0 +1 @@ +Add debug logs wherever we change current logcontext. diff --git a/changelog.d/18971.misc b/changelog.d/18971.misc new file mode 100644 index 0000000000..2d417d1319 --- /dev/null +++ b/changelog.d/18971.misc @@ -0,0 +1 @@ +Update dockerfile metadata to fix broken link; point to documentation website. \ No newline at end of file diff --git a/changelog.d/18973.misc b/changelog.d/18973.misc new file mode 100644 index 0000000000..e88fd74795 --- /dev/null +++ b/changelog.d/18973.misc @@ -0,0 +1 @@ +Note that the code is additionally licensed under the [Element Commercial license](https://github.com/element-hq/synapse/blob/develop/LICENSE-COMMERCIAL) in SPDX expression field configs. \ No newline at end of file diff --git a/changelog.d/18974.misc b/changelog.d/18974.misc new file mode 100644 index 0000000000..ca300a17a5 --- /dev/null +++ b/changelog.d/18974.misc @@ -0,0 +1 @@ +Fix logcontext handling in `timeout_deferred` tests. diff --git a/changelog.d/18988.misc b/changelog.d/18988.misc new file mode 100644 index 0000000000..14dbe92812 --- /dev/null +++ b/changelog.d/18988.misc @@ -0,0 +1 @@ +Remove internal `ReplicationUploadKeysForUserRestServlet` as a follow-up to the work in https://github.com/element-hq/synapse/pull/18581 that moved device changes off the main process. \ No newline at end of file diff --git a/changelog.d/18989.removal b/changelog.d/18989.removal new file mode 100644 index 0000000000..356b9ffe3a --- /dev/null +++ b/changelog.d/18989.removal @@ -0,0 +1 @@ +Remove deprecated `LoggingContext.set_current_context`/`LoggingContext.current_context` methods which already have equivalent bare methods in `synapse.logging.context`. diff --git a/changelog.d/18990.misc b/changelog.d/18990.misc new file mode 100644 index 0000000000..f7f8ac5ffd --- /dev/null +++ b/changelog.d/18990.misc @@ -0,0 +1 @@ +Switch task scheduler from raw logcontext manipulation to using the dedicated logcontext utils. diff --git a/changelog.d/18992.misc b/changelog.d/18992.misc new file mode 100644 index 0000000000..ba4470bff1 --- /dev/null +++ b/changelog.d/18992.misc @@ -0,0 +1 @@ +Remove `MockClock()` in tests. diff --git a/changelog.d/18998.doc b/changelog.d/18998.doc new file mode 100644 index 0000000000..9ddc2d41c0 --- /dev/null +++ b/changelog.d/18998.doc @@ -0,0 +1 @@ +Fix documentation for `rc_room_creation` and `rc_reports` to clarify that a `per_user` rate limit is not supported. diff --git a/changelog.d/19007.misc b/changelog.d/19007.misc new file mode 100644 index 0000000000..720623e98e --- /dev/null +++ b/changelog.d/19007.misc @@ -0,0 +1 @@ +Switch back to our own custom `LogContextScopeManager` instead of OpenTracing's `ContextVarsScopeManager` which was causing problems when using the experimental `SYNAPSE_ASYNC_IO_REACTOR` option with tracing enabled. diff --git a/debian/changelog b/debian/changelog index 36fffe4d58..c4450e0f52 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,29 @@ +matrix-synapse-py3 (1.139.0) stable; urgency=medium + + * New Synapse release 1.139.0. + + -- Synapse Packaging team Tue, 30 Sep 2025 11:58:55 +0100 + +matrix-synapse-py3 (1.139.0~rc3) stable; urgency=medium + + * New Synapse release 1.139.0rc3. + + -- Synapse Packaging team Thu, 25 Sep 2025 12:13:23 +0100 + +matrix-synapse-py3 (1.138.2) stable; urgency=medium + + * The licensing specifier has been updated to add an optional + `LicenseRef-Element-Commercial` license. The code was already licensed in + this manner - the debian metadata was just not updated to reflect it. + + -- Synapse Packaging team Thu, 25 Sep 2025 12:17:17 +0100 + +matrix-synapse-py3 (1.138.1) stable; urgency=medium + + * New Synapse release 1.138.1. + + -- Synapse Packaging team Wed, 24 Sep 2025 11:32:38 +0100 + matrix-synapse-py3 (1.139.0~rc2) stable; urgency=medium * New Synapse release 1.139.0rc2. @@ -10,24 +36,6 @@ matrix-synapse-py3 (1.139.0~rc1) stable; urgency=medium -- Synapse Packaging team Tue, 23 Sep 2025 13:24:50 +0100 -matrix-synapse-py3 (1.138.2) stable; urgency=medium - - * New Synapse release 1.138.2. - - -- Synapse Packaging team Wed, 24 Sep 2025 12:26:16 +0100 - -matrix-synapse-py3 (1.138.1) stable; urgency=medium - - * New Synapse release 1.138.1. - - -- Synapse Packaging team Wed, 24 Sep 2025 11:32:38 +0100 - -matrix-synapse-py3 (1.138.0) stable; urgency=medium - - * New Synapse release 1.138.0. - - -- Synapse Packaging team Tue, 09 Sep 2025 11:21:25 +0100 - matrix-synapse-py3 (1.138.0~rc1) stable; urgency=medium * New synapse release 1.138.0rc1. diff --git a/debian/copyright b/debian/copyright index 9e407ce425..9814133edf 100644 --- a/debian/copyright +++ b/debian/copyright @@ -8,7 +8,7 @@ License: Apache-2.0 Files: * Copyright: 2023 New Vector Ltd -License: AGPL-3.0-or-later +License: AGPL-3.0-or-later or LicenseRef-Element-Commercial Files: synapse/config/saml2.py Copyright: 2015, Ericsson diff --git a/docker/Dockerfile b/docker/Dockerfile index 15c458fa28..727bc8bf5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -171,10 +171,10 @@ FROM docker.io/library/python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION} ARG TARGETARCH -LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' -LABEL org.opencontainers.image.documentation='https://github.com/element-hq/synapse/blob/master/docker/README.md' +LABEL org.opencontainers.image.url='https://github.com/element-hq/synapse' +LABEL org.opencontainers.image.documentation='https://element-hq.github.io/synapse/latest/' LABEL org.opencontainers.image.source='https://github.com/element-hq/synapse.git' -LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later' +LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later OR LicenseRef-Element-Commercial' # On the runtime image, /lib is a symlink to /usr/lib, so we need to copy the # libraries to the right place, else the `COPY` won't work. diff --git a/docs/log_contexts.md b/docs/log_contexts.md index 4e0c0e64f7..76710e10e0 100644 --- a/docs/log_contexts.md +++ b/docs/log_contexts.md @@ -143,8 +143,7 @@ cares about. The following sections describe pitfalls and helpful patterns when implementing these rules. -Always await your awaitables ----------------------------- +## Always await your awaitables Whenever you get an awaitable back from a function, you should `await` on it as soon as possible. Do not pass go; do not do any logging; do not @@ -203,6 +202,171 @@ async def sleep(seconds): return await context.make_deferred_yieldable(get_sleep_deferred(seconds)) ``` +## Deferred callbacks + +When a deferred callback is called, it inherits the current logcontext. The deferred +callback chain can resume a coroutine, which if following our logcontext rules, will +restore its own logcontext, then run: + + - until it yields control back to the reactor, setting the sentinel logcontext + - or until it finishes, restoring the logcontext it was started with (calling context) + +This behavior creates two specific issues: + +**Issue 1:** The first issue is that the callback may have reset the logcontext to the +sentinel before returning. This means our calling function will continue with the +sentinel logcontext instead of the logcontext it was started with (bad). + +**Issue 2:** The second issue is that the current logcontext that called the deferred +callback could finish before the callback finishes (bad). + +In the following example, the deferred callback is called with the "main" logcontext and +runs until we yield control back to the reactor in the `await` inside `clock.sleep(0)`. +Since `clock.sleep(0)` follows our logcontext rules, it sets the logcontext to the +sentinel before yielding control back to the reactor. Our `main` function continues with +the sentinel logcontext (first bad thing) instead of the "main" logcontext. Then the +`with LoggingContext("main")` block exits, finishing the "main" logcontext and yielding +control back to the reactor again. Finally, later on when `clock.sleep(0)` completes, +our `with LoggingContext("competing")` block exits, and restores the previous "main" +logcontext which has already finished, resulting in `WARNING: Re-starting finished log +context main` and leaking the `main` logcontext into the reactor which will then +erronously be associated with the next task the reactor picks up. + +```python +async def competing_callback(): + # Since this is run with the "main" logcontext, when the "competing" + # logcontext exits, it will restore the previous "main" logcontext which has + # already finished and results in "WARNING: Re-starting finished log context main" + # and leaking the `main` logcontext into the reactor. + with LoggingContext("competing"): + await clock.sleep(0) + +def main(): + with LoggingContext("main"): + d = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + # Call the callback within the "main" logcontext. + d.callback(None) + # Bad: This will be logged against sentinel logcontext + logger.debug("ugh") + +main() +``` + +**Solution 1:** We could of course fix this by following the general rule of "always +await your awaitables": + +```python +async def main(): + with LoggingContext("main"): + d = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + d.callback(None) + # Wait for `d` to finish before continuing so the "main" logcontext is + # still active. This works because `d` already follows our logcontext + # rules. If not, we would also have to use `make_deferred_yieldable(d)`. + await d + # Good: This will be logged against the "main" logcontext + logger.debug("phew") +``` + +**Solution 2:** We could also fix this by surrounding the call to `d.callback` with a +`PreserveLoggingContext`, which will reset the logcontext to the sentinel before calling +the callback, and restore the "main" logcontext afterwards before continuing the `main` +function. This solves the problem because when the "competing" logcontext exits, it will +restore the sentinel logcontext which is never finished by its nature, so there is no +warning and no leakage into the reactor. + +```python +async def main(): + with LoggingContext("main"): + d = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + d.callback(None) + with PreserveLoggingContext(): + # Call the callback with the sentinel logcontext. + d.callback(None) + # Good: This will be logged against the "main" logcontext + logger.debug("phew") +``` + +**Solution 3:** But let's say you *do* want to run (fire-and-forget) the deferred +callback in the current context without running into issues: + +We can solve the first issue by using `run_in_background(...)` to run the callback in +the current logcontext and it handles the magic behind the scenes of a) restoring the +calling logcontext before returning to the caller and b) resetting the logcontext to the +sentinel after the deferred completes and we yield control back to the reactor to avoid +leaking the logcontext into the reactor. + +To solve the second issue, we can extend the lifetime of the "main" logcontext by +avoiding the `LoggingContext`'s context manager lifetime methods +(`__enter__`/`__exit__`). We can still set "main" as the current logcontext by using +`PreserveLoggingContext` and passing in the "main" logcontext. + + +```python +async def main(): + main_context = LoggingContext("main") + with PreserveLoggingContext(main_context): + d = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + # The whole lambda will be run in the "main" logcontext. But we're using + # a trick to return the deferred `d` itself so that `run_in_background` + # will wait on that to complete and reset the logcontext to the sentinel + # when it does to avoid leaking the "main" logcontext into the reactor. + run_in_background(lambda: (d.callback(None), d)[1]) + # Good: This will be logged against the "main" logcontext + logger.debug("phew") + +... + +# Wherever possible, it's best to finish the logcontext by calling `__exit__` at some +# point. This allows us to catch bugs if we later try to erroneously restart a finished +# logcontext. +# +# Since the "main" logcontext stores the `LoggingContext.previous_context` when it is +# created, we can wrap this call in `PreserveLoggingContext()` to restore the correct +# previous logcontext. Our goal is to have the calling context remain unchanged after +# finishing the "main" logcontext. +with PreserveLoggingContext(): + # Finish the "main" logcontext + with main_context: + # Empty block - We're just trying to call `__exit__` on the "main" context + # manager to finish it. We can't call `__exit__` directly as the code expects us + # to `__enter__` before calling `__exit__` to `start`/`stop` things + # appropriately. And in any case, it's probably best not to call the internal + # methods directly. + pass +``` + +The same thing applies if you have some deferreds stored somewhere which you want to +callback in the current logcontext. + + +### Deferred errbacks and cancellations + +The same care should be taken when calling errbacks on deferreds. An errback and +callback act the same in this regard (see section above). + +```python +d = defer.Deferred() +d.addErrback(some_other_function) +d.errback(failure) +``` + +Additionally, cancellation is the same as directly calling the errback with a +`twisted.internet.defer.CancelledError`: + +```python +d = defer.Deferred() +d.addErrback(some_other_function) +d.cancel() +``` + + + + ## Fire-and-forget Sometimes you want to fire off a chain of execution, but not wait for @@ -384,3 +548,19 @@ chain are dropped. Dropping the the reference to an awaitable you're supposed to be awaiting is bad practice, so this doesn't actually happen too much. Unfortunately, when it does happen, it will lead to leaked logcontexts which are incredibly hard to track down. + + +## Debugging logcontext issues + +Debugging logcontext issues can be tricky as leaking or losing a logcontext will surface +downstream and can point to an unrelated part of the codebase. It's best to enable debug +logging for `synapse.logging.context.debug` (needs to be explicitly configured) and go +backwards in the logs from the point where the issue is observed to find the root cause. + +`log.config.yaml` +```yaml +loggers: + # Unlike other loggers, this one needs to be explicitly configured to see debug logs. + synapse.logging.context.debug: + level: DEBUG +``` diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 49b7e06bb3..0f15a9dcc5 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -195,12 +195,15 @@ _Changed in Synapse v1.132.0: Added the `room_config` argument. Callbacks that o async def user_may_create_room(user_id: str, room_config: synapse.module_api.JsonDict) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] ``` -Called when processing a room creation request. +Called when processing a room creation or room upgrade request. The arguments passed to this callback are: * `user_id`: The Matrix user ID of the user (e.g. `@alice:example.com`). -* `room_config`: The contents of the body of a [/createRoom request](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom) as a dictionary. +* `room_config`: The contents of the body of the [`/createRoom` request](https://spec.matrix.org/v1.15/client-server-api/#post_matrixclientv3createroom) as a dictionary. + For a [room upgrade request](https://spec.matrix.org/v1.15/client-server-api/#post_matrixclientv3roomsroomidupgrade) it is a synthesised subset of what an equivalent + `/createRoom` request would have looked like. Specifically, it contains the `creation_content` (linking to the previous room) and `initial_state` (containing a + subset of the state of the previous room). The callback must return one of: - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 4571141ab3..5b49085973 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2006,9 +2006,8 @@ This setting has the following sub-options: Default configuration: ```yaml rc_reports: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 ``` Example configuration: @@ -2031,9 +2030,8 @@ This setting has the following sub-options: Default configuration: ```yaml rc_room_creation: - per_user: - per_second: 0.016 - burst_count: 10.0 + per_second: 0.016 + burst_count: 10.0 ``` Example configuration: diff --git a/poetry.lock b/poetry.lock index 8759b43dbe..6ff90ed361 100644 --- a/poetry.lock +++ b/poetry.lock @@ -68,63 +68,75 @@ visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] [[package]] name = "bcrypt" -version = "4.3.0" +version = "5.0.0" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59e1aa0e2cd871b08ca146ed08445038f42ff75968c7ae50d2fdd7860ade2180"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:0042b2e342e9ae3d2ed22727c1262f76cc4f345683b5c1715f0250cf4277294f"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74a8d21a09f5e025a9a23e7c0fd2c7fe8e7503e4d356c0a2c1486ba010619f09"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:0142b2cb84a009f8452c8c5a33ace5e3dfec4159e7735f5afe9a4d50a8ea722d"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:12fa6ce40cde3f0b899729dbd7d5e8811cb892d31b6f7d0334a1f37748b789fd"}, - {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:5bd3cca1f2aa5dbcf39e2aa13dd094ea181f48959e1071265de49cc2b82525af"}, - {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:335a420cfd63fc5bc27308e929bee231c15c85cc4c496610ffb17923abf7f231"}, - {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:0e30e5e67aed0187a1764911af023043b4542e70a7461ad20e837e94d23e1d6c"}, - {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3b8d62290ebefd49ee0b3ce7500f5dbdcf13b81402c05f6dafab9a1e1b27212f"}, - {file = "bcrypt-4.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef6630e0ec01376f59a006dc72918b1bf436c3b571b80fa1968d775fa02fe7d"}, - {file = "bcrypt-4.3.0-cp313-cp313t-win32.whl", hash = "sha256:7a4be4cbf241afee43f1c3969b9103a41b40bcb3a3f467ab19f891d9bc4642e4"}, - {file = "bcrypt-4.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c1949bf259a388863ced887c7861da1df681cb2388645766c89fdfd9004c669"}, - {file = "bcrypt-4.3.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:f81b0ed2639568bf14749112298f9e4e2b28853dab50a8b357e31798686a036d"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:864f8f19adbe13b7de11ba15d85d4a428c7e2f344bac110f667676a0ff84924b"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e36506d001e93bffe59754397572f21bb5dc7c83f54454c990c74a468cd589e"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:842d08d75d9fe9fb94b18b071090220697f9f184d4547179b60734846461ed59"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7c03296b85cb87db865d91da79bf63d5609284fc0cab9472fdd8367bbd830753"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:62f26585e8b219cdc909b6a0069efc5e4267e25d4a3770a364ac58024f62a761"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:beeefe437218a65322fbd0069eb437e7c98137e08f22c4660ac2dc795c31f8bb"}, - {file = "bcrypt-4.3.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:97eea7408db3a5bcce4a55d13245ab3fa566e23b4c67cd227062bb49e26c585d"}, - {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:191354ebfe305e84f344c5964c7cd5f924a3bfc5d405c75ad07f232b6dffb49f"}, - {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:41261d64150858eeb5ff43c753c4b216991e0ae16614a308a15d909503617732"}, - {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:33752b1ba962ee793fa2b6321404bf20011fe45b9afd2a842139de3011898fef"}, - {file = "bcrypt-4.3.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:50e6e80a4bfd23a25f5c05b90167c19030cf9f87930f7cb2eacb99f45d1c3304"}, - {file = "bcrypt-4.3.0-cp38-abi3-win32.whl", hash = "sha256:67a561c4d9fb9465ec866177e7aebcad08fe23aaf6fbd692a6fab69088abfc51"}, - {file = "bcrypt-4.3.0-cp38-abi3-win_amd64.whl", hash = "sha256:584027857bc2843772114717a7490a37f68da563b3620f78a849bcb54dc11e62"}, - {file = "bcrypt-4.3.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0d3efb1157edebfd9128e4e46e2ac1a64e0c1fe46fb023158a407c7892b0f8c3"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08bacc884fd302b611226c01014eca277d48f0a05187666bca23aac0dad6fe24"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6746e6fec103fcd509b96bacdfdaa2fbde9a553245dbada284435173a6f1aef"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:afe327968aaf13fc143a56a3360cb27d4ad0345e34da12c7290f1b00b8fe9a8b"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d9af79d322e735b1fc33404b5765108ae0ff232d4b54666d46730f8ac1a43676"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f1e3ffa1365e8702dc48c8b360fef8d7afeca482809c5e45e653af82ccd088c1"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3004df1b323d10021fda07a813fd33e0fd57bef0e9a480bb143877f6cba996fe"}, - {file = "bcrypt-4.3.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:531457e5c839d8caea9b589a1bcfe3756b0547d7814e9ce3d437f17da75c32b0"}, - {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:17a854d9a7a476a89dcef6c8bd119ad23e0f82557afbd2c442777a16408e614f"}, - {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6fb1fd3ab08c0cbc6826a2e0447610c6f09e983a281b919ed721ad32236b8b23"}, - {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e965a9c1e9a393b8005031ff52583cedc15b7884fce7deb8b0346388837d6cfe"}, - {file = "bcrypt-4.3.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:79e70b8342a33b52b55d93b3a59223a844962bef479f6a0ea318ebbcadf71505"}, - {file = "bcrypt-4.3.0-cp39-abi3-win32.whl", hash = "sha256:b4d4e57f0a63fd0b358eb765063ff661328f69a04494427265950c71b992a39a"}, - {file = "bcrypt-4.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b"}, - {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c950d682f0952bafcceaf709761da0a32a942272fad381081b51096ffa46cea1"}, - {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:107d53b5c67e0bbc3f03ebf5b030e0403d24dda980f8e244795335ba7b4a027d"}, - {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b693dbb82b3c27a1604a3dff5bfc5418a7e6a781bb795288141e5f80cf3a3492"}, - {file = "bcrypt-4.3.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:b6354d3760fcd31994a14c89659dee887f1351a06e5dac3c1142307172a79f90"}, - {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a839320bf27d474e52ef8cb16449bb2ce0ba03ca9f44daba6d93fa1d8828e48a"}, - {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bdc6a24e754a555d7316fa4774e64c6c3997d27ed2d1964d55920c7c227bc4ce"}, - {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:55a935b8e9a1d2def0626c4269db3fcd26728cbff1e84f0341465c31c4ee56d8"}, - {file = "bcrypt-4.3.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:57967b7a28d855313a963aaea51bf6df89f833db4320da458e5b3c5ab6d4c938"}, - {file = "bcrypt-4.3.0.tar.gz", hash = "sha256:3a3fd2204178b6d2adcf09cb4f6426ffef54762577a7c9b54c159008cb288c18"}, + {file = "bcrypt-5.0.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f3c08197f3039bec79cee59a606d62b96b16669cff3949f21e74796b6e3cd2be"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:200af71bc25f22006f4069060c88ed36f8aa4ff7f53e67ff04d2ab3f1e79a5b2"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c58b56cdfb03202b3bcc9fd8daee8e8e9b6d7e3163aa97c631dfcfcc24d36c86"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bfd2a34de661f34d0bda43c3e4e79df586e4716ef401fe31ea39d69d581ef23"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:83e787d7a84dbbfba6f250dd7a5efd689e935f03dd83b0f919d39349e1f23f83"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:38cac74101777a6a7d3b3e3cfefa57089b5ada650dce2baf0cbdd9d65db22a9e"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:741449132f64b3524e95cd30e5cd3343006ce146088f074f31ab26b94e6c75ba"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win32.whl", hash = "sha256:9d52ed507c2488eddd6a95bccee4e808d3234fa78dd370e24bac65a21212b861"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f6984a24db30548fd39a44360532898c33528b74aedf81c26cf29c51ee47057e"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9fffdb387abe6aa775af36ef16f55e318dcda4194ddbf82007a6f21da29de8f5"}, + {file = "bcrypt-5.0.0-cp314-cp314t-macosx_10_12_universal2.whl", hash = "sha256:4870a52610537037adb382444fefd3706d96d663ac44cbb2f37e3919dca3d7ef"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48f753100931605686f74e27a7b49238122aa761a9aefe9373265b8b7aa43ea4"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f70aadb7a809305226daedf75d90379c397b094755a710d7014b8b117df1ebbf"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:744d3c6b164caa658adcb72cb8cc9ad9b4b75c7db507ab4bc2480474a51989da"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a28bc05039bdf3289d757f49d616ab3efe8cf40d8e8001ccdd621cd4f98f4fc9"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7f277a4b3390ab4bebe597800a90da0edae882c6196d3038a73adf446c4f969f"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:79cfa161eda8d2ddf29acad370356b47f02387153b11d46042e93a0a95127493"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a5393eae5722bcef046a990b84dff02b954904c36a194f6cfc817d7dca6c6f0b"}, + {file = "bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4c94dec1b5ab5d522750cb059bb9409ea8872d4494fd152b53cca99f1ddd8c"}, + {file = "bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0cae4cb350934dfd74c020525eeae0a5f79257e8a201c0c176f4b84fdbf2a4b4"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win32.whl", hash = "sha256:b17366316c654e1ad0306a6858e189fc835eca39f7eb2cafd6aaca8ce0c40a2e"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:92864f54fb48b4c718fc92a32825d0e42265a627f956bc0361fe869f1adc3e7d"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dd19cf5184a90c873009244586396a6a884d591a5323f0e8a5922560718d4993"}, + {file = "bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb"}, + {file = "bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538"}, + {file = "bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9"}, + {file = "bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980"}, + {file = "bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172"}, + {file = "bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683"}, + {file = "bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2"}, + {file = "bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7edda91d5ab52b15636d9c30da87d2cc84f426c72b9dba7a9b4fe142ba11f534"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:046ad6db88edb3c5ece4369af997938fb1c19d6a699b9c1b27b0db432faae4c4"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dcd58e2b3a908b5ecc9b9df2f0085592506ac2d5110786018ee5e160f28e0911"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:6b8f520b61e8781efee73cba14e3e8c9556ccfb375623f4f97429544734545b4"}, + {file = "bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd"}, ] [package.extras] @@ -997,107 +1009,153 @@ pyasn1 = ">=0.4.6" [[package]] name = "lxml" -version = "6.0.0" +version = "6.0.2" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, - {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc"}, - {file = "lxml-6.0.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76"}, - {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541"}, - {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b"}, - {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7"}, - {file = "lxml-6.0.0-cp310-cp310-win32.whl", hash = "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452"}, - {file = "lxml-6.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e"}, - {file = "lxml-6.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8"}, - {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36"}, - {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e"}, - {file = "lxml-6.0.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58"}, - {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2"}, - {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851"}, - {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f"}, - {file = "lxml-6.0.0-cp311-cp311-win32.whl", hash = "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c"}, - {file = "lxml-6.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816"}, - {file = "lxml-6.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab"}, - {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108"}, - {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da"}, - {file = "lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e"}, - {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741"}, - {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3"}, - {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16"}, - {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0"}, - {file = "lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a"}, - {file = "lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3"}, - {file = "lxml-6.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb"}, - {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da"}, - {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29"}, - {file = "lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4"}, - {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca"}, - {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf"}, - {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f"}, - {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef"}, - {file = "lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181"}, - {file = "lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e"}, - {file = "lxml-6.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03"}, - {file = "lxml-6.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db"}, - {file = "lxml-6.0.0-cp38-cp38-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994"}, - {file = "lxml-6.0.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d"}, - {file = "lxml-6.0.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97"}, - {file = "lxml-6.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b"}, - {file = "lxml-6.0.0-cp38-cp38-win32.whl", hash = "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54"}, - {file = "lxml-6.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a"}, - {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261"}, - {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281"}, - {file = "lxml-6.0.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922"}, - {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0"}, - {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a"}, - {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5"}, - {file = "lxml-6.0.0-cp39-cp39-win32.whl", hash = "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256"}, - {file = "lxml-6.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae"}, - {file = "lxml-6.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065"}, - {file = "lxml-6.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734"}, - {file = "lxml-6.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349"}, - {file = "lxml-6.0.0.tar.gz", hash = "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72"}, + {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, + {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c"}, + {file = "lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b"}, + {file = "lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0"}, + {file = "lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5"}, + {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607"}, + {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7"}, + {file = "lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46"}, + {file = "lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078"}, + {file = "lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285"}, + {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456"}, + {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322"}, + {file = "lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849"}, + {file = "lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f"}, + {file = "lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6"}, + {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77"}, + {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314"}, + {file = "lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2"}, + {file = "lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7"}, + {file = "lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf"}, + {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe"}, + {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c"}, + {file = "lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b"}, + {file = "lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed"}, + {file = "lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8"}, + {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d"}, + {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f"}, + {file = "lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312"}, + {file = "lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca"}, + {file = "lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c"}, + {file = "lxml-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a656ca105115f6b766bba324f23a67914d9c728dafec57638e2b92a9dcd76c62"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c54d83a2188a10ebdba573f16bd97135d06c9ef60c3dc495315c7a28c80a263f"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:1ea99340b3c729beea786f78c38f60f4795622f36e305d9c9be402201efdc3b7"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af85529ae8d2a453feee4c780d9406a5e3b17cee0dd75c18bd31adcd584debc3"}, + {file = "lxml-6.0.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fe659f6b5d10fb5a17f00a50eb903eb277a71ee35df4615db573c069bcf967ac"}, + {file = "lxml-6.0.2-cp38-cp38-win32.whl", hash = "sha256:5921d924aa5468c939d95c9814fa9f9b5935a6ff4e679e26aaf2951f74043512"}, + {file = "lxml-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:0aa7070978f893954008ab73bb9e3c24a7c56c054e00566a21b553dc18105fca"}, + {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2c8458c2cdd29589a8367c09c8f030f1d202be673f0ca224ec18590b3b9fb694"}, + {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fee0851639d06276e6b387f1c190eb9d7f06f7f53514e966b26bae46481ec90"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b2142a376b40b6736dfc214fd2902409e9e3857eff554fed2d3c60f097e62a62"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6b5b39cc7e2998f968f05309e666103b53e2edd01df8dc51b90d734c0825444"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4aec24d6b72ee457ec665344a29acb2d35937d5192faebe429ea02633151aad"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:b42f4d86b451c2f9d06ffb4f8bbc776e04df3ba070b9fe2657804b1b40277c48"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cdaefac66e8b8f30e37a9b4768a391e1f8a16a7526d5bc77a7928408ef68e93"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:b738f7e648735714bbb82bdfd030203360cfeab7f6e8a34772b3c8c8b820568c"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daf42de090d59db025af61ce6bdb2521f0f102ea0e6ea310f13c17610a97da4c"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:66328dabea70b5ba7e53d94aa774b733cf66686535f3bc9250a7aab53a91caaf"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:e237b807d68a61fc3b1e845407e27e5eb8ef69bc93fe8505337c1acb4ee300b6"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ac02dc29fd397608f8eb15ac1610ae2f2f0154b03f631e6d724d9e2ad4ee2c84"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:817ef43a0c0b4a77bd166dc9a09a555394105ff3374777ad41f453526e37f9cb"}, + {file = "lxml-6.0.2-cp39-cp39-win32.whl", hash = "sha256:bc532422ff26b304cfb62b328826bd995c96154ffd2bac4544f37dbb95ecaa8f"}, + {file = "lxml-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:995e783eb0374c120f528f807443ad5a83a656a8624c467ea73781fc5f8a8304"}, + {file = "lxml-6.0.2-cp39-cp39-win_arm64.whl", hash = "sha256:08b9d5e803c2e4725ae9e8559ee880e5328ed61aa0935244e0515d7d9dbec0aa"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e"}, + {file = "lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62"}, ] [package.extras] @@ -1531,14 +1589,14 @@ files = [ [[package]] name = "phonenumbers" -version = "9.0.14" +version = "9.0.15" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-9.0.14-py2.py3-none-any.whl", hash = "sha256:6bdf5c46dbfefa1d941d122432d1958418d1dfe3f8c8c81d4c8e80f5442ea41f"}, - {file = "phonenumbers-9.0.14.tar.gz", hash = "sha256:98afb3e86bf9ae02cc7c98ca44fa8827babb72842f90da9884c5d998937572ae"}, + {file = "phonenumbers-9.0.15-py2.py3-none-any.whl", hash = "sha256:269b73bc05258e8fd57582770b9559307099ea677c8f1dc5272476f661344776"}, + {file = "phonenumbers-9.0.15.tar.gz", hash = "sha256:345ff7f23768332d866f37732f815cdf1d33c7f0961246562a5c5b78c12c3ff3"}, ] [[package]] @@ -2562,14 +2620,14 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve [[package]] name = "setuptools-rust" -version = "1.11.1" +version = "1.12.0" description = "Setuptools Rust extension plugin" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools_rust-1.11.1-py3-none-any.whl", hash = "sha256:5eaaddaed268dc24a527ffa659ce56b22d3cf17b781247b779efd611031fe8ea"}, - {file = "setuptools_rust-1.11.1.tar.gz", hash = "sha256:7dabc4392252ced314b8050d63276e05fdc5d32398fc7d3cce1f6a6ac35b76c0"}, + {file = "setuptools_rust-1.12.0-py3-none-any.whl", hash = "sha256:7e7db90547f224a835b45f5ad90c983340828a345554a9a660bdb2de8605dcdd"}, + {file = "setuptools_rust-1.12.0.tar.gz", hash = "sha256:d94a93f0c97751c17014565f07bdc324bee45d396cd1bba83d8e7af92b945f0c"}, ] [package.dependencies] @@ -2794,20 +2852,20 @@ docs = ["sphinx", "sphinx-rtd-theme"] [[package]] name = "twine" -version = "6.1.0" +version = "6.2.0" description = "Collection of utilities for publishing packages on PyPI" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384"}, - {file = "twine-6.1.0.tar.gz", hash = "sha256:be324f6272eff91d07ee93f251edf232fc647935dd585ac003539b42404a8dbd"}, + {file = "twine-6.2.0-py3-none-any.whl", hash = "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8"}, + {file = "twine-6.2.0.tar.gz", hash = "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf"}, ] [package.dependencies] id = "*" importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} -keyring = {version = ">=15.1", markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +keyring = {version = ">=21.2.0", markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} packaging = ">=24.0" readme-renderer = ">=35.0" requests = ">=2.20" @@ -2817,7 +2875,7 @@ rich = ">=12.0.0" urllib3 = ">=1.26.0" [package.extras] -keyring = ["keyring (>=15.1)"] +keyring = ["keyring (>=21.2.0)"] [[package]] name = "twisted" @@ -3038,14 +3096,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.14.1" +version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, - {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index c548a652e9..a79c19ae90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,10 +101,10 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.138.2" +version = "1.139.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] -license = "AGPL-3.0-or-later" +license = "AGPL-3.0-or-later OR LicenseRef-Element-Commercial" readme = "README.rst" repository = "https://github.com/element-hq/synapse" packages = [ diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index b6cdf98f55..e67dae169f 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,7 +12,7 @@ * . */ -use std::{collections::HashMap, future::Future}; +use std::{collections::HashMap, future::Future, sync::OnceLock}; use anyhow::Context; use futures::TryStreamExt; @@ -299,5 +299,22 @@ where }); }); - Ok(deferred) + // Make the deferred follow the Synapse logcontext rules + make_deferred_yieldable(py, &deferred) +} + +static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); + +/// Given a deferred, make it follow the Synapse logcontext rules +fn make_deferred_yieldable<'py>( + py: Python<'py>, + deferred: &Bound<'py, PyAny>, +) -> PyResult> { + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { + let sys = PyModule::import(py, "synapse.logging.context").unwrap(); + let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); + func + }); + + make_deferred_yieldable.call1(py, (deferred,))?.extract(py) } diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 0a658979d9..2d74fee1cb 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2259,9 +2259,8 @@ properties: Setting this to a high value allows users to report content quickly, possibly in duplicate. This can result in higher database usage. default: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 examples: - per_second: 2.0 burst_count: 20.0 @@ -2270,9 +2269,8 @@ properties: description: >- Sets rate limits for how often users are able to create rooms. default: - per_user: - per_second: 0.016 - burst_count: 10.0 + per_second: 0.016 + burst_count: 10.0 examples: - per_second: 1.0 burst_count: 5.0 diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index e170aabdae..0b854cdba5 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -68,18 +68,42 @@ PROMETHEUS_METRIC_MISSING_FROM_LIST_TO_CHECK = ErrorCode( category="per-homeserver-tenant-metrics", ) +PREFER_SYNAPSE_CLOCK_CALL_LATER = ErrorCode( + "call-later-not-tracked", + "Prefer using `synapse.util.Clock.call_later` instead of `reactor.callLater`", + category="synapse-reactor-clock", +) + +PREFER_SYNAPSE_CLOCK_LOOPING_CALL = ErrorCode( + "prefer-synapse-clock-looping-call", + "Prefer using `synapse.util.Clock.looping_call` instead of `task.LoopingCall`", + category="synapse-reactor-clock", +) + PREFER_SYNAPSE_CLOCK_CALL_WHEN_RUNNING = ErrorCode( "prefer-synapse-clock-call-when-running", - "`synapse.util.Clock.call_when_running` should be used instead of `reactor.callWhenRunning`", + "Prefer using `synapse.util.Clock.call_when_running` instead of `reactor.callWhenRunning`", category="synapse-reactor-clock", ) PREFER_SYNAPSE_CLOCK_ADD_SYSTEM_EVENT_TRIGGER = ErrorCode( "prefer-synapse-clock-add-system-event-trigger", - "`synapse.util.Clock.add_system_event_trigger` should be used instead of `reactor.addSystemEventTrigger`", + "Prefer using `synapse.util.Clock.add_system_event_trigger` instead of `reactor.addSystemEventTrigger`", category="synapse-reactor-clock", ) +MULTIPLE_INTERNAL_CLOCKS_CREATED = ErrorCode( + "multiple-internal-clocks", + "Only one instance of `clock.Clock` should be created", + category="synapse-reactor-clock", +) + +UNTRACKED_BACKGROUND_PROCESS = ErrorCode( + "untracked-background-process", + "Prefer using `HomeServer.run_as_background_process` method over the bare `run_as_background_process`", + category="synapse-tracked-calls", +) + class Sentinel(enum.Enum): # defining a sentinel in this way allows mypy to correctly handle the @@ -222,6 +246,18 @@ class SynapsePlugin(Plugin): # callback, let's just pass it in while we have it. return lambda ctx: check_prometheus_metric_instantiation(ctx, fullname) + if fullname == "twisted.internet.task.LoopingCall": + return check_looping_call + + if fullname == "synapse.util.clock.Clock": + return check_clock_creation + + if ( + fullname + == "synapse.metrics.background_process_metrics.run_as_background_process" + ): + return check_background_process + return None def get_method_signature_hook( @@ -241,6 +277,13 @@ class SynapsePlugin(Plugin): ): return check_is_cacheable_wrapper + if fullname in ( + "twisted.internet.interfaces.IReactorTime.callLater", + "synapse.types.ISynapseThreadlessReactor.callLater", + "synapse.types.ISynapseReactor.callLater", + ): + return check_call_later + if fullname in ( "twisted.internet.interfaces.IReactorCore.callWhenRunning", "synapse.types.ISynapseThreadlessReactor.callWhenRunning", @@ -258,6 +301,78 @@ class SynapsePlugin(Plugin): return None +def check_clock_creation(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that the only `clock.Clock` instance is the one used by the `HomeServer`. + This is so that the `HomeServer` can cancel any tracked delayed or looping calls + during server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected the only `clock.Clock` instance to be the one used by the `HomeServer`. " + "This is so that the `HomeServer` can cancel any tracked delayed or looping calls " + "during server shutdown", + ctx.context, + code=MULTIPLE_INTERNAL_CLOCKS_CREATED, + ) + + return signature + + +def check_call_later(ctx: MethodSigContext) -> CallableType: + """ + Ensure that the `reactor.callLater` callsites aren't used. + + `synapse.util.Clock.call_later` should always be used instead of `reactor.callLater`. + This is because the `synapse.util.Clock` tracks delayed calls in order to cancel any + outstanding calls during server shutdown. Delayed calls which are either short lived + (<~60s) or frequently called and can be tracked via other means could be candidates for + using `synapse.util.Clock.call_later` with `call_later_cancel_on_shutdown` set to + `False`. There shouldn't be a need to use `reactor.callLater` outside of tests or the + `Clock` class itself. If a need arises, you can use a type ignore comment to disable the + check, e.g. `# type: ignore[call-later-not-tracked]`. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected all `reactor.callLater` calls to use `synapse.util.Clock.call_later` " + "instead. This is so that long lived calls can be tracked for cancellation during " + "server shutdown", + ctx.context, + code=PREFER_SYNAPSE_CLOCK_CALL_LATER, + ) + + return signature + + +def check_looping_call(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that the `task.LoopingCall` callsites aren't used. + + `synapse.util.Clock.looping_call` should always be used instead of `task.LoopingCall`. + `synapse.util.Clock` tracks looping calls in order to cancel any outstanding calls + during server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected all `task.LoopingCall` instances to use `synapse.util.Clock.looping_call` " + "instead. This is so that long lived calls can be tracked for cancellation during " + "server shutdown", + ctx.context, + code=PREFER_SYNAPSE_CLOCK_LOOPING_CALL, + ) + + return signature + + def check_call_when_running(ctx: MethodSigContext) -> CallableType: """ Ensure that the `reactor.callWhenRunning` callsites aren't used. @@ -312,6 +427,27 @@ def check_add_system_event_trigger(ctx: MethodSigContext) -> CallableType: return signature +def check_background_process(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that calls to `run_as_background_process` use the `HomeServer` method. + This is so that the `HomeServer` can cancel any running background processes during + server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Prefer using `HomeServer.run_as_background_process` method over the bare " + "`run_as_background_process`. This is so that the `HomeServer` can cancel " + "any background processes during server shutdown", + ctx.context, + code=UNTRACKED_BACKGROUND_PROCESS, + ) + + return signature + + def analyze_prometheus_metric_classes(ctx: ClassDefContext) -> None: """ Cross-check the list of Prometheus metric classes against the diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py index 8878e364e2..f66c01040c 100755 --- a/synapse/_scripts/generate_workers_map.py +++ b/synapse/_scripts/generate_workers_map.py @@ -157,7 +157,12 @@ def get_registered_paths_for_default( # TODO We only do this to avoid an error, but don't need the database etc hs.setup() registered_paths = get_registered_paths_for_hs(hs) - hs.cleanup() + # NOTE: a more robust implementation would properly shutdown/cleanup each server + # to avoid resource buildup. + # However, the call to `shutdown` is `async` so it would require additional complexity here. + # We are intentionally skipping this cleanup because this is a short-lived, one-off + # utility script where the simpler approach is sufficient and we shouldn't run into + # any resource buildup issues. return registered_paths diff --git a/synapse/_scripts/update_synapse_database.py b/synapse/_scripts/update_synapse_database.py index caaecda161..ad02f0ed88 100644 --- a/synapse/_scripts/update_synapse_database.py +++ b/synapse/_scripts/update_synapse_database.py @@ -28,7 +28,6 @@ import yaml from twisted.internet import defer, reactor as reactor_ from synapse.config.homeserver import HomeServerConfig -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.server import HomeServer from synapse.storage import DataStore from synapse.types import ISynapseReactor @@ -53,7 +52,6 @@ class MockHomeserver(HomeServer): def run_background_updates(hs: HomeServer) -> None: - server_name = hs.hostname main = hs.get_datastores().main state = hs.get_datastores().state @@ -67,9 +65,8 @@ def run_background_updates(hs: HomeServer) -> None: def run() -> None: # Apply all background updates on the database. defer.ensureDeferred( - run_as_background_process( + hs.run_as_background_process( "background_updates", - server_name, run_background_updates, ) ) diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index ef82ea9cc7..baa6b27336 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -33,7 +33,6 @@ from synapse.api.errors import ( UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import ( active_span, force_tracing, @@ -229,13 +228,12 @@ class MasDelegatedAuth(BaseAuth): try: with start_active_span("mas-introspect-token"): inject_request_headers(raw_headers) - with PreserveLoggingContext(): - resp_body = await self._rust_http_client.post( - url=self._introspection_endpoint, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) + resp_body = await self._rust_http_client.post( + url=self._introspection_endpoint, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) except HttpResponseException as e: end_time = self._clock.time() introspection_response_timer.labels( diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 11a89dd3ed..b6adcc83dc 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -38,7 +38,6 @@ from synapse.api.errors import ( UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import ( active_span, force_tracing, @@ -327,13 +326,12 @@ class MSC3861DelegatedAuth(BaseAuth): try: with start_active_span("mas-introspect-token"): inject_request_headers(raw_headers) - with PreserveLoggingContext(): - resp_body = await self._rust_http_client.post( - url=uri, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) + resp_body = await self._rust_http_client.post( + url=uri, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) except HttpResponseException as e: end_time = self._clock.time() introspection_response_timer.labels( diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d8e1d30b44..655f684ecf 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -28,6 +28,7 @@ import sys import traceback import warnings from textwrap import indent +from threading import Thread from typing import ( TYPE_CHECKING, Any, @@ -40,6 +41,7 @@ from typing import ( Tuple, cast, ) +from wsgiref.simple_server import WSGIServer from cryptography.utils import CryptographyDeprecationWarning from typing_extensions import ParamSpec @@ -97,22 +99,47 @@ reactor = cast(ISynapseReactor, _reactor) logger = logging.getLogger(__name__) -# list of tuples of function, args list, kwargs dict -_sighup_callbacks: List[ - Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]] -] = [] +_instance_id_to_sighup_callbacks_map: Dict[ + str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] +] = {} +""" +Map from homeserver instance_id to a list of callbacks. + +We use `instance_id` instead of `server_name` because it's possible to have multiple +workers running in the same process with the same `server_name`. +""" P = ParamSpec("P") -def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: +def register_sighup( + homeserver_instance_id: str, + func: Callable[P, None], + *args: P.args, + **kwargs: P.kwargs, +) -> None: """ Register a function to be called when a SIGHUP occurs. Args: + homeserver_instance_id: The unique ID for this Synapse process instance + (`hs.get_instance_id()`) that this hook is associated with. func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.append((func, args, kwargs)) + + _instance_id_to_sighup_callbacks_map.setdefault(homeserver_instance_id, []).append( + (func, args, kwargs) + ) + + +def unregister_sighups(instance_id: str) -> None: + """ + Unregister all sighup functions associated with this Synapse instance. + + Args: + instance_id: Unique ID for this Synapse process instance. + """ + _instance_id_to_sighup_callbacks_map.pop(instance_id, []) def start_worker_reactor( @@ -281,7 +308,9 @@ def register_start( clock.call_when_running(lambda: defer.ensureDeferred(wrapper())) -def listen_metrics(bind_addresses: StrCollection, port: int) -> None: +def listen_metrics( + bind_addresses: StrCollection, port: int +) -> List[Tuple[WSGIServer, Thread]]: """ Start Prometheus metrics server. @@ -294,14 +323,22 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: bytecode at a time), this still works because the metrics thread can preempt the Twisted reactor thread between bytecode boundaries and the metrics thread gets scheduled with roughly equal priority to the Twisted reactor thread. + + Returns: + List of WSGIServer with the thread they are running on. """ from prometheus_client import start_http_server as start_http_server_prometheus from synapse.metrics import RegistryProxy + servers: List[Tuple[WSGIServer, Thread]] = [] for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) - start_http_server_prometheus(port, addr=host, registry=RegistryProxy) + server, thread = start_http_server_prometheus( + port, addr=host, registry=RegistryProxy + ) + servers.append((server, thread)) + return servers def listen_manhole( @@ -309,7 +346,7 @@ def listen_manhole( port: int, manhole_settings: ManholeConfig, manhole_globals: dict, -) -> None: +) -> List[Port]: # twisted.conch.manhole 21.1.0 uses "int_from_bytes", which produces a confusing # warning. It's fixed by https://github.com/twisted/twisted/pull/1522), so # suppress the warning for now. @@ -321,7 +358,7 @@ def listen_manhole( from synapse.util.manhole import manhole - listen_tcp( + return listen_tcp( bind_addresses, port, manhole(settings=manhole_settings, globals=manhole_globals), @@ -498,7 +535,7 @@ def refresh_certificate(hs: "HomeServer") -> None: logger.info("Context factories updated.") -async def start(hs: "HomeServer") -> None: +async def start(hs: "HomeServer", freeze: bool = True) -> None: """ Start a Synapse server or worker. @@ -509,6 +546,11 @@ async def start(hs: "HomeServer") -> None: Args: hs: homeserver instance + freeze: whether to freeze the homeserver base objects in the garbage collector. + May improve garbage collection performance by marking objects with an effectively + static lifetime as frozen so they don't need to be considered for cleanup. + If you ever want to `shutdown` the homeserver, this needs to be + False otherwise the homeserver cannot be garbage collected after `shutdown`. """ server_name = hs.hostname reactor = hs.get_reactor() @@ -541,12 +583,17 @@ async def start(hs: "HomeServer") -> None: # we're not using systemd. sdnotify(b"RELOADING=1") - for i, args, kwargs in _sighup_callbacks: - i(*args, **kwargs) + for sighup_callbacks in _instance_id_to_sighup_callbacks_map.values(): + for func, args, kwargs in sighup_callbacks: + func(*args, **kwargs) sdnotify(b"READY=1") - return run_as_background_process( + # It's okay to ignore the linter error here and call + # `run_as_background_process` directly because `_handle_sighup` operates + # outside of the scope of a specific `HomeServer` instance and holds no + # references to it which would prevent a clean shutdown. + return run_as_background_process( # type: ignore[untracked-background-process] "sighup", server_name, _handle_sighup, @@ -564,8 +611,8 @@ async def start(hs: "HomeServer") -> None: signal.signal(signal.SIGHUP, run_sighup) - register_sighup(refresh_certificate, hs) - register_sighup(reload_cache_config, hs.config) + register_sighup(hs.get_instance_id(), refresh_certificate, hs) + register_sighup(hs.get_instance_id(), reload_cache_config, hs.config) # Apply the cache config. hs.config.caches.resize_all_caches() @@ -599,11 +646,15 @@ async def start(hs: "HomeServer") -> None: hs.get_pusherpool().start() def log_shutdown() -> None: - with LoggingContext("log_shutdown"): + with LoggingContext(name="log_shutdown", server_name=server_name): logger.info("Shutting down...") # Log when we start the shut down process. - hs.get_clock().add_system_event_trigger("before", "shutdown", log_shutdown) + hs.register_sync_shutdown_handler( + phase="before", + eventType="shutdown", + shutdown_func=log_shutdown, + ) setup_sentry(hs) setup_sdnotify(hs) @@ -632,18 +683,24 @@ async def start(hs: "HomeServer") -> None: # `REQUIRED_ON_BACKGROUND_TASK_STARTUP` start_phone_stats_home(hs) - # We now freeze all allocated objects in the hopes that (almost) - # everything currently allocated are things that will be used for the - # rest of time. Doing so means less work each GC (hopefully). - # - # PyPy does not (yet?) implement gc.freeze() - if hasattr(gc, "freeze"): - gc.collect() - gc.freeze() + if freeze: + # We now freeze all allocated objects in the hopes that (almost) + # everything currently allocated are things that will be used for the + # rest of time. Doing so means less work each GC (hopefully). + # + # Note that freezing the homeserver object means that it won't be able to be + # garbage collected in the case of attempting an in-memory `shutdown`. This only + # needs to be considered if such a case is desirable. Exiting the entire Python + # process will function expectedly either way. + # + # PyPy does not (yet?) implement gc.freeze() + if hasattr(gc, "freeze"): + gc.collect() + gc.freeze() - # Speed up shutdowns by freezing all allocated objects. This moves everything - # into the permanent generation and excludes them from the final GC. - atexit.register(gc.freeze) + # Speed up process exit by freezing all allocated objects. This moves everything + # into the permanent generation and excludes them from the final GC. + atexit.register(gc.freeze) def reload_cache_config(config: HomeServerConfig) -> None: diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index c0c594577e..bc84dbdf49 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -329,7 +329,7 @@ def start(config: HomeServerConfig, args: argparse.Namespace) -> None: # command. async def run() -> None: - with LoggingContext(name="command"): + with LoggingContext(name="command", server_name=config.server.server_name): await _base.start(ss) await args.func(ss, args) @@ -342,5 +342,5 @@ def start(config: HomeServerConfig, args: argparse.Namespace) -> None: if __name__ == "__main__": homeserver_config, args = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config, args) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 351cf93b76..1a9b0ad155 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -26,7 +26,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index fbc85b2f34..7e8b47c20a 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -278,11 +278,13 @@ class GenericWorkerServer(HomeServer): self._listen_http(listener) elif listener.type == "manhole": if isinstance(listener, TCPListenerConfig): - _base.listen_manhole( - listener.bind_addresses, - listener.port, - manhole_settings=self.config.server.manhole_settings, - manhole_globals={"hs": self}, + self._listening_services.extend( + _base.listen_manhole( + listener.bind_addresses, + listener.port, + manhole_settings=self.config.server.manhole_settings, + manhole_globals={"hs": self}, + ) ) else: raise ConfigError( @@ -296,9 +298,11 @@ class GenericWorkerServer(HomeServer): ) else: if isinstance(listener, TCPListenerConfig): - _base.listen_metrics( - listener.bind_addresses, - listener.port, + self._metrics_listeners.extend( + _base.listen_metrics( + listener.bind_addresses, + listener.port, + ) ) else: raise ConfigError( @@ -386,7 +390,7 @@ def start(config: HomeServerConfig) -> None: def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 159075e886..3c691906ca 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -22,7 +22,7 @@ import logging import os import sys -from typing import Dict, Iterable, List +from typing import Dict, Iterable, List, Optional from twisted.internet.tcp import Port from twisted.web.resource import EncodingResourceWrapper, Resource @@ -70,6 +70,7 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.rest.well_known import well_known_resource from synapse.server import HomeServer from synapse.storage import DataStore +from synapse.types import ISynapseReactor from synapse.util.check_dependencies import VERSION, check_requirements from synapse.util.httpresourcetree import create_resource_tree from synapse.util.module_loader import load_module @@ -277,11 +278,13 @@ class SynapseHomeServer(HomeServer): ) elif listener.type == "manhole": if isinstance(listener, TCPListenerConfig): - _base.listen_manhole( - listener.bind_addresses, - listener.port, - manhole_settings=self.config.server.manhole_settings, - manhole_globals={"hs": self}, + self._listening_services.extend( + _base.listen_manhole( + listener.bind_addresses, + listener.port, + manhole_settings=self.config.server.manhole_settings, + manhole_globals={"hs": self}, + ) ) else: raise ConfigError( @@ -294,9 +297,11 @@ class SynapseHomeServer(HomeServer): ) else: if isinstance(listener, TCPListenerConfig): - _base.listen_metrics( - listener.bind_addresses, - listener.port, + self._metrics_listeners.extend( + _base.listen_metrics( + listener.bind_addresses, + listener.port, + ) ) else: raise ConfigError( @@ -340,12 +345,23 @@ def load_or_generate_config(argv_options: List[str]) -> HomeServerConfig: return config -def setup(config: HomeServerConfig) -> SynapseHomeServer: +def setup( + config: HomeServerConfig, + reactor: Optional[ISynapseReactor] = None, + freeze: bool = True, +) -> SynapseHomeServer: """ Create and setup a Synapse homeserver instance given a configuration. Args: config: The configuration for the homeserver. + reactor: Optionally provide a reactor to use. Can be useful in different + scenarios that you want control over the reactor, such as tests. + freeze: whether to freeze the homeserver base objects in the garbage collector. + May improve garbage collection performance by marking objects with an effectively + static lifetime as frozen so they don't need to be considered for cleanup. + If you ever want to `shutdown` the homeserver, this needs to be + False otherwise the homeserver cannot be garbage collected after `shutdown`. Returns: A homeserver instance. @@ -384,6 +400,7 @@ def setup(config: HomeServerConfig) -> SynapseHomeServer: config.server.server_name, config=config, version_string=f"Synapse/{VERSION}", + reactor=reactor, ) setup_logging(hs, config, use_worker_options=False) @@ -405,7 +422,7 @@ def setup(config: HomeServerConfig) -> SynapseHomeServer: # Loading the provider metadata also ensures the provider config is valid. await oidc.load_metadata() - await _base.start(hs) + await _base.start(hs, freeze) hs.get_datastores().main.db_pool.updates.start_doing_background_updates() @@ -429,7 +446,7 @@ def run(hs: HomeServer) -> None: def main() -> None: homeserver_config = load_or_generate_config(sys.argv[1:]) - with LoggingContext("main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): # check base requirements check_requirements() hs = setup(homeserver_config) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 69d3ac78fd..4bbc33cba2 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -29,19 +29,18 @@ from prometheus_client import Gauge from twisted.internet import defer from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.types import JsonDict -from synapse.util.constants import ONE_HOUR_SECONDS, ONE_MINUTE_SECONDS +from synapse.util.constants import ( + MILLISECONDS_PER_SECOND, + ONE_HOUR_SECONDS, + ONE_MINUTE_SECONDS, +) if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger("synapse.app.homeserver") -MILLISECONDS_PER_SECOND = 1000 - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS """ We wait 5 minutes to send the first set of stats as the server can be quite busy the @@ -85,8 +84,6 @@ def phone_stats_home( stats: JsonDict, stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process, ) -> "defer.Deferred[None]": - server_name = hs.hostname - async def _phone_stats_home( hs: "HomeServer", stats: JsonDict, @@ -200,8 +197,8 @@ def phone_stats_home( except Exception as e: logger.warning("Error reporting stats: %s", e) - return run_as_background_process( - "phone_stats_home", server_name, _phone_stats_home, hs, stats, stats_process + return hs.run_as_background_process( + "phone_stats_home", _phone_stats_home, hs, stats, stats_process ) @@ -263,9 +260,8 @@ def start_phone_stats_home(hs: "HomeServer") -> None: float(hs.config.server.max_mau_value) ) - return run_as_background_process( + return hs.run_as_background_process( "generate_monthly_active_users", - server_name, _generate_monthly_active_users, ) @@ -285,10 +281,16 @@ def start_phone_stats_home(hs: "HomeServer") -> None: # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process - clock.call_later(0, performance_stats_init) + clock.call_later( + 0, + performance_stats_init, + ) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later( - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, + phone_stats_home, + hs, + stats, ) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 95a253dbb9..823e1908b5 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index b6385381b4..f64d82e41f 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext def main() -> None: homeserver_config = load_config(sys.argv[1:]) - with LoggingContext(name="main"): + with LoggingContext(name="main", server_name=homeserver_config.server.server_name): start(homeserver_config) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 2d8d382e68..1d0735ca1d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -23,15 +23,33 @@ import logging import re from enum import Enum -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Pattern, Sequence +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Pattern, + Sequence, + cast, +) import attr from netaddr import IPSet +from twisted.internet import reactor + from synapse.api.constants import EventTypes from synapse.events import EventBase -from synapse.types import DeviceListUpdates, JsonDict, JsonMapping, UserID +from synapse.types import ( + DeviceListUpdates, + ISynapseThreadlessReactor, + JsonDict, + JsonMapping, + UserID, +) from synapse.util.caches.descriptors import _CacheContext, cached +from synapse.util.clock import Clock if TYPE_CHECKING: from synapse.appservice.api import ApplicationServiceApi @@ -98,6 +116,15 @@ class ApplicationService: self.sender = sender # The application service user should be part of the server's domain. self.server_name = sender.domain # nb must be called this for @cached + + # Ideally we would require passing in the `HomeServer` `Clock` instance. + # However this is not currently possible as there are places which use + # `@cached` that aren't aware of the `HomeServer` instance. + # nb must be called this for @cached + self.clock = Clock( + cast(ISynapseThreadlessReactor, reactor), server_name=self.server_name + ) # type: ignore[multiple-internal-clocks] + self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index c8678406a1..b4de759b67 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -81,7 +81,6 @@ from synapse.appservice import ( from synapse.appservice.api import ApplicationServiceApi from synapse.events import EventBase from synapse.logging.context import run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main import DataStore from synapse.types import DeviceListUpdates, JsonMapping from synapse.util.clock import Clock @@ -200,6 +199,7 @@ class _ServiceQueuer: ) self.server_name = hs.hostname self.clock = hs.get_clock() + self.hs = hs self._store = hs.get_datastores().main def start_background_request(self, service: ApplicationService) -> None: @@ -207,9 +207,7 @@ class _ServiceQueuer: if service.id in self.requests_in_flight: return - run_as_background_process( - "as-sender", self.server_name, self._send_request, service - ) + self.hs.run_as_background_process("as-sender", self._send_request, service) async def _send_request(self, service: ApplicationService) -> None: # sanity-check: we shouldn't get here if this service already has a sender @@ -361,6 +359,7 @@ class _TransactionController: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() + self.hs = hs self.store = hs.get_datastores().main self.as_api = hs.get_application_service_api() @@ -448,6 +447,7 @@ class _TransactionController: recoverer = self.RECOVERER_CLASS( self.server_name, self.clock, + self.hs, self.store, self.as_api, service, @@ -494,6 +494,7 @@ class _Recoverer: self, server_name: str, clock: Clock, + hs: "HomeServer", store: DataStore, as_api: ApplicationServiceApi, service: ApplicationService, @@ -501,6 +502,7 @@ class _Recoverer: ): self.server_name = server_name self.clock = clock + self.hs = hs self.store = store self.as_api = as_api self.service = service @@ -513,9 +515,8 @@ class _Recoverer: logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, - run_as_background_process, + self.hs.run_as_background_process, "as-recoverer", - self.server_name, self.retry, ) @@ -535,9 +536,8 @@ class _Recoverer: if self.scheduled_recovery: self.clock.cancel_call_later(self.scheduled_recovery) # Run a retry, which will resechedule a recovery if it fails. - run_as_background_process( + self.hs.run_as_background_process( "retry", - self.server_name, self.retry, ) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index f3b6e9f887..6de4c12c96 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -601,7 +601,7 @@ class RootConfig: @classmethod def load_config_with_parser( - cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv: List[str] + cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv_options: List[str] ) -> Tuple[TRootConfig, argparse.Namespace]: """Parse the commandline and config files with the given parser @@ -611,14 +611,14 @@ class RootConfig: Args: parser - argv + argv_options: The options passed to Synapse. Usually `sys.argv[1:]`. Returns: Returns the parsed config object and the parsed argparse.Namespace object from parser.parse_args(..)` """ - config_args = parser.parse_args(argv) + config_args = parser.parse_args(argv_options) config_files = find_config_files(search_paths=config_args.config_path) obj = cls(config_files) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 3f86ec1169..9dde4c4003 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -40,7 +40,6 @@ from twisted.logger import ( ) from synapse.logging.context import LoggingContextFilter -from synapse.logging.filter import MetadataFilter from synapse.synapse_rust import reset_logging_config from synapse.types import JsonDict @@ -213,13 +212,11 @@ def _setup_stdlib_logging( # writes. log_context_filter = LoggingContextFilter() - log_metadata_filter = MetadataFilter({"server_name": config.server.server_name}) old_factory = logging.getLogRecordFactory() def factory(*args: Any, **kwargs: Any) -> logging.LogRecord: record = old_factory(*args, **kwargs) log_context_filter.filter(record) - log_metadata_filter.filter(record) return record logging.setLogRecordFactory(factory) @@ -348,7 +345,9 @@ def setup_logging( # Add a SIGHUP handler to reload the logging configuration, if one is available. from synapse.app import _base as appbase - appbase.register_sighup(_reload_logging_config, log_config_path) + appbase.register_sighup( + hs.get_instance_id(), _reload_logging_config, log_config_path + ) # Log immediately so we can grep backwards. logger.warning("***** STARTING SERVER *****") diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 8c59772e56..258bc29357 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -172,7 +172,7 @@ class Keyring: _FetchKeyRequest, Dict[str, Dict[str, FetchKeyResult]] ] = BatchingQueue( name="keyring_server", - server_name=self.server_name, + hs=hs, clock=hs.get_clock(), # The method called to fetch each key process_batch_callback=self._inner_fetch_key_requests, @@ -194,6 +194,14 @@ class Keyring: valid_until_ts=2**63, # fake future timestamp ) + def shutdown(self) -> None: + """ + Prepares the KeyRing for garbage collection by shutting down it's queues. + """ + self._fetch_keys_queue.shutdown() + for key_fetcher in self._key_fetchers: + key_fetcher.shutdown() + async def verify_json_for_server( self, server_name: str, @@ -316,7 +324,7 @@ class Keyring: if key_result.valid_until_ts < verify_request.minimum_valid_until_ts: continue - await self._process_json(key_result.verify_key, verify_request) + await self.process_json(key_result.verify_key, verify_request) verified = True if not verified: @@ -326,7 +334,7 @@ class Keyring: Codes.UNAUTHORIZED, ) - async def _process_json( + async def process_json( self, verify_key: VerifyKey, verify_request: VerifyJsonRequest ) -> None: """Processes the `VerifyJsonRequest`. Raises if the signature can't be @@ -479,11 +487,17 @@ class KeyFetcher(metaclass=abc.ABCMeta): self.server_name = hs.hostname self._queue = BatchingQueue( name=self.__class__.__name__, - server_name=self.server_name, + hs=hs, clock=hs.get_clock(), process_batch_callback=self._fetch_keys, ) + def shutdown(self) -> None: + """ + Prepares the KeyFetcher for garbage collection by shutting down it's queue. + """ + self._queue.shutdown() + async def get_keys( self, server_name: str, key_ids: List[str], minimum_valid_until_ts: int ) -> Dict[str, FetchKeyResult]: diff --git a/synapse/events/auto_accept_invites.py b/synapse/events/auto_accept_invites.py index 6873ee9d31..9e17edd227 100644 --- a/synapse/events/auto_accept_invites.py +++ b/synapse/events/auto_accept_invites.py @@ -119,7 +119,6 @@ class InviteAutoAccepter: event.state_key, event.room_id, "join", - bg_start_span=False, ) if is_direct_message: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 542d9650d4..8c91336dbc 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -148,6 +148,7 @@ class FederationClient(FederationBase): self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( cache_name="get_pdu_cache", server_name=self.server_name, + hs=self.hs, clock=self._clock, max_len=1000, expiry_ms=120 * 1000, @@ -167,6 +168,7 @@ class FederationClient(FederationBase): ] = ExpiringCache( cache_name="get_room_hierarchy_cache", server_name=self.server_name, + hs=self.hs, clock=self._clock, max_len=1000, expiry_ms=5 * 60 * 1000, @@ -495,6 +497,43 @@ class FederationClient(FederationBase): ) return RECOMMENDATION_OK + @trace + @tag_args + async def ask_policy_server_to_sign_event( + self, destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> Optional[JsonDict]: + """Requests that the destination server (typically a policy server) + sign the event as not spam. + + If the policy server could not be contacted or the policy server + returned an error, this returns no signature. + + Args: + destination: The remote homeserver to ask (a policy server) + pdu: The event to sign + timeout: How long to try (in ms) the destination for before + giving up. None indicates no timeout. + Returns: + The signature from the policy server, structured in the same was as the 'signatures' + JSON in the event e.g { "$policy_server_via_domain" : { "ed25519:policy_server": "signature_base64" }} + """ + logger.debug( + "ask_policy_server_to_sign_event for event_id=%s from %s", + pdu.event_id, + destination, + ) + try: + return await self.transport_layer.ask_policy_server_to_sign_event( + destination, pdu, timeout=timeout + ) + except Exception as e: + logger.warning( + "ask_policy_server_to_sign_event: server %s responded with error: %s", + destination, + e, + ) + return None + @trace @tag_args async def get_pdu( diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a8d5c3c280..e7da8fda0d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,7 @@ class FederationServer(FederationBase): # with FederationHandlerRegistry. hs.get_directory_handler() - self._server_linearizer = Linearizer("fed_server") + self._server_linearizer = Linearizer(name="fed_server", clock=hs.get_clock()) # origins that we are currently processing a transaction from. # a dict from origin to txn id. diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 2fdee9ac54..759df9836b 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -144,6 +144,9 @@ class FederationRemoteSendQueue(AbstractFederationSender): self.clock.looping_call(self._clear_queue, 30 * 1000) + def shutdown(self) -> None: + """Stops this federation sender instance from sending further transactions.""" + def _next_pos(self) -> int: pos = self.pos self.pos += 1 diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 8e3619d1bc..4410ffc5c5 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -168,7 +168,6 @@ from synapse.metrics import ( events_processed_counter, ) from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.types import ( @@ -232,6 +231,11 @@ WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5 class AbstractFederationSender(metaclass=abc.ABCMeta): + @abc.abstractmethod + def shutdown(self) -> None: + """Stops this federation sender instance from sending further transactions.""" + raise NotImplementedError() + @abc.abstractmethod def notify_new_events(self, max_token: RoomStreamToken) -> None: """This gets called when we have some new events we might want to @@ -326,6 +330,7 @@ class _DestinationWakeupQueue: _MAX_TIME_IN_QUEUE = 30.0 sender: "FederationSender" = attr.ib() + hs: "HomeServer" = attr.ib() server_name: str = attr.ib() """ Our homeserver name (used to label metrics) (`hs.hostname`). @@ -453,18 +458,30 @@ class FederationSender(AbstractFederationSender): 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second ) self._destination_wakeup_queue = _DestinationWakeupQueue( - self, self.server_name, self.clock, max_delay_s=rr_txn_interval_per_room_s + self, + hs, + self.server_name, + self.clock, + max_delay_s=rr_txn_interval_per_room_s, ) + # It is important for `_is_shutdown` to be instantiated before the looping call + # for `wake_destinations_needing_catchup`. + self._is_shutdown = False + # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( - run_as_background_process, + self.hs.run_as_background_process, WAKEUP_RETRY_PERIOD_SEC * 1000.0, "wake_destinations_needing_catchup", - self.server_name, self._wake_destinations_needing_catchup, ) + def shutdown(self) -> None: + self._is_shutdown = True + for queue in self._per_destination_queues.values(): + queue.shutdown() + def _get_per_destination_queue( self, destination: str ) -> Optional[PerDestinationQueue]: @@ -503,16 +520,15 @@ class FederationSender(AbstractFederationSender): return # fire off a processing loop in the background - run_as_background_process( + self.hs.run_as_background_process( "process_event_queue_for_federation", - self.server_name, self._process_event_queue_loop, ) async def _process_event_queue_loop(self) -> None: try: self._is_processing = True - while True: + while not self._is_shutdown: last_token = await self.store.get_federation_out_pos("events") ( next_token, @@ -1123,7 +1139,7 @@ class FederationSender(AbstractFederationSender): last_processed: Optional[str] = None - while True: + while not self._is_shutdown: destinations_to_wake = ( await self.store.get_catch_up_outstanding_destinations(last_processed) ) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 4c844d403a..845af92fac 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -28,6 +28,8 @@ from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Tupl import attr from prometheus_client import Counter +from twisted.internet import defer + from synapse.api.constants import EduTypes from synapse.api.errors import ( FederationDeniedError, @@ -41,7 +43,6 @@ from synapse.handlers.presence import format_user_presence_state from synapse.logging import issue9533_logger from synapse.logging.opentracing import SynapseTags, set_tag from synapse.metrics import SERVER_NAME_LABEL, sent_transactions_counter -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, ReadReceipt from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter from synapse.visibility import filter_events_for_server @@ -79,6 +80,7 @@ MAX_PRESENCE_STATES_PER_EDU = 50 class PerDestinationQueue: """ Manages the per-destination transmission queues. + Runs until `shutdown()` is called on the queue. Args: hs @@ -94,6 +96,7 @@ class PerDestinationQueue: destination: str, ): self.server_name = hs.hostname + self._hs = hs self._clock = hs.get_clock() self._storage_controllers = hs.get_storage_controllers() self._store = hs.get_datastores().main @@ -117,6 +120,8 @@ class PerDestinationQueue: self._destination = destination self.transmission_loop_running = False + self._transmission_loop_enabled = True + self.active_transmission_loop: Optional[defer.Deferred] = None # Flag to signal to any running transmission loop that there is new data # queued up to be sent. @@ -171,6 +176,20 @@ class PerDestinationQueue: def __str__(self) -> str: return "PerDestinationQueue[%s]" % self._destination + def shutdown(self) -> None: + """Instruct the queue to stop processing any further requests""" + self._transmission_loop_enabled = False + # The transaction manager must be shutdown before cancelling the active + # transmission loop. Otherwise the transmission loop can enter a new cycle of + # sleeping before retrying since the shutdown flag of the _transaction_manager + # hasn't been set yet. + self._transaction_manager.shutdown() + try: + if self.active_transmission_loop is not None: + self.active_transmission_loop.cancel() + except Exception: + pass + def pending_pdu_count(self) -> int: return len(self._pending_pdus) @@ -309,11 +328,14 @@ class PerDestinationQueue: ) return + if not self._transmission_loop_enabled: + logger.warning("Shutdown has been requested. Not sending transaction") + return + logger.debug("TX [%s] Starting transaction loop", self._destination) - run_as_background_process( + self.active_transmission_loop = self._hs.run_as_background_process( "federation_transaction_transmission_loop", - self.server_name, self._transaction_transmission_loop, ) @@ -321,13 +343,13 @@ class PerDestinationQueue: pending_pdus: List[EventBase] = [] try: self.transmission_loop_running = True - # This will throw if we wouldn't retry. We do this here so we fail # quickly, but we will later check this again in the http client, # hence why we throw the result away. await get_retry_limiter( destination=self._destination, our_server_name=self.server_name, + hs=self._hs, clock=self._clock, store=self._store, ) @@ -339,7 +361,7 @@ class PerDestinationQueue: # not caught up yet return - while True: + while self._transmission_loop_enabled: self._new_data_to_send = False async with _TransactionQueueManager(self) as ( @@ -352,8 +374,8 @@ class PerDestinationQueue: # If we've gotten told about new things to send during # checking for things to send, we try looking again. # Otherwise new PDUs or EDUs might arrive in the meantime, - # but not get sent because we hold the - # `transmission_loop_running` flag. + # but not get sent because we currently have an + # `_active_transmission_loop` running. if self._new_data_to_send: continue else: @@ -442,6 +464,7 @@ class PerDestinationQueue: ) finally: # We want to be *very* sure we clear this after we stop processing + self.active_transmission_loop = None self.transmission_loop_running = False async def _catch_up_transmission_loop(self) -> None: @@ -469,7 +492,7 @@ class PerDestinationQueue: last_successful_stream_ordering: int = _tmp_last_successful_stream_ordering # get at most 50 catchup room/PDUs - while True: + while self._transmission_loop_enabled: event_ids = await self._store.get_catch_up_room_event_ids( self._destination, last_successful_stream_ordering ) diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index b548d9ed70..f47c011487 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -72,6 +72,12 @@ class TransactionManager: # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) + self._is_shutdown = False + + def shutdown(self) -> None: + self._is_shutdown = True + self._transport_layer.shutdown() + @measure_func("_send_new_transaction") async def send_new_transaction( self, @@ -86,6 +92,12 @@ class TransactionManager: edus: List of EDUs to send """ + if self._is_shutdown: + logger.warning( + "TransactionManager has been shutdown, not sending transaction" + ) + return + # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 62bf96ce91..02e56e8e27 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -70,6 +70,9 @@ class TransportLayerClient: self.client = hs.get_federation_http_client() self._is_mine_server_name = hs.is_mine_server_name + def shutdown(self) -> None: + self.client.shutdown() + async def get_room_state_ids( self, destination: str, room_id: str, event_id: str ) -> JsonDict: @@ -170,6 +173,32 @@ class TransportLayerClient: timeout=timeout, ) + async def ask_policy_server_to_sign_event( + self, destination: str, event: EventBase, timeout: Optional[int] = None + ) -> JsonDict: + """Requests that the destination server (typically a policy server) + sign the event as not spam. + + If the policy server could not be contacted or the policy server + returned an error, this raises that error. + + Args: + destination: The host name of the policy server / homeserver. + event: The event to sign. + timeout: How long to try (in ms) the destination for before giving up. + None indicates no timeout. + Returns: + The signature from the policy server, structured in the same was as the 'signatures' + JSON in the event e.g { "$policy_server_via_domain" : { "ed25519:policy_server": "signature_base64" }} + """ + return await self.client.post_json( + destination=destination, + path="/_matrix/policy/unstable/org.matrix.msc4284/sign", + data=event.get_pdu_json(), + ignore_backoff=True, + timeout=timeout, + ) + async def backfill( self, destination: str, room_id: str, event_tuples: Collection[str], limit: int ) -> Optional[Union[JsonDict, list]]: diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 39a22b8cbb..eed50ef69a 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -37,10 +37,8 @@ logger = logging.getLogger(__name__) class AccountValidityHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.config = hs.config self.store = hs.get_datastores().main self.send_email_handler = hs.get_send_email_handler() diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5bd239e5fe..6536d9fe51 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -47,7 +47,6 @@ from synapse.metrics import ( event_processing_loop_room_count, ) from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.storage.databases.main.directory import RoomAliasMapping @@ -76,9 +75,8 @@ events_processed_counter = Counter( class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() @@ -98,7 +96,7 @@ class ApplicationServicesHandler: self.is_processing = False self._ephemeral_events_linearizer = Linearizer( - name="appservice_ephemeral_events" + name="appservice_ephemeral_events", clock=hs.get_clock() ) def notify_interested_services(self, max_token: RoomStreamToken) -> None: @@ -171,8 +169,8 @@ class ApplicationServicesHandler: except Exception: logger.error("Application Services Failure") - run_as_background_process( - "as_scheduler", self.server_name, start_scheduler + self.hs.run_as_background_process( + "as_scheduler", start_scheduler ) self.started_scheduler = True diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index c0684380a7..204dffd288 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership from synapse.api.errors import SynapseError -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.deactivate_account import ( ReplicationNotifyAccountDeactivatedServlet, ) @@ -272,8 +271,8 @@ class DeactivateAccountHandler: pending deactivation, if it isn't already running. """ if not self._user_parter_running: - run_as_background_process( - "user_parter_loop", self.server_name, self._user_parter_loop + self.hs.run_as_background_process( + "user_parter_loop", self._user_parter_loop ) async def _user_parter_loop(self) -> None: diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index d47e3fd263..79dd3e8416 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -24,9 +24,6 @@ from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.logging.context import make_deferred_yieldable from synapse.logging.opentracing import set_tag from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.replication.http.delayed_events import ( ReplicationAddedDelayedEventRestServlet, ) @@ -58,6 +55,7 @@ logger = logging.getLogger(__name__) class DelayedEventsHandler: def __init__(self, hs: "HomeServer"): + self.hs = hs self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -94,7 +92,10 @@ class DelayedEventsHandler: hs.get_notifier().add_replication_callback(self.notify_new_event) # Kick off again (without blocking) to catch any missed notifications # that may have fired before the callback was added. - self._clock.call_later(0, self.notify_new_event) + self._clock.call_later( + 0, + self.notify_new_event, + ) # Delayed events that are already marked as processed on startup might not have been # sent properly on the last run of the server, so unmark them to send them again. @@ -112,15 +113,14 @@ class DelayedEventsHandler: self._schedule_next_at(next_send_ts) # Can send the events in background after having awaited on marking them as processed - run_as_background_process( + self.hs.run_as_background_process( "_send_events", - self.server_name, self._send_events, events, ) - self._initialized_from_db = run_as_background_process( - "_schedule_db_events", self.server_name, _schedule_db_events + self._initialized_from_db = self.hs.run_as_background_process( + "_schedule_db_events", _schedule_db_events ) else: self._repl_client = ReplicationAddedDelayedEventRestServlet.make_client(hs) @@ -145,9 +145,7 @@ class DelayedEventsHandler: finally: self._event_processing = False - run_as_background_process( - "delayed_events.notify_new_event", self.server_name, process - ) + self.hs.run_as_background_process("delayed_events.notify_new_event", process) async def _unsafe_process_new_event(self) -> None: # We purposefully fetch the current max room stream ordering before @@ -542,9 +540,8 @@ class DelayedEventsHandler: if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( delay_sec, - run_as_background_process, + self.hs.run_as_background_process, "_send_on_timeout", - self.server_name, self._send_on_timeout, ) else: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 65b8a95f8e..c6024597b7 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -47,7 +47,6 @@ from synapse.api.errors import ( ) from synapse.logging.opentracing import log_kv, set_tag, trace from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.http.devices import ( @@ -125,7 +124,7 @@ class DeviceHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func - self.hs = hs + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() @@ -191,10 +190,9 @@ class DeviceHandler: and self._delete_stale_devices_after is not None ): self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, DELETE_STALE_DEVICES_INTERVAL_MS, desc="delete_stale_devices", - server_name=self.server_name, func=self._delete_stale_devices, ) @@ -963,10 +961,9 @@ class DeviceWriterHandler(DeviceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname # nb must be called this for @measure_func + self.hs = hs # nb must be called this for @wrap_as_background_process - self.server_name = ( - hs.hostname - ) # nb must be called this for @measure_func and @wrap_as_background_process # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by # `synapse.app.generic_worker.FederationSenderHandler` when it sees it @@ -1444,14 +1441,18 @@ class DeviceListUpdater(DeviceListWorkerUpdater): def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): super().__init__(hs) - self.server_name = hs.hostname + self.hs = hs self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_device_list") - self._resync_linearizer = Linearizer(name="remote_device_resync") + self._remote_edu_linearizer = Linearizer( + name="remote_device_list", clock=self.clock + ) + self._resync_linearizer = Linearizer( + name="remote_device_resync", clock=self.clock + ) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[ @@ -1464,6 +1465,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( cache_name="device_update_edu", server_name=self.server_name, + hs=self.hs, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -1473,9 +1475,8 @@ class DeviceListUpdater(DeviceListWorkerUpdater): # Attempt to resync out of sync device lists every 30s. self._resync_retry_lock = Lock() self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, 30 * 1000, - server_name=self.server_name, func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", ) @@ -1595,9 +1596,8 @@ class DeviceListUpdater(DeviceListWorkerUpdater): if resync: # We mark as stale up front in case we get restarted. await self.store.mark_remote_users_device_caches_as_stale([user_id]) - run_as_background_process( + self.hs.run_as_background_process( "_maybe_retry_device_resync", - self.server_name, self.multi_user_device_resync, [user_id], False, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index fa3d207a90..791a0fa684 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -112,8 +112,7 @@ class E2eKeysHandler: # Limit the number of in-flight requests from a single device. self._query_devices_linearizer = Linearizer( - name="query_devices", - max_count=10, + name="query_devices", max_count=10, clock=hs.get_clock() ) self._query_appservices_for_otks = ( @@ -873,9 +872,7 @@ class E2eKeysHandler: log_kv( {"message": "Did not update one_time_keys", "reason": "no keys given"} ) - fallback_keys = keys.get("fallback_keys") or keys.get( - "org.matrix.msc2732.fallback_keys" - ) + fallback_keys = keys.get("fallback_keys") if fallback_keys and isinstance(fallback_keys, dict): log_kv( { @@ -1765,7 +1762,9 @@ class SigningKeyEduUpdater: assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_signing_key") + self._remote_edu_linearizer = Linearizer( + name="remote_signing_key", clock=self.clock + ) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 34aae7ef3c..adc20f4ad0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -72,7 +72,6 @@ from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import nested_logging_context from synapse.logging.opentracing import SynapseTags, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.module_api import NOT_SPAM from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.invite_rule import InviteRule @@ -160,7 +159,7 @@ class FederationHandler: self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._room_backfill = Linearizer("room_backfill") + self._room_backfill = Linearizer(name="room_backfill", clock=self.clock) self._third_party_event_rules = ( hs.get_module_api_callbacks().third_party_event_rules @@ -180,16 +179,16 @@ class FederationHandler: # When the lock is held for a given room, no other concurrent code may # partial state or un-partial state the room. self._is_partial_state_room_linearizer = Linearizer( - name="_is_partial_state_room_linearizer" + name="_is_partial_state_room_linearizer", + clock=self.clock, ) # if this is the main process, fire off a background process to resume # any partial-state-resync operations which were in flight when we # were shut down. if not hs.config.worker.worker_app: - run_as_background_process( + self.hs.run_as_background_process( "resume_sync_partial_state_room", - self.server_name, self._resume_partial_state_room_sync, ) @@ -317,9 +316,8 @@ class FederationHandler: logger.debug( "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." ) - run_as_background_process( + self.hs.run_as_background_process( "_maybe_backfill_inner_anyway_with_max_depth", - self.server_name, self.maybe_backfill, room_id=room_id, # We use `MAX_DEPTH` so that we find all backfill points next @@ -801,9 +799,8 @@ class FederationHandler: # lots of requests for missing prev_events which we do actually # have. Hence we fire off the background task, but don't wait for it. - run_as_background_process( + self.hs.run_as_background_process( "handle_queued_pdus", - self.server_name, self._handle_queued_pdus, room_queue, ) @@ -1876,9 +1873,8 @@ class FederationHandler: room_id=room_id, ) - run_as_background_process( + self.hs.run_as_background_process( desc="sync_partial_state_room", - server_name=self.server_name, func=_sync_partial_state_room_wrapper, ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 1e47b4ef4f..d6390b79c7 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -81,7 +81,6 @@ from synapse.logging.opentracing import ( trace, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.federation import ( ReplicationFederationSendEventsRestServlet, ) @@ -153,6 +152,7 @@ class FederationEventHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self._clock = hs.get_clock() self._store = hs.get_datastores().main self._state_store = hs.get_datastores().state @@ -175,6 +175,7 @@ class FederationEventHandler: ) self._notifier = hs.get_notifier() + self._server_name = hs.hostname self._is_mine_id = hs.is_mine_id self._is_mine_server_name = hs.is_mine_server_name self._instance_name = hs.get_instance_name() @@ -191,7 +192,7 @@ class FederationEventHandler: # federation event staging area. self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {} - self._room_pdu_linearizer = Linearizer("fed_room_pdu") + self._room_pdu_linearizer = Linearizer(name="fed_room_pdu", clock=self._clock) async def on_receive_pdu(self, origin: str, pdu: EventBase) -> None: """Process a PDU received via a federation /send/ transaction @@ -974,9 +975,8 @@ class FederationEventHandler: # Process previously failed backfill events in the background to not waste # time on something that is likely to fail again. if len(events_with_failed_pull_attempts) > 0: - run_as_background_process( + self.hs.run_as_background_process( "_process_new_pulled_events_with_failed_pull_attempts", - self.server_name, _process_new_pulled_events, events_with_failed_pull_attempts, ) @@ -1568,9 +1568,8 @@ class FederationEventHandler: resync = True if resync: - run_as_background_process( + self.hs.run_as_background_process( "resync_device_due_to_pdu", - self.server_name, self._resync_device, event.sender, ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c8c86d8749..e874b60000 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -67,7 +67,6 @@ from synapse.handlers.directory import DirectoryHandler from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.send_events import ReplicationSendEventsRestServlet from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -99,6 +98,7 @@ class MessageHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.state = hs.get_state_handler() @@ -113,8 +113,8 @@ class MessageHandler: self._scheduled_expiry: Optional[IDelayedCall] = None if not hs.config.worker.worker_app: - run_as_background_process( - "_schedule_next_expiry", self.server_name, self._schedule_next_expiry + self.hs.run_as_background_process( + "_schedule_next_expiry", self._schedule_next_expiry ) async def get_room_data( @@ -444,9 +444,8 @@ class MessageHandler: self._scheduled_expiry = self.clock.call_later( delay, - run_as_background_process, + self.hs.run_as_background_process, "_expire_event", - self.server_name, self._expire_event, event_id, ) @@ -513,7 +512,9 @@ class EventCreationHandler: # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room - self.limiter = Linearizer(max_count=1, name="room_event_creation_limit") + self.limiter = Linearizer( + max_count=1, name="room_event_creation_limit", clock=self.clock + ) self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() @@ -546,9 +547,8 @@ class EventCreationHandler: and self.config.server.cleanup_extremities_with_dummy_events ): self.clock.looping_call( - lambda: run_as_background_process( + lambda: self.hs.run_as_background_process( "send_dummy_events_to_fill_extremities", - self.server_name, self._send_dummy_events_to_fill_extremities, ), 5 * 60 * 1000, @@ -568,6 +568,7 @@ class EventCreationHandler: self._external_cache_joined_hosts_updates = ExpiringCache( cache_name="_external_cache_joined_hosts_updates", server_name=self.server_name, + hs=self.hs, clock=self.clock, expiry_ms=30 * 60 * 1000, ) @@ -1138,6 +1139,12 @@ class EventCreationHandler: assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % ( event.sender, ) + # if this room uses a policy server, try to get a signature now. + # We use verify=False here as we are about to call is_event_allowed on the same event + # which will do sig checks. + await self._policy_handler.ask_policy_server_to_sign_event( + event, verify=False + ) policy_allowed = await self._policy_handler.is_event_allowed(event) if not policy_allowed: @@ -2105,9 +2112,8 @@ class EventCreationHandler: if event.type == EventTypes.Message: # We don't want to block sending messages on any presence code. This # matters as sometimes presence code can take a while. - run_as_background_process( + self.hs.run_as_background_process( "bump_presence_active_time", - self.server_name, self._bump_active_time, requester.user, requester.device_id, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index df1a7e714c..02a67581e7 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -29,7 +29,6 @@ from synapse.api.filtering import Filter from synapse.events.utils import SerializeEventConfig from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.admin._base import assert_user_is_admin from synapse.streams.config import PaginationConfig from synapse.types import ( @@ -116,10 +115,9 @@ class PaginationHandler: logger.info("Setting up purge job with config: %s", job) self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, job.interval, "purge_history_for_rooms_in_range", - self.server_name, self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, job.longest_max_lifetime, @@ -244,9 +242,8 @@ class PaginationHandler: # We want to purge everything, including local events, and to run the purge in # the background so that it's not blocking any other operation apart from # other purges in the same room. - run_as_background_process( + self.hs.run_as_background_process( PURGE_HISTORY_ACTION_NAME, - self.server_name, self.purge_history, room_id, token, @@ -604,9 +601,8 @@ class PaginationHandler: # Otherwise, we can backfill in the background for eventual # consistency's sake but we don't need to block the client waiting # for a costly federation call and processing. - run_as_background_process( + self.hs.run_as_background_process( "maybe_backfill_in_the_background", - self.server_name, self.hs.get_federation_handler().maybe_backfill, room_id, curr_topo, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 63cdf66198..1610683066 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -107,7 +107,6 @@ from synapse.events.presence_router import PresenceRouter from synapse.logging.context import run_in_background from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.http.presence import ( @@ -537,19 +536,15 @@ class WorkerPresenceHandler(BasePresenceHandler): self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self._send_stop_syncing_loop = self.clock.looping_call( - self.send_stop_syncing, UPDATE_SYNCING_USERS_MS - ) - - hs.get_clock().add_system_event_trigger( - "before", - "shutdown", - run_as_background_process, - "generic_presence.on_shutdown", - self.server_name, - self._on_shutdown, + self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS_MS) + + hs.register_async_shutdown_handler( + phase="before", + eventType="shutdown", + shutdown_func=self._on_shutdown, ) + @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: if self._track_presence: self.hs.get_replication_command_handler().send_command( @@ -779,9 +774,7 @@ class WorkerPresenceHandler(BasePresenceHandler): class PresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.wheel_timer: WheelTimer[str] = WheelTimer() self.notifier = hs.get_notifier() @@ -842,13 +835,10 @@ class PresenceHandler(BasePresenceHandler): # have not yet been persisted self.unpersisted_users_changes: Set[str] = set() - hs.get_clock().add_system_event_trigger( - "before", - "shutdown", - run_as_background_process, - "presence.on_shutdown", - self.server_name, - self._on_shutdown, + hs.register_async_shutdown_handler( + phase="before", + eventType="shutdown", + shutdown_func=self._on_shutdown, ) # Keeps track of the number of *ongoing* syncs on this process. While @@ -872,14 +862,19 @@ class PresenceHandler(BasePresenceHandler): ] = {} self.external_process_last_updated_ms: Dict[str, int] = {} - self.external_sync_linearizer = Linearizer(name="external_sync_linearizer") + self.external_sync_linearizer = Linearizer( + name="external_sync_linearizer", clock=self.clock + ) if self._track_presence: # Start a LoopingCall in 30s that fires every 5s. # The initial delay is to allow disconnected clients a chance to # reconnect before we treat them as offline. self.clock.call_later( - 30, self.clock.looping_call, self._handle_timeouts, 5000 + 30, + self.clock.looping_call, + self._handle_timeouts, + 5000, ) # Presence information is persisted, whether or not it is being tracked @@ -906,6 +901,7 @@ class PresenceHandler(BasePresenceHandler): self._event_pos = self.store.get_room_max_stream_ordering() self._event_processing = False + @wrap_as_background_process("PresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: """Gets called when shutting down. This lets us persist any updates that we haven't yet persisted, e.g. updates that only changes some internal @@ -1537,8 +1533,8 @@ class PresenceHandler(BasePresenceHandler): finally: self._event_processing = False - run_as_background_process( - "presence.notify_new_event", self.server_name, _process_presence + self.hs.run_as_background_process( + "presence.notify_new_event", _process_presence ) async def _unsafe_process(self) -> None: diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dbff28e7fb..9dda89d85b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -56,8 +56,8 @@ class ProfileHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() # nb must be called this for @cached self.store = hs.get_datastores().main - self.clock = hs.get_clock() self.hs = hs self.federation = hs.get_federation_client() diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index fb39c8e04b..85d2dd62bb 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,7 +36,9 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer(name="read_marker") + self.read_marker_linearizer = Linearizer( + name="read_marker", clock=hs.get_clock() + ) async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 5761a7f70b..c3ff0cfaf8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,7 +23,14 @@ """Contains functions for registering clients.""" import logging -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict +from typing import ( + TYPE_CHECKING, + Iterable, + List, + Optional, + Tuple, + TypedDict, +) from prometheus_client import Counter diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 47bd139ca7..db6dc5efd0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -597,7 +597,7 @@ class RoomCreationHandler: new_room_version, additional_creators=additional_creators, ) - initial_state = {} + initial_state: MutableStateMap = {} # Replicate relevant room events types_to_copy: List[Tuple[str, Optional[str]]] = [ @@ -693,14 +693,23 @@ class RoomCreationHandler: additional_creators, ) - # We construct what the body of a call to /createRoom would look like for passing - # to the spam checker. We don't include a preset here, as we expect the + # We construct a subset of what the body of a call to /createRoom would look like + # for passing to the spam checker. We don't include a preset here, as we expect the # initial state to contain everything we need. + # TODO: given we are upgrading, it would make sense to pass the room_version + # TODO: the preset might be useful too spam_check = await self._spam_checker_module_callbacks.user_may_create_room( user_id, { "creation_content": creation_content, - "initial_state": list(initial_state.items()), + "initial_state": [ + { + "type": state_key[0], + "state_key": state_key[1], + "content": event_content, + } + for state_key, event_content in initial_state.items() + ], }, ) if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5ba64912c9..2ab9b70f8c 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -50,7 +50,6 @@ from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.invite_rule import InviteRule @@ -114,8 +113,12 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): if self.hs.config.server.include_profile_data_on_invite: self._membership_types_to_include_profile_data_in.add(Membership.INVITE) - self.member_linearizer: Linearizer = Linearizer(name="member") - self.member_as_limiter = Linearizer(max_count=10, name="member_as_limiter") + self.member_linearizer: Linearizer = Linearizer( + name="member", clock=hs.get_clock() + ) + self.member_as_limiter = Linearizer( + max_count=10, name="member_as_limiter", clock=hs.get_clock() + ) self.clock = hs.get_clock() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker @@ -2186,7 +2189,10 @@ class RoomForgetterHandler(StateDeltasHandler): self._notifier.add_replication_callback(self.notify_new_event) # We kick this off to pick up outstanding work from before the last restart. - self._clock.call_later(0, self.notify_new_event) + self._clock.call_later( + 0, + self.notify_new_event, + ) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" @@ -2201,9 +2207,7 @@ class RoomForgetterHandler(StateDeltasHandler): finally: self._is_processing = False - run_as_background_process( - "room_forgetter.notify_new_event", self.server_name, process - ) + self._hs.run_as_background_process("room_forgetter.notify_new_event", process) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 170c477d6f..0663a36714 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -17,6 +17,11 @@ import logging from typing import TYPE_CHECKING +from signedjson.key import decode_verify_key_bytes +from unpaddedbase64 import decode_base64 + +from synapse.api.errors import SynapseError +from synapse.crypto.keyring import VerifyJsonRequest from synapse.events import EventBase from synapse.types.handlers.policy_server import RECOMMENDATION_OK from synapse.util.stringutils import parse_and_validate_server_name @@ -26,6 +31,9 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +POLICY_SERVER_EVENT_TYPE = "org.matrix.msc4284.policy" +POLICY_SERVER_KEY_ID = "ed25519:policy_server" + class RoomPolicyHandler: def __init__(self, hs: "HomeServer"): @@ -54,11 +62,11 @@ class RoomPolicyHandler: Returns: bool: True if the event is allowed in the room, False otherwise. """ - if event.type == "org.matrix.msc4284.policy" and event.state_key is not None: + if event.type == POLICY_SERVER_EVENT_TYPE and event.state_key is not None: return True # always allow policy server change events policy_event = await self._storage_controllers.state.get_current_state_event( - event.room_id, "org.matrix.msc4284.policy", "" + event.room_id, POLICY_SERVER_EVENT_TYPE, "" ) if not policy_event: return True # no policy server == default allow @@ -81,6 +89,22 @@ class RoomPolicyHandler: if not is_in_room: return True # policy server not in room == default allow + # Check if the event has been signed with the public key in the policy server state event. + # If it is, we can save an HTTP hit. + # We actually want to get the policy server state event BEFORE THE EVENT rather than + # the current state value, else changing the public key will cause all of these checks to fail. + # However, if we are checking outlier events (which we will due to is_event_allowed being called + # near the edges at _check_sigs_and_hash) we won't know the state before the event, so the + # only safe option is to use the current state + public_key = policy_event.content.get("public_key", None) + if public_key is not None and isinstance(public_key, str): + valid = await self._verify_policy_server_signature( + event, policy_server, public_key + ) + if valid: + return True + # fallthrough to hit /check manually + # At this point, the server appears valid and is in the room, so ask it to check # the event. recommendation = await self._federation_client.get_pdu_policy_recommendation( @@ -90,3 +114,73 @@ class RoomPolicyHandler: return False return True # default allow + + async def _verify_policy_server_signature( + self, event: EventBase, policy_server: str, public_key: str + ) -> bool: + # check the event is signed with this (via, public_key). + verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) + try: + key_bytes = decode_base64(public_key) + verify_key = decode_verify_key_bytes(POLICY_SERVER_KEY_ID, key_bytes) + # We would normally use KeyRing.verify_event_for_server but we can't here as we don't + # want to fetch the server key, and instead want to use the public key in the state event. + await self._hs.get_keyring().process_json(verify_key, verify_json_req) + # if the event is correctly signed by the public key in the policy server state event = Allow + return True + except Exception as ex: + logger.warning( + "failed to verify event using public key in policy server event: %s", ex + ) + return False + + async def ask_policy_server_to_sign_event( + self, event: EventBase, verify: bool = False + ) -> None: + """Ask the policy server to sign this event. The signature is added to the event signatures block. + + Does nothing if there is no policy server state event in the room. If the policy server + refuses to sign the event (as it's marked as spam) does nothing. + + Args: + event: The event to sign + verify: If True, verify that the signature is correctly signed by the public_key in the + policy server state event. + Raises: + if verify=True and the policy server signed the event with an invalid signature. Does + not raise if the policy server refuses to sign the event. + """ + policy_event = await self._storage_controllers.state.get_current_state_event( + event.room_id, POLICY_SERVER_EVENT_TYPE, "" + ) + if not policy_event: + return + policy_server = policy_event.content.get("via", None) + if policy_server is None or not isinstance(policy_server, str): + return + # Only ask to sign events if the policy state event has a public_key (so they can be subsequently verified) + public_key = policy_event.content.get("public_key", None) + if public_key is None or not isinstance(public_key, str): + return + + # Ask the policy server to sign this event. + # We set a smallish timeout here as we don't want to block event sending too long. + signature = await self._federation_client.ask_policy_server_to_sign_event( + policy_server, + event, + timeout=3000, + ) + if ( + # the policy server returns {} if it refuses to sign the event. + signature and len(signature) > 0 + ): + event.signatures.update(signature) + if verify: + is_valid = await self._verify_policy_server_signature( + event, policy_server, public_key + ) + if not is_valid: + raise SynapseError( + 500, + f"policy server {policy_server} failed to sign event correctly", + ) diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index eec420cbb1..735cfa0a0f 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -224,7 +224,7 @@ class SsoHandler: ) # a lock on the mappings - self._mapping_lock = Linearizer(name="sso_user_mapping", clock=hs.get_clock()) + self._mapping_lock = Linearizer(clock=hs.get_clock(), name="sso_user_mapping") # a map from session id to session data self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {} diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index a2602ea818..5b4a2cc62d 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -33,7 +33,6 @@ from typing import ( from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import JsonDict from synapse.util.events import get_plain_text_topic_from_event_content @@ -75,7 +74,10 @@ class StatsHandler: # We kick this off so that we don't have to wait for a change before # we start populating stats - self.clock.call_later(0, self.notify_new_event) + self.clock.call_later( + 0, + self.notify_new_event, + ) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" @@ -90,7 +92,7 @@ class StatsHandler: finally: self._is_processing = False - run_as_background_process("stats.notify_new_event", self.server_name, process) + self.hs.run_as_background_process("stats.notify_new_event", process) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 4a68fdcc76..6f0522d5bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -323,6 +323,7 @@ class SyncHandler: ] = ExpiringCache( cache_name="lazy_loaded_members_cache", server_name=self.server_name, + hs=hs, clock=self.clock, max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, @@ -980,7 +981,11 @@ class SyncHandler: ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) + cache = LruCache( + max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, + clock=self.clock, + server_name=self.server_name, + ) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 6a7b36ea0c..77c5b747c3 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -28,7 +28,6 @@ from synapse.api.constants import EduTypes from synapse.api.errors import AuthError, ShadowBanError, SynapseError from synapse.appservice import ApplicationService from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.streams import TypingStream @@ -78,11 +77,10 @@ class FollowerTypingHandler: """ def __init__(self, hs: "HomeServer"): + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.clock = hs.get_clock() self.is_mine_id = hs.is_mine_id self.is_mine_server_name = hs.is_mine_server_name @@ -144,9 +142,8 @@ class FollowerTypingHandler: if self.federation and self.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: - run_as_background_process( + self.hs.run_as_background_process( "typing._push_remote", - self.server_name, self._push_remote, member=member, typing=True, @@ -220,9 +217,8 @@ class FollowerTypingHandler: self._rooms_updated.add(row.room_id) if self.federation: - run_as_background_process( + self.hs.run_as_background_process( "_send_changes_in_typing_to_remotes", - self.server_name, self._send_changes_in_typing_to_remotes, row.room_id, prev_typing, @@ -384,9 +380,8 @@ class TypingWriterHandler(FollowerTypingHandler): def _push_update(self, member: RoomMember, typing: bool) -> None: if self.hs.is_mine_id(member.user_id): # Only send updates for changes to our own users. - run_as_background_process( + self.hs.run_as_background_process( "typing._push_remote", - self.server_name, self._push_remote, member, typing, diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 130099a239..28961f5925 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -36,7 +36,6 @@ from synapse.api.constants import ( from synapse.api.errors import Codes, SynapseError from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.databases.main.user_directory import SearchResult from synapse.storage.roommember import ProfileInfo @@ -137,11 +136,15 @@ class UserDirectoryHandler(StateDeltasHandler): # We kick this off so that we don't have to wait for a change before # we start populating the user directory - self.clock.call_later(0, self.notify_new_event) + self.clock.call_later( + 0, + self.notify_new_event, + ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( - 10, self.kick_off_remote_profile_refresh_process + 10, + self.kick_off_remote_profile_refresh_process, ) async def search_users( @@ -193,9 +196,7 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_processing = False self._is_processing = True - run_as_background_process( - "user_directory.notify_new_event", self.server_name, process - ) + self._hs.run_as_background_process("user_directory.notify_new_event", process) async def handle_local_profile_change( self, user_id: str, profile: ProfileInfo @@ -609,8 +610,8 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_refreshing_remote_profiles = False self._is_refreshing_remote_profiles = True - run_as_background_process( - "user_directory.refresh_remote_profiles", self.server_name, process + self._hs.run_as_background_process( + "user_directory.refresh_remote_profiles", process ) async def _unsafe_refresh_remote_profiles(self) -> None: @@ -655,8 +656,9 @@ class UserDirectoryHandler(StateDeltasHandler): if not users: return _, _, next_try_at_ts = users[0] + delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2 self._refresh_remote_profiles_call_later = self.clock.call_later( - ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2, + delay, self.kick_off_remote_profile_refresh_process, ) @@ -692,9 +694,8 @@ class UserDirectoryHandler(StateDeltasHandler): self._is_refreshing_remote_profiles_for_servers.remove(server_name) self._is_refreshing_remote_profiles_for_servers.add(server_name) - run_as_background_process( + self._hs.run_as_background_process( "user_directory.refresh_remote_profiles_for_remote_server", - self.server_name, process, ) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 0b375790dd..ca1e2b166c 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -37,13 +37,13 @@ from weakref import WeakSet import attr from twisted.internet import defer -from twisted.internet.interfaces import IReactorTime from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import start_active_span from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore from synapse.util.async_helpers import timeout_deferred +from synapse.util.clock import Clock from synapse.util.constants import ONE_MINUTE_SECONDS if TYPE_CHECKING: @@ -66,10 +66,8 @@ class WorkerLocksHandler: """ def __init__(self, hs: "HomeServer") -> None: - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process - self._reactor = hs.get_reactor() + self.hs = hs # nb must be called this for @wrap_as_background_process + self._clock = hs.get_clock() self._store = hs.get_datastores().main self._clock = hs.get_clock() self._notifier = hs.get_notifier() @@ -98,7 +96,7 @@ class WorkerLocksHandler: """ lock = WaitingLock( - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, lock_name=lock_name, @@ -129,7 +127,7 @@ class WorkerLocksHandler: """ lock = WaitingLock( - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, lock_name=lock_name, @@ -160,7 +158,7 @@ class WorkerLocksHandler: lock = WaitingMultiLock( lock_names=lock_names, write=write, - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, ) @@ -197,7 +195,11 @@ class WorkerLocksHandler: if not deferred.called: deferred.callback(None) - self._clock.call_later(0, _wake_all_locks, locks) + self._clock.call_later( + 0, + _wake_all_locks, + locks, + ) @wrap_as_background_process("_cleanup_locks") async def _cleanup_locks(self) -> None: @@ -207,7 +209,7 @@ class WorkerLocksHandler: @attr.s(auto_attribs=True, eq=False) class WaitingLock: - reactor: IReactorTime + clock: Clock store: LockStore handler: WorkerLocksHandler lock_name: str @@ -246,10 +248,11 @@ class WaitingLock: # periodically wake up in case the lock was released but we # weren't notified. with PreserveLoggingContext(): + timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=self._get_next_retry_interval(), - reactor=self.reactor, + timeout=timeout, + clock=self.clock, ) except Exception: pass @@ -290,7 +293,7 @@ class WaitingMultiLock: write: bool - reactor: IReactorTime + clock: Clock store: LockStore handler: WorkerLocksHandler @@ -323,10 +326,11 @@ class WaitingMultiLock: # periodically wake up in case the lock was released but we # weren't notified. with PreserveLoggingContext(): + timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=self._get_next_retry_interval(), - reactor=self.reactor, + timeout=timeout, + clock=self.clock, ) except Exception: pass diff --git a/synapse/http/client.py b/synapse/http/client.py index bbb0efe8b5..370cdc3568 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -54,7 +54,6 @@ from twisted.internet.interfaces import ( IOpenSSLContextFactory, IReactorCore, IReactorPluggableNameResolver, - IReactorTime, IResolutionReceiver, ITCPTransport, ) @@ -88,6 +87,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence from synapse.util.async_helpers import timeout_deferred +from synapse.util.clock import Clock from synapse.util.json import json_decoder if TYPE_CHECKING: @@ -165,16 +165,17 @@ def _is_ip_blocked( _EPSILON = 0.00000001 -def _make_scheduler( - reactor: IReactorTime, -) -> Callable[[Callable[[], object]], IDelayedCall]: +def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. (This is effectively just a copy from `twisted.internet.task`) """ def _scheduler(x: Callable[[], object]) -> IDelayedCall: - return reactor.callLater(_EPSILON, x) + return clock.call_later( + _EPSILON, + x, + ) return _scheduler @@ -367,7 +368,7 @@ class BaseHttpClient: # We use this for our body producers to ensure that they use the correct # reactor. - self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_reactor())) + self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_clock())) async def request( self, @@ -436,9 +437,9 @@ class BaseHttpClient: # we use our own timeout mechanism rather than treq's as a workaround # for https://twistedmatrix.com/trac/ticket/9534. request_deferred = timeout_deferred( - request_deferred, - 60, - self.hs.get_reactor(), + deferred=request_deferred, + timeout=60, + clock=self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors @@ -763,7 +764,11 @@ class BaseHttpClient: d = read_body_with_max_size(response, output_stream, max_size) # Ensure that the body is not read forever. - d = timeout_deferred(d, 30, self.hs.get_reactor()) + d = timeout_deferred( + deferred=d, + timeout=30, + clock=self.hs.get_clock(), + ) length = await make_deferred_yieldable(d) except BodyExceededMaxSize: @@ -957,9 +962,9 @@ class ReplicationClient(BaseHttpClient): # for https://twistedmatrix.com/trac/ticket/9534. # (Updated url https://github.com/twisted/twisted/issues/9534) request_deferred = timeout_deferred( - request_deferred, - 60, - self.hs.get_reactor(), + deferred=request_deferred, + timeout=60, + clock=self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 2d152d441e..9d87514be0 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -67,6 +67,9 @@ class MatrixFederationAgent: Args: reactor: twisted reactor to use for underlying requests + clock: Internal `HomeServer` clock used to track delayed and looping calls. + Should be obtained from `hs.get_clock()`. + tls_client_options_factory: factory to use for fetching client tls options, or none to disable TLS. @@ -97,6 +100,7 @@ class MatrixFederationAgent: *, server_name: str, reactor: ISynapseReactor, + clock: Clock, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, ip_allowlist: Optional[IPSet], @@ -109,6 +113,7 @@ class MatrixFederationAgent: Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). reactor + clock: Should be the `hs` clock from `hs.get_clock()` tls_client_options_factory user_agent ip_allowlist @@ -124,7 +129,6 @@ class MatrixFederationAgent: # addresses, to prevent DNS rebinding. reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist) - self._clock = Clock(reactor) self._pool = HTTPConnectionPool(reactor) self._pool.retryAutomatically = False self._pool.maxPersistentPerHost = 5 @@ -147,6 +151,7 @@ class MatrixFederationAgent: _well_known_resolver = WellKnownResolver( server_name=server_name, reactor=reactor, + clock=clock, agent=BlocklistingAgentWrapper( ProxyAgent( reactor=reactor, diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 49e58c8228..2f52abcc03 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -90,6 +90,7 @@ class WellKnownResolver: self, server_name: str, reactor: ISynapseThreadlessReactor, + clock: Clock, agent: IAgent, user_agent: bytes, well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None, @@ -99,6 +100,7 @@ class WellKnownResolver: Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). reactor + clock: Should be the `hs` clock from `hs.get_clock()` agent user_agent well_known_cache @@ -107,7 +109,7 @@ class WellKnownResolver: self.server_name = server_name self._reactor = reactor - self._clock = Clock(reactor) + self._clock = clock if well_known_cache is None: well_known_cache = TTLCache( diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 419e4444ab..4d72c72d01 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -90,6 +90,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict from synapse.util.async_helpers import AwakenableSleeper, Linearizer, timeout_deferred +from synapse.util.clock import Clock from synapse.util.json import json_decoder from synapse.util.metrics import Measure from synapse.util.stringutils import parse_and_validate_server_name @@ -270,6 +271,7 @@ class LegacyJsonSendParser(_BaseJsonParser[Tuple[int, JsonDict]]): async def _handle_response( + clock: Clock, reactor: IReactorTime, timeout_sec: float, request: MatrixFederationRequest, @@ -299,7 +301,11 @@ async def _handle_response( check_content_type_is(response.headers, parser.CONTENT_TYPE) d = read_body_with_max_size(response, parser, max_response_size) - d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor) + d = timeout_deferred( + deferred=d, + timeout=timeout_sec, + clock=clock, + ) length = await make_deferred_yieldable(d) @@ -411,6 +417,7 @@ class MatrixFederationHttpClient: self.server_name = hs.hostname self.reactor = hs.get_reactor() + self.clock = hs.get_clock() user_agent = hs.version_string if hs.config.server.user_agent_suffix: @@ -424,6 +431,7 @@ class MatrixFederationHttpClient: federation_agent: IAgent = MatrixFederationAgent( server_name=self.server_name, reactor=self.reactor, + clock=self.clock, tls_client_options_factory=tls_client_options_factory, user_agent=user_agent.encode("ascii"), ip_allowlist=hs.config.server.federation_ip_range_allowlist, @@ -457,7 +465,6 @@ class MatrixFederationHttpClient: ip_blocklist=hs.config.server.federation_ip_range_blocklist, ) - self.clock = hs.get_clock() self._store = hs.get_datastores().main self.version_string_bytes = hs.version_string.encode("ascii") self.default_timeout_seconds = hs.config.federation.client_timeout_ms / 1000 @@ -470,9 +477,9 @@ class MatrixFederationHttpClient: self.max_long_retries = hs.config.federation.max_long_retries self.max_short_retries = hs.config.federation.max_short_retries - self._cooperator = Cooperator(scheduler=_make_scheduler(self.reactor)) + self._cooperator = Cooperator(scheduler=_make_scheduler(self.clock)) - self._sleeper = AwakenableSleeper(self.reactor) + self._sleeper = AwakenableSleeper(self.clock) self._simple_http_client = SimpleHttpClient( hs, @@ -481,7 +488,13 @@ class MatrixFederationHttpClient: use_proxy=True, ) - self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6) + self.remote_download_linearizer = Linearizer( + name="remote_download_linearizer", max_count=6, clock=self.clock + ) + self._is_shutdown = False + + def shutdown(self) -> None: + self._is_shutdown = True def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" @@ -627,6 +640,7 @@ class MatrixFederationHttpClient: limiter = await synapse.util.retryutils.get_retry_limiter( destination=request.destination, our_server_name=self.server_name, + hs=self.hs, clock=self.clock, store=self._store, backoff_on_404=backoff_on_404, @@ -673,7 +687,7 @@ class MatrixFederationHttpClient: (b"", b"", path_bytes, None, query_bytes, b"") ) - while True: + while not self._is_shutdown: try: json = request.get_json() if json: @@ -731,9 +745,9 @@ class MatrixFederationHttpClient: bodyProducer=producer, ) request_deferred = timeout_deferred( - request_deferred, + deferred=request_deferred, timeout=_sec_timeout, - reactor=self.reactor, + clock=self.clock, ) response = await make_deferred_yieldable(request_deferred) @@ -791,7 +805,9 @@ class MatrixFederationHttpClient: # Update transactions table? d = treq.content(response) d = timeout_deferred( - d, timeout=_sec_timeout, reactor=self.reactor + deferred=d, + timeout=_sec_timeout, + clock=self.clock, ) try: @@ -860,6 +876,15 @@ class MatrixFederationHttpClient: delay_seconds, ) + if self._is_shutdown: + # Immediately fail sending the request instead of starting a + # potentially long sleep after the server has requested + # shutdown. + # This is the code path followed when the + # `federation_transaction_transmission_loop` has been + # cancelled. + raise + # Sleep for the calculated delay, or wake up immediately # if we get notified that the server is back up. await self._sleeper.sleep( @@ -1072,6 +1097,7 @@ class MatrixFederationHttpClient: parser = cast(ByteParser[T], JsonParser()) body = await _handle_response( + self.clock, self.reactor, _sec_timeout, request, @@ -1150,7 +1176,13 @@ class MatrixFederationHttpClient: _sec_timeout = self.default_timeout_seconds body = await _handle_response( - self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser() + self.clock, + self.reactor, + _sec_timeout, + request, + response, + start_ms, + parser=JsonParser(), ) return body @@ -1356,6 +1388,7 @@ class MatrixFederationHttpClient: parser = cast(ByteParser[T], JsonParser()) body = await _handle_response( + self.clock, self.reactor, _sec_timeout, request, @@ -1429,7 +1462,13 @@ class MatrixFederationHttpClient: _sec_timeout = self.default_timeout_seconds body = await _handle_response( - self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser() + self.clock, + self.reactor, + _sec_timeout, + request, + response, + start_ms, + parser=JsonParser(), ) return body diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 9b044f3b0a..fa17432984 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -161,12 +161,12 @@ class ProxyResource(_AsyncResource): bodyProducer=QuieterFileBodyProducer(request.content), ) request_deferred = timeout_deferred( - request_deferred, + deferred=request_deferred, # This should be set longer than the timeout in `MatrixFederationHttpClient` # so that it has enough time to complete and pass us the data before we give # up. timeout=90, - reactor=self.reactor, + clock=self._clock, ) response = await make_deferred_yieldable(request_deferred) diff --git a/synapse/http/server.py b/synapse/http/server.py index 0b987f6aa9..d5af8758ac 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -411,8 +411,26 @@ class DirectServeJsonResource(_AsyncResource): # Clock is optional as this class is exposed to the module API. clock: Optional[Clock] = None, ): + """ + Args: + canonical_json: TODO + extract_context: TODO + clock: This is expected to be passed in by any Synapse code. + Only optional for the Module API. + """ + if clock is None: - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) + # Ideally we wouldn't ignore the linter error here and instead enforce a + # required `Clock` be passed into the `__init__` function. + # However, this would change the function signature which is currently being + # exported to the module api. Since we don't want to break that api, we have + # to settle with ignoring the linter error here. + # As of the time of writing this, all Synapse internal usages of + # `DirectServeJsonResource` pass in the existing homeserver clock instance. + clock = Clock( # type: ignore[multiple-internal-clocks] + cast(ISynapseThreadlessReactor, reactor), + server_name="synapse_module_running_from_unknown_server", + ) super().__init__(clock, extract_context) self.canonical_json = canonical_json @@ -590,8 +608,24 @@ class DirectServeHtmlResource(_AsyncResource): # Clock is optional as this class is exposed to the module API. clock: Optional[Clock] = None, ): + """ + Args: + extract_context: TODO + clock: This is expected to be passed in by any Synapse code. + Only optional for the Module API. + """ if clock is None: - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) + # Ideally we wouldn't ignore the linter error here and instead enforce a + # required `Clock` be passed into the `__init__` function. + # However, this would change the function signature which is currently being + # exported to the module api. Since we don't want to break that api, we have + # to settle with ignoring the linter error here. + # As of the time of writing this, all Synapse internal usages of + # `DirectServeHtmlResource` pass in the existing homeserver clock instance. + clock = Clock( # type: ignore[multiple-internal-clocks] + cast(ISynapseThreadlessReactor, reactor), + server_name="synapse_module_running_from_unknown_server", + ) super().__init__(clock, extract_context) diff --git a/synapse/http/site.py b/synapse/http/site.py index 55088fc190..f4f326cfde 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -22,7 +22,7 @@ import contextlib import logging import time from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Generator, List, Optional, Tuple, Union import attr from zope.interface import implementer @@ -30,6 +30,7 @@ from zope.interface import implementer from twisted.internet.address import UNIXAddress from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress +from twisted.internet.protocol import Protocol from twisted.python.failure import Failure from twisted.web.http import HTTPChannel from twisted.web.resource import IResource, Resource @@ -302,10 +303,15 @@ class SynapseRequest(Request): # this is called once a Resource has been found to serve the request; in our # case the Resource in question will normally be a JsonResource. - # create a LogContext for this request + # Create a LogContext for this request + # + # We only care about associating logs and tallying up metrics at the per-request + # level so we don't worry about setting the `parent_context`; preventing us from + # unnecessarily piling up metrics on the main process's context. request_id = self.get_request_id() self.logcontext = LoggingContext( - request_id, + name=request_id, + server_name=self.our_server_name, request=ContextRequest( request_id=request_id, ip_address=self.get_client_ip_if_available(), @@ -655,6 +661,70 @@ class _XForwardedForAddress: host: str +class SynapseProtocol(HTTPChannel): + """ + Synapse-specific twisted http Protocol. + + This is a small wrapper around the twisted HTTPChannel so we can track active + connections in order to close any outstanding connections on shutdown. + """ + + def __init__( + self, + site: "SynapseSite", + our_server_name: str, + max_request_body_size: int, + request_id_header: Optional[str], + request_class: type, + ): + super().__init__() + self.factory: SynapseSite = site + self.site = site + self.our_server_name = our_server_name + self.max_request_body_size = max_request_body_size + self.request_id_header = request_id_header + self.request_class = request_class + + def connectionMade(self) -> None: + """ + Called when a connection is made. + + This may be considered the initializer of the protocol, because + it is called when the connection is completed. + + Add the connection to the factory's connection list when it's established. + """ + super().connectionMade() + self.factory.addConnection(self) + + def connectionLost(self, reason: Failure) -> None: # type: ignore[override] + """ + Called when the connection is shut down. + + Clear any circular references here, and any external references to this + Protocol. The connection has been closed. In our case, we need to remove the + connection from the factory's connection list, when it's lost. + """ + super().connectionLost(reason) + self.factory.removeConnection(self) + + def requestFactory(self, http_channel: HTTPChannel, queued: bool) -> SynapseRequest: # type: ignore[override] + """ + A callable used to build `twisted.web.iweb.IRequest` objects. + + Use our own custom SynapseRequest type instead of the regular + twisted.web.server.Request. + """ + return self.request_class( + self, + self.factory, + our_server_name=self.our_server_name, + max_request_body_size=self.max_request_body_size, + queued=queued, + request_id_header=self.request_id_header, + ) + + class SynapseSite(ProxySite): """ Synapse-specific twisted http Site @@ -705,23 +775,44 @@ class SynapseSite(ProxySite): assert config.http_options is not None proxied = config.http_options.x_forwarded - request_class = XForwardedForRequest if proxied else SynapseRequest + self.request_class = XForwardedForRequest if proxied else SynapseRequest - request_id_header = config.http_options.request_id_header + self.request_id_header = config.http_options.request_id_header + self.max_request_body_size = max_request_body_size - def request_factory(channel: HTTPChannel, queued: bool) -> Request: - return request_class( - channel, - self, - our_server_name=self.server_name, - max_request_body_size=max_request_body_size, - queued=queued, - request_id_header=request_id_header, - ) - - self.requestFactory = request_factory # type: ignore self.access_logger = logging.getLogger(logger_name) self.server_version_string = server_version_string.encode("ascii") + self.connections: List[Protocol] = [] + + def buildProtocol(self, addr: IAddress) -> SynapseProtocol: + protocol = SynapseProtocol( + self, + self.server_name, + self.max_request_body_size, + self.request_id_header, + self.request_class, + ) + return protocol + + def addConnection(self, protocol: Protocol) -> None: + self.connections.append(protocol) + + def removeConnection(self, protocol: Protocol) -> None: + if protocol in self.connections: + self.connections.remove(protocol) + + def stopFactory(self) -> None: + super().stopFactory() + + # Shutdown any connections which are still active. + # These can be long lived HTTP connections which wouldn't normally be closed + # when calling `shutdown` on the respective `Port`. + # Closing the connections here is required for us to fully shutdown the + # `SynapseHomeServer` in order for it to be garbage collected. + for protocol in self.connections[:]: + if protocol.transport is not None: + protocol.transport.loseConnection() + self.connections.clear() def log(self, request: SynapseRequest) -> None: # type: ignore[override] pass diff --git a/synapse/logging/context.py b/synapse/logging/context.py index b5b434f3a8..1b9c770311 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -33,7 +33,6 @@ See doc/log_contexts.rst for details on how this works. import logging import threading import typing -import warnings from types import TracebackType from typing import ( TYPE_CHECKING, @@ -55,11 +54,29 @@ from typing_extensions import ParamSpec from twisted.internet import defer, threads from twisted.python.threadpool import ThreadPool +from synapse.logging.loggers import ExplicitlyConfiguredLogger +from synapse.util.stringutils import random_string + if TYPE_CHECKING: + from synapse.logging.scopecontextmanager import _LogContextScope from synapse.types import ISynapseReactor logger = logging.getLogger(__name__) +original_logger_class = logging.getLoggerClass() +logging.setLoggerClass(ExplicitlyConfiguredLogger) +logcontext_debug_logger = logging.getLogger("synapse.logging.context.debug") +""" +A logger for debugging when the logcontext switches. + +Because this is very noisy and probably something only developers want to see when +debugging logcontext problems, we want people to explictly opt-in before seeing anything +in the logs. Requires explicitly setting `synapse.logging.context.debug` in the logging +configuration and does not inherit the log level from the parent logger. +""" +# Restore the original logger class +logging.setLoggerClass(original_logger_class) + try: import resource @@ -238,13 +255,22 @@ class _Sentinel: we should always know which server the logs are coming from. """ - __slots__ = ["previous_context", "finished", "request", "tag"] + __slots__ = [ + "previous_context", + "finished", + "scope", + "server_name", + "request", + "tag", + ] def __init__(self) -> None: # Minimal set for compatibility with LoggingContext self.previous_context = None self.finished = False + self.server_name = "unknown_server_from_sentinel_context" self.request = None + self.scope = None self.tag = None def __str__(self) -> str: @@ -282,14 +308,19 @@ class LoggingContext: child to the parent Args: - name: Name for the context for logging. If this is omitted, it is - inherited from the parent context. + name: Name for the context for logging. + server_name: The name of the server this context is associated with + (`config.server.server_name` or `hs.hostname`) parent_context (LoggingContext|None): The parent of the new context + request: Synapse Request Context object. Useful to associate all the logs + happening to a given request. + """ __slots__ = [ "previous_context", "name", + "server_name", "parent_context", "_resource_usage", "usage_start", @@ -297,11 +328,14 @@ class LoggingContext: "finished", "request", "tag", + "scope", ] def __init__( self, - name: Optional[str] = None, + *, + name: str, + server_name: str, parent_context: "Optional[LoggingContext]" = None, request: Optional[ContextRequest] = None, ) -> None: @@ -314,9 +348,12 @@ class LoggingContext: # if the context is not currently active. self.usage_start: Optional[resource.struct_rusage] = None + self.name = name + self.server_name = server_name self.main_thread = get_thread_id() self.request = None self.tag = "" + self.scope: Optional["_LogContextScope"] = None # keep track of whether we have hit the __exit__ block for this context # (suggesting that the the thing that created the context thinks it should @@ -325,69 +362,24 @@ class LoggingContext: self.parent_context = parent_context + # Inherit some fields from the parent context if self.parent_context is not None: - # we track the current request_id + # which request this corresponds to self.request = self.parent_context.request + # we also track the current scope: + self.scope = self.parent_context.scope + if request is not None: # the request param overrides the request from the parent context self.request = request - # if we don't have a `name`, but do have a parent context, use its name. - if self.parent_context and name is None: - name = str(self.parent_context) - if name is None: - raise ValueError( - "LoggingContext must be given either a name or a parent context" - ) - self.name = name - def __str__(self) -> str: return self.name - @classmethod - def current_context(cls) -> LoggingContextOrSentinel: - """Get the current logging context from thread local storage - - This exists for backwards compatibility. ``current_context()`` should be - called directly. - - Returns: - The current logging context - """ - warnings.warn( - "synapse.logging.context.LoggingContext.current_context() is deprecated " - "in favor of synapse.logging.context.current_context().", - DeprecationWarning, - stacklevel=2, - ) - return current_context() - - @classmethod - def set_current_context( - cls, context: LoggingContextOrSentinel - ) -> LoggingContextOrSentinel: - """Set the current logging context in thread local storage - - This exists for backwards compatibility. ``set_current_context()`` should be - called directly. - - Args: - context: The context to activate. - - Returns: - The context that was previously active - """ - warnings.warn( - "synapse.logging.context.LoggingContext.set_current_context() is deprecated " - "in favor of synapse.logging.context.set_current_context().", - DeprecationWarning, - stacklevel=2, - ) - return set_current_context(context) - def __enter__(self) -> "LoggingContext": """Enters this logging context into thread local storage""" + logcontext_debug_logger.debug("LoggingContext(%s).__enter__", self.name) old_context = set_current_context(self) if self.previous_context != old_context: logcontext_error( @@ -410,6 +402,9 @@ class LoggingContext: Returns: None to avoid suppressing any exceptions that were thrown. """ + logcontext_debug_logger.debug( + "LoggingContext(%s).__exit__ --> %s", self.name, self.previous_context + ) current = set_current_context(self.previous_context) if current is not self: if current is SENTINEL_CONTEXT: @@ -588,7 +583,26 @@ class LoggingContextFilter(logging.Filter): record. """ - def __init__(self, request: str = ""): + def __init__( + self, + # `request` is here for backwards compatibility since we previously recommended + # people manually configure `LoggingContextFilter` like the following. + # + # ```yaml + # filters: + # context: + # (): synapse.logging.context.LoggingContextFilter + # request: "" + # ``` + # + # TODO: Since we now configure `LoggingContextFilter` automatically since #8051 + # (2020-08-11), we could consider removing this useless parameter. This would + # require people to remove their own manual configuration of + # `LoggingContextFilter` as it would cause `TypeError: Filter.__init__() got an + # unexpected keyword argument 'request'` -> `ValueError: Unable to configure + # filter 'context'` + request: str = "", + ): self._default_request = request def filter(self, record: logging.LogRecord) -> Literal[True]: @@ -598,11 +612,13 @@ class LoggingContextFilter(logging.Filter): """ context = current_context() record.request = self._default_request + record.server_name = "unknown_server_from_no_context" # context should never be None, but if it somehow ends up being, then # we end up in a death spiral of infinite loops, so let's check, for # robustness' sake. if context is not None: + record.server_name = context.server_name # Logging is interested in the request ID. Note that for backwards # compatibility this is stored as the "request" on the record. record.request = str(context) @@ -637,14 +653,21 @@ class PreserveLoggingContext: reactor back to the code). """ - __slots__ = ["_old_context", "_new_context"] + __slots__ = ["_old_context", "_new_context", "_instance_id"] def __init__( self, new_context: LoggingContextOrSentinel = SENTINEL_CONTEXT ) -> None: self._new_context = new_context + self._instance_id = random_string(5) def __enter__(self) -> None: + logcontext_debug_logger.debug( + "PreserveLoggingContext(%s).__enter__ %s --> %s", + self._instance_id, + current_context(), + self._new_context, + ) self._old_context = set_current_context(self._new_context) def __exit__( @@ -653,6 +676,12 @@ class PreserveLoggingContext: value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: + logcontext_debug_logger.debug( + "PreserveLoggingContext(%s).__exit %s --> %s", + self._instance_id, + current_context(), + self._old_context, + ) context = set_current_context(self._old_context) if context != self._new_context: @@ -728,12 +757,15 @@ def nested_logging_context(suffix: str) -> LoggingContext: "Starting nested logging context from sentinel context: metrics will be lost" ) parent_context = None + server_name = "unknown_server_from_sentinel_context" else: assert isinstance(curr_context, LoggingContext) parent_context = curr_context + server_name = parent_context.server_name prefix = str(curr_context) return LoggingContext( - prefix + "-" + suffix, + name=prefix + "-" + suffix, + server_name=server_name, parent_context=parent_context, ) @@ -829,7 +861,11 @@ def run_in_background( Note that the returned Deferred does not follow the synapse logcontext rules. """ + instance_id = random_string(5) calling_context = current_context() + logcontext_debug_logger.debug( + "run_in_background(%s): called with logcontext=%s", instance_id, calling_context + ) try: # (kick off the task in the current context) res = f(*args, **kwargs) @@ -871,6 +907,11 @@ def run_in_background( # to reset the logcontext to the sentinel logcontext as that would run # immediately (remember our goal is to maintain the calling logcontext when we # return). + logcontext_debug_logger.debug( + "run_in_background(%s): deferred already completed and the function should have maintained the logcontext %s", + instance_id, + calling_context, + ) return d # Since the function we called may follow the Synapse logcontext rules (Rules for @@ -881,6 +922,11 @@ def run_in_background( # # Our goal is to have the caller logcontext unchanged after firing off the # background task and returning. + logcontext_debug_logger.debug( + "run_in_background(%s): restoring calling logcontext %s", + instance_id, + calling_context, + ) set_current_context(calling_context) # If the function we called is playing nice and following the Synapse logcontext @@ -896,7 +942,23 @@ def run_in_background( # which is supposed to have a single entry and exit point. But # by spawning off another deferred, we are effectively # adding a new exit point.) - d.addBoth(_set_context_cb, SENTINEL_CONTEXT) + if logcontext_debug_logger.isEnabledFor(logging.DEBUG): + + def _log_set_context_cb( + result: ResultT, context: LoggingContextOrSentinel + ) -> ResultT: + logcontext_debug_logger.debug( + "run_in_background(%s): resetting logcontext to %s", + instance_id, + context, + ) + set_current_context(context) + return result + + d.addBoth(_log_set_context_cb, SENTINEL_CONTEXT) + else: + d.addBoth(_set_context_cb, SENTINEL_CONTEXT) + return d @@ -952,10 +1014,21 @@ def make_deferred_yieldable(deferred: "defer.Deferred[T]") -> "defer.Deferred[T] restores the old context once the awaitable completes (execution passes from the reactor back to the code). """ + instance_id = random_string(5) + logcontext_debug_logger.debug( + "make_deferred_yieldable(%s): called with logcontext=%s", + instance_id, + current_context(), + ) + # The deferred has already completed if deferred.called and not deferred.paused: # it looks like this deferred is ready to run any callbacks we give it # immediately. We may as well optimise out the logcontext faffery. + logcontext_debug_logger.debug( + "make_deferred_yieldable(%s): deferred already completed and the function should have maintained the logcontext", + instance_id, + ) return deferred # Our goal is to have the caller logcontext unchanged after they yield/await the @@ -967,8 +1040,31 @@ def make_deferred_yieldable(deferred: "defer.Deferred[T]") -> "defer.Deferred[T] # does) while the deferred runs in the reactor event loop, we reset the logcontext # and add a callback to the deferred to restore it so the caller's logcontext is # active when the deferred completes. - prev_context = set_current_context(SENTINEL_CONTEXT) - deferred.addBoth(_set_context_cb, prev_context) + + logcontext_debug_logger.debug( + "make_deferred_yieldable(%s): resetting logcontext to %s", + instance_id, + SENTINEL_CONTEXT, + ) + calling_context = set_current_context(SENTINEL_CONTEXT) + + if logcontext_debug_logger.isEnabledFor(logging.DEBUG): + + def _log_set_context_cb( + result: ResultT, context: LoggingContextOrSentinel + ) -> ResultT: + logcontext_debug_logger.debug( + "make_deferred_yieldable(%s): restoring calling logcontext to %s", + instance_id, + context, + ) + set_current_context(context) + return result + + deferred.addBoth(_log_set_context_cb, calling_context) + else: + deferred.addBoth(_set_context_cb, calling_context) + return deferred @@ -1058,12 +1154,18 @@ def defer_to_threadpool( "Calling defer_to_threadpool from sentinel context: metrics will be lost" ) parent_context = None + server_name = "unknown_server_from_sentinel_context" else: assert isinstance(curr_context, LoggingContext) parent_context = curr_context + server_name = parent_context.server_name def g() -> R: - with LoggingContext(str(curr_context), parent_context=parent_context): + with LoggingContext( + name=str(curr_context), + server_name=server_name, + parent_context=parent_context, + ): return f(*args, **kwargs) return make_deferred_yieldable(threads.deferToThreadPool(reactor, threadpool, g)) diff --git a/synapse/logging/filter.py b/synapse/logging/filter.py deleted file mode 100644 index 16de488dbc..0000000000 --- a/synapse/logging/filter.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2020 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -import logging -from typing import Literal - - -class MetadataFilter(logging.Filter): - """Logging filter that adds constant values to each record. - - Args: - metadata: Key-value pairs to add to each record. - """ - - def __init__(self, metadata: dict): - self._metadata = metadata - - def filter(self, record: logging.LogRecord) -> Literal[True]: - for key, value in self._metadata.items(): - setattr(record, key, value) - return True diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 405939d518..1c89a358df 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -251,17 +251,18 @@ class _DummyTagNames: try: import opentracing import opentracing.tags - from opentracing.scope_managers.contextvars import ContextVarsScopeManager tags = opentracing.tags except ImportError: opentracing = None # type: ignore[assignment] tags = _DummyTagNames # type: ignore[assignment] - ContextVarsScopeManager = None # type: ignore try: from jaeger_client import Config as JaegerConfig + + from synapse.logging.scopecontextmanager import LogContextScopeManager except ImportError: JaegerConfig = None # type: ignore + LogContextScopeManager = None # type: ignore try: @@ -483,7 +484,7 @@ def init_tracer(hs: "HomeServer") -> None: config = JaegerConfig( config=jaeger_config, service_name=f"{hs.config.server.server_name} {instance_name_by_type}", - scope_manager=ContextVarsScopeManager(), + scope_manager=LogContextScopeManager(), metrics_factory=PrometheusMetricsFactory(), ) @@ -576,7 +577,9 @@ def start_active_span_follows_from( operation_name: str, contexts: Collection, child_of: Optional[Union["opentracing.Span", "opentracing.SpanContext"]] = None, + tags: Optional[Dict[str, str]] = None, start_time: Optional[float] = None, + ignore_active_span: bool = False, *, inherit_force_tracing: bool = False, tracer: Optional["opentracing.Tracer"] = None, @@ -591,9 +594,16 @@ def start_active_span_follows_from( span will be the parent. (If there is no currently active span, the first span in `contexts` will be the parent.) + tags: an optional dictionary of span tags. The caller gives up ownership of that + dictionary, because the :class:`Tracer` may use it as-is to avoid extra data + copying. + start_time: optional override for the start time of the created span. Seconds since the epoch. + ignore_active_span: an explicit flag that ignores the current active + scope and creates a root span. + inherit_force_tracing: if set, and any of the previous contexts have had tracing forced, the new span will also have tracing forced. tracer: override the opentracing tracer. By default the global tracer is used. @@ -606,7 +616,9 @@ def start_active_span_follows_from( operation_name, child_of=child_of, references=references, + tags=tags, start_time=start_time, + ignore_active_span=ignore_active_span, tracer=tracer, ) @@ -672,9 +684,21 @@ def start_active_span_from_edu( # Opentracing setters for tags, logs, etc @only_if_tracing -def active_span() -> Optional["opentracing.Span"]: - """Get the currently active span, if any""" - return opentracing.tracer.active_span +def active_span( + *, + tracer: Optional["opentracing.Tracer"] = None, +) -> Optional["opentracing.Span"]: + """ + Get the currently active span, if any + + Args: + tracer: override the opentracing tracer. By default the global tracer is used. + """ + if tracer is None: + # use the global tracer by default + tracer = opentracing.tracer + + return tracer.active_span @ensure_active_span("set a tag") diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py new file mode 100644 index 0000000000..feaadc4d87 --- /dev/null +++ b/synapse/logging/scopecontextmanager.py @@ -0,0 +1,161 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import logging +from typing import Optional + +from opentracing import Scope, ScopeManager, Span + +from synapse.logging.context import ( + LoggingContext, + current_context, + nested_logging_context, +) + +logger = logging.getLogger(__name__) + + +class LogContextScopeManager(ScopeManager): + """ + The LogContextScopeManager tracks the active scope in opentracing + by using the log contexts which are native to synapse. This is so + that the basic opentracing api can be used across twisted defereds. + + It would be nice just to use opentracing's ContextVarsScopeManager, + but currently that doesn't work due to https://twistedmatrix.com/trac/ticket/10301. + """ + + def __init__(self) -> None: + pass + + @property + def active(self) -> Optional[Scope]: + """ + Returns the currently active Scope which can be used to access the + currently active Scope.span. + If there is a non-null Scope, its wrapped Span + becomes an implicit parent of any newly-created Span at + Tracer.start_active_span() time. + + Return: + The Scope that is active, or None if not available. + """ + ctx = current_context() + return ctx.scope + + def activate(self, span: Span, finish_on_close: bool) -> Scope: + """ + Makes a Span active. + Args + span: the span that should become active. + finish_on_close: whether Span should be automatically finished when + Scope.close() is called. + + Returns: + Scope to control the end of the active period for + *span*. It is a programming error to neglect to call + Scope.close() on the returned instance. + """ + + ctx = current_context() + + if not ctx: + logger.error("Tried to activate scope outside of loggingcontext") + return Scope(None, span) # type: ignore[arg-type] + + if ctx.scope is not None: + # start a new logging context as a child of the existing one. + # Doing so -- rather than updating the existing logcontext -- means that + # creating several concurrent spans under the same logcontext works + # correctly. + ctx = nested_logging_context("") + enter_logcontext = True + else: + # if there is no span currently associated with the current logcontext, we + # just store the scope in it. + # + # This feels a bit dubious, but it does hack around a problem where a + # span outlasts its parent logcontext (which would otherwise lead to + # "Re-starting finished log context" errors). + enter_logcontext = False + + scope = _LogContextScope(self, span, ctx, enter_logcontext, finish_on_close) + ctx.scope = scope + if enter_logcontext: + ctx.__enter__() + + return scope + + +class _LogContextScope(Scope): + """ + A custom opentracing scope, associated with a LogContext + + * When the scope is closed, the logcontext's active scope is reset to None. + and - if enter_logcontext was set - the logcontext is finished too. + """ + + def __init__( + self, + manager: LogContextScopeManager, + span: Span, + logcontext: LoggingContext, + enter_logcontext: bool, + finish_on_close: bool, + ): + """ + Args: + manager: + the manager that is responsible for this scope. + span: + the opentracing span which this scope represents the local + lifetime for. + logcontext: + the log context to which this scope is attached. + enter_logcontext: + if True the log context will be exited when the scope is finished + finish_on_close: + if True finish the span when the scope is closed + """ + super().__init__(manager, span) + self.logcontext = logcontext + self._finish_on_close = finish_on_close + self._enter_logcontext = enter_logcontext + + def __str__(self) -> str: + return f"Scope<{self.span}>" + + def close(self) -> None: + active_scope = self.manager.active + if active_scope is not self: + logger.error( + "Closing scope %s which is not the currently-active one %s", + self, + active_scope, + ) + + if self._finish_on_close: + self.span.finish() + + self.logcontext.scope = None + + if self._enter_logcontext: + self.logcontext.__exit__(None, None, None) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 15b28074fd..d3a9a66f5a 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -704,6 +704,7 @@ class ThreadedFileSender: def __init__(self, hs: "HomeServer") -> None: self.reactor = hs.get_reactor() + self.clock = hs.get_clock() self.thread_pool = hs.get_media_sender_thread_pool() self.file: Optional[BinaryIO] = None @@ -712,7 +713,7 @@ class ThreadedFileSender: # Signals if the thread should keep reading/sending data. Set means # continue, clear means pause. - self.wakeup_event = DeferredEvent(self.reactor) + self.wakeup_event = DeferredEvent(self.clock) # Signals if the thread should terminate, e.g. because the consumer has # gone away. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 6a0f276672..e278308b1d 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -68,7 +68,6 @@ from synapse.media.media_storage import ( from synapse.media.storage_provider import StorageProviderWrapper from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID @@ -110,7 +109,7 @@ class MediaRepository: self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails self.thumbnail_requirements = hs.config.media.thumbnail_requirements - self.remote_media_linearizer = Linearizer(name="media_remote") + self.remote_media_linearizer = Linearizer(name="media_remote", clock=self.clock) self.recently_accessed_remotes: Set[Tuple[str, str]] = set() self.recently_accessed_locals: Set[str] = set() @@ -189,16 +188,14 @@ class MediaRepository: self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository def _start_update_recently_accessed(self) -> Deferred: - return run_as_background_process( + return self.hs.run_as_background_process( "update_recently_accessed_media", - self.server_name, self._update_recently_accessed, ) def _start_apply_media_retention_rules(self) -> Deferred: - return run_as_background_process( + return self.hs.run_as_background_process( "apply_media_retention_rules", - self.server_name, self._apply_media_retention_rules, ) diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 81204913f7..1a82cc46e3 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -44,7 +44,6 @@ from synapse.media._base import FileInfo, get_filename_from_headers from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter from synapse.media.oembed import OEmbedProvider from synapse.media.preview_html import decode_body, parse_html_to_open_graph -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, UserID from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.expiringcache import ExpiringCache @@ -167,6 +166,7 @@ class UrlPreviewer: media_storage: MediaStorage, ): self.clock = hs.get_clock() + self.hs = hs self.filepaths = media_repo.filepaths self.max_spider_size = hs.config.media.max_spider_size self.server_name = hs.hostname @@ -201,15 +201,14 @@ class UrlPreviewer: self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( cache_name="url_previews", server_name=self.server_name, + hs=self.hs, clock=self.clock, # don't spider URLs more often than once an hour expiry_ms=ONE_HOUR, ) if self._worker_run_media_background_jobs: - self._cleaner_loop = self.clock.looping_call( - self._start_expire_url_cache_data, 10 * 1000 - ) + self.clock.looping_call(self._start_expire_url_cache_data, 10 * 1000) async def preview(self, url: str, user: UserID, ts: int) -> bytes: # the in-memory cache: @@ -739,8 +738,8 @@ class UrlPreviewer: return open_graph_result, oembed_response.author_name, expiration_ms def _start_expire_url_cache_data(self) -> Deferred: - return run_as_background_process( - "expire_url_cache_data", self.server_name, self._expire_url_cache_data + return self.hs.run_as_background_process( + "expire_url_cache_data", self._expire_url_cache_data ) async def _expire_url_cache_data(self) -> None: diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py index e7783b05e6..1da871f18f 100644 --- a/synapse/metrics/_gc.py +++ b/synapse/metrics/_gc.py @@ -138,7 +138,9 @@ def install_gc_manager() -> None: gc_time.labels(i).observe(end - start) gc_unreachable.labels(i).set(unreachable) - gc_task = task.LoopingCall(_maybe_gc) + # We can ignore the lint here since this looping call does not hold a `HomeServer` + # reference so can be cleaned up by other means on shutdown. + gc_task = task.LoopingCall(_maybe_gc) # type: ignore[prefer-synapse-clock-looping-call] gc_task.start(0.1) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 633705b02a..05e84038ac 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -20,7 +20,7 @@ import logging import threading -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext from functools import wraps from types import TracebackType from typing import ( @@ -28,7 +28,9 @@ from typing import ( Any, Awaitable, Callable, + ContextManager, Dict, + Generator, Iterable, Optional, Protocol, @@ -49,7 +51,12 @@ from synapse.logging.context import ( LoggingContext, PreserveLoggingContext, ) -from synapse.logging.opentracing import SynapseTags, start_active_span +from synapse.logging.opentracing import ( + SynapseTags, + active_span, + start_active_span, + start_active_span_follows_from, +) from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics._types import Collector @@ -59,6 +66,13 @@ if TYPE_CHECKING: # Old versions don't have `LiteralString` from typing_extensions import LiteralString + from synapse.server import HomeServer + + try: + import opentracing + except ImportError: + opentracing = None # type: ignore[assignment] + logger = logging.getLogger(__name__) @@ -216,6 +230,7 @@ def run_as_background_process( func: Callable[..., Awaitable[Optional[R]]], *args: Any, bg_start_span: bool = True, + test_only_tracer: Optional["opentracing.Tracer"] = None, **kwargs: Any, ) -> "defer.Deferred[Optional[R]]": """Run the given function in its own logcontext, with resource metrics @@ -241,6 +256,8 @@ def run_as_background_process( bg_start_span: Whether to start an opentracing span. Defaults to True. Should only be disabled for processes that will not log to or tag a span. + test_only_tracer: Set the OpenTracing tracer to use. This is only useful for + tests. args: positional args for func kwargs: keyword args for func @@ -250,6 +267,12 @@ def run_as_background_process( rules. """ + # Since we track the tracing scope in the `LoggingContext`, before we move to the + # sentinel logcontext (or a new `LoggingContext`), grab the currently active + # tracing span (if any) so that we can create a cross-link to the background process + # trace. + original_active_tracing_span = active_span(tracer=test_only_tracer) + async def run() -> Optional[R]: with _bg_metrics_lock: count = _background_process_counts.get(desc, 0) @@ -264,15 +287,101 @@ def run_as_background_process( with BackgroundProcessLoggingContext( name=desc, server_name=server_name, instance_id=count - ) as context: + ) as logging_context: try: if bg_start_span: - ctx = start_active_span( - f"bgproc.{desc}", tags={SynapseTags.REQUEST_ID: str(context)} - ) + # If there is already an active span (e.g. because this background + # process was started as part of handling a request for example), + # because this is a long-running background task that may serve a + # broader purpose than the request that kicked it off, we don't want + # it to be a direct child of the currently active trace connected to + # the request. We only want a loose reference to jump between the + # traces. + # + # For example, when making a `/messages` request, when approaching a + # gap, we may kick off a background process to fetch missing events + # from federation. The `/messages` request trace should't include + # the entire time taken and details around fetching the missing + # events since the request doesn't rely on the result, it was just + # part of the heuristic to initiate things. + # + # We don't care about the value from the context manager as it's not + # used (so we just use `Any` for the type). Ideally, we'd be able to + # mark this as unused like an `assert_never` of sorts. + tracing_scope: ContextManager[Any] + if original_active_tracing_span is not None: + # With the OpenTracing client that we're using, it's impossible to + # create a disconnected root span while also providing `references` + # so we first create a bare root span, then create a child span that + # includes the references that we want. + root_tracing_scope = start_active_span( + f"bgproc.{desc}", + tags={SynapseTags.REQUEST_ID: str(logging_context)}, + # Create a root span for the background process (disconnected + # from other spans) + ignore_active_span=True, + tracer=test_only_tracer, + ) + + # Also add a span in the original request trace that cross-links + # to background process trace. We immediately finish the span as + # this is just a marker to follow where the real work is being + # done. + # + # In OpenTracing, `FOLLOWS_FROM` indicates parent-child + # relationship whereas we just want a cross-link to the + # downstream trace. This is a bit hacky, but the closest we + # can get to in OpenTracing land. If we ever migrate to + # OpenTelemetry, we should use a normal `Link` for this. + with start_active_span_follows_from( + f"start_bgproc.{desc}", + child_of=original_active_tracing_span, + ignore_active_span=True, + # Create the `FOLLOWS_FROM` reference to the background + # process span so there is a loose coupling between the two + # traces and it's easy to jump between. + contexts=[root_tracing_scope.span.context], + tracer=test_only_tracer, + ): + pass + + # Then start the tracing scope that we're going to use for + # the duration of the background process within the root + # span we just created. + child_tracing_scope = start_active_span_follows_from( + f"bgproc_child.{desc}", + child_of=root_tracing_scope.span, + ignore_active_span=True, + tags={SynapseTags.REQUEST_ID: str(logging_context)}, + # Create the `FOLLOWS_FROM` reference to the request's + # span so there is a loose coupling between the two + # traces and it's easy to jump between. + contexts=[original_active_tracing_span.context], + tracer=test_only_tracer, + ) + + # For easy usage down below, we create a context manager that + # combines both scopes. + @contextmanager + def combined_context_manager() -> Generator[None, None, None]: + with root_tracing_scope, child_tracing_scope: + yield + + tracing_scope = combined_context_manager() + + else: + # Otherwise, when there is no active span, we will be creating + # a disconnected root span already and we don't have to + # worry about cross-linking to anything. + tracing_scope = start_active_span( + f"bgproc.{desc}", + tags={SynapseTags.REQUEST_ID: str(logging_context)}, + tracer=test_only_tracer, + ) else: - ctx = nullcontext() # type: ignore[assignment] - with ctx: + tracing_scope = nullcontext() + + with tracing_scope: return await func(*args, **kwargs) except Exception: logger.exception( @@ -308,11 +417,11 @@ def run_as_background_process( P = ParamSpec("P") -class HasServerName(Protocol): - server_name: str +class HasHomeServer(Protocol): + hs: "HomeServer" """ - The homeserver name that this cache is associated with (used to label the metric) - (`hs.hostname`). + The homeserver that this cache is associated with (used to label the metric and + track backgroun processes for clean shutdown). """ @@ -342,27 +451,22 @@ def wrap_as_background_process( """ def wrapper( - func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]], + func: Callable[Concatenate[HasHomeServer, P], Awaitable[Optional[R]]], ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrapped_func( - self: HasServerName, *args: P.args, **kwargs: P.kwargs + self: HasHomeServer, *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - assert self.server_name is not None, ( - "The `server_name` attribute must be set on the object where `@wrap_as_background_process` decorator is used." + assert self.hs is not None, ( + "The `hs` attribute must be set on the object where `@wrap_as_background_process` decorator is used." ) - return run_as_background_process( + return self.hs.run_as_background_process( desc, - self.server_name, func, self, *args, - # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. - # Argument 4 to "run_as_background_process" has incompatible type - # "**P.kwargs"; expected "bool" - # See https://github.com/python/mypy/issues/8862 - **kwargs, # type: ignore[arg-type] + **kwargs, ) # There are some shenanigans here, because we're decorating a method but @@ -401,7 +505,7 @@ class BackgroundProcessLoggingContext(LoggingContext): """ if instance_id is None: instance_id = id(self) - super().__init__("%s-%s" % (name, instance_id)) + super().__init__(name="%s-%s" % (name, instance_id), server_name=server_name) self._proc: Optional[_BackgroundProcess] = _BackgroundProcess( desc=name, server_name=server_name, ctx=self ) diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index cd1c3c8649..43e0913d27 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -23,7 +23,6 @@ from typing import TYPE_CHECKING import attr from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process if TYPE_CHECKING: from synapse.server import HomeServer @@ -52,6 +51,7 @@ class CommonUsageMetricsManager: self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() + self._hs = hs async def get_metrics(self) -> CommonUsageMetrics: """Get the CommonUsageMetrics object. If no collection has happened yet, do it @@ -64,16 +64,14 @@ class CommonUsageMetricsManager: async def setup(self) -> None: """Keep the gauges for common usage metrics up to date.""" - run_as_background_process( + self._hs.run_as_background_process( desc="common_usage_metrics_update_gauges", - server_name=self.server_name, func=self._update_gauges, ) self._clock.looping_call( - run_as_background_process, + self._hs.run_as_background_process, 5 * 60 * 1000, desc="common_usage_metrics_update_gauges", - server_name=self.server_name, func=self._update_gauges, ) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 7a419145e0..12a31dd2ab 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -275,7 +275,15 @@ def run_as_background_process( # function instead. stub_server_name = "synapse_module_running_from_unknown_server" - return _run_as_background_process( + # Ignore the linter error here. Since this is leveraging the + # `run_as_background_process` function directly and we don't want to break the + # module api, we need to keep the function signature the same. This means we don't + # have access to the running `HomeServer` and cannot track this background process + # for cleanup during shutdown. + # This is not an issue during runtime and is only potentially problematic if the + # application cares about being able to garbage collect `HomeServer` instances + # during runtime. + return _run_as_background_process( # type: ignore[untracked-background-process] desc, stub_server_name, func, @@ -1402,7 +1410,7 @@ class ModuleApi: if self._hs.config.worker.run_background_tasks or run_on_all_instances: self._clock.looping_call( - self.run_as_background_process, + self._hs.run_as_background_process, msec, desc, lambda: maybe_awaitable(f(*args, **kwargs)), @@ -1460,7 +1468,7 @@ class ModuleApi: return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, - self.run_as_background_process, + self._hs.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), ) @@ -1701,8 +1709,8 @@ class ModuleApi: Note that the returned Deferred does not follow the synapse logcontext rules. """ - return _run_as_background_process( - desc, self.server_name, func, *args, bg_start_span=bg_start_span, **kwargs + return self._hs.run_as_background_process( + desc, func, *args, bg_start_span=bg_start_span, **kwargs ) async def defer_to_thread( diff --git a/synapse/notifier.py b/synapse/notifier.py index e684df4866..9169f50c4d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -676,9 +676,16 @@ class Notifier: # is a new token. listener = user_stream.new_listener(prev_token) listener = timeout_deferred( - listener, - (end_time - now) / 1000.0, - self.hs.get_reactor(), + deferred=listener, + timeout=(end_time - now) / 1000.0, + # We don't track these calls since they are constantly being + # overridden by new calls to /sync and they don't hold the + # `HomeServer` in memory on shutdown. It is safe to let them + # timeout of their own accord after shutting down since it + # won't delay shutdown and there won't be any adverse + # behaviour. + cancel_on_shutdown=False, + clock=self.hs.get_clock(), ) log_kv( diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 09ca14584a..1484bc8fc0 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -25,7 +25,6 @@ from typing import TYPE_CHECKING, Dict, List, Optional from twisted.internet.error import AlreadyCalled, AlreadyCancelled from twisted.internet.interfaces import IDelayedCall -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.push import Pusher, PusherConfig, PusherConfigException, ThrottleParams from synapse.push.mailer import Mailer from synapse.push.push_types import EmailReason @@ -118,7 +117,7 @@ class EmailPusher(Pusher): if self._is_processing: return - run_as_background_process("emailpush.process", self.server_name, self._process) + self.hs.run_as_background_process("emailpush.process", self._process) def _pause_processing(self) -> None: """Used by tests to temporarily pause processing of events. @@ -228,8 +227,10 @@ class EmailPusher(Pusher): self.timed_call = None if soonest_due_at is not None: - self.timed_call = self.hs.get_reactor().callLater( - self.seconds_until(soonest_due_at), self.on_timer + delay = self.seconds_until(soonest_due_at) + self.timed_call = self.hs.get_clock().call_later( + delay, + self.on_timer, ) async def save_last_stream_ordering_and_success( diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5946a6e972..5cac5de8cb 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -32,7 +32,6 @@ from synapse.api.constants import EventTypes from synapse.events import EventBase from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.storage.databases.main.event_push_actions import HttpPushAction from synapse.types import JsonDict, JsonMapping @@ -182,8 +181,8 @@ class HttpPusher(Pusher): # We could check the receipts are actually m.read receipts here, # but currently that's the only type of receipt anyway... - run_as_background_process( - "http_pusher.on_new_receipts", self.server_name, self._update_badge + self.hs.run_as_background_process( + "http_pusher.on_new_receipts", self._update_badge ) async def _update_badge(self) -> None: @@ -219,7 +218,7 @@ class HttpPusher(Pusher): if self.failing_since and self.timed_call and self.timed_call.active(): return - run_as_background_process("httppush.process", self.server_name, self._process) + self.hs.run_as_background_process("httppush.process", self._process) async def _process(self) -> None: # we should never get here if we are already processing @@ -336,8 +335,9 @@ class HttpPusher(Pusher): ) else: logger.info("Push failed: delaying for %ds", self.backoff_delay) - self.timed_call = self.hs.get_reactor().callLater( - self.backoff_delay, self.on_timer + self.timed_call = self.hs.get_clock().call_later( + self.backoff_delay, + self.on_timer, ) self.backoff_delay = min( self.backoff_delay * 2, self.MAX_BACKOFF_SEC diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d1f79ec999..977c55b683 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -27,7 +27,6 @@ from prometheus_client import Gauge from synapse.api.errors import Codes, SynapseError from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.push import Pusher, PusherConfig, PusherConfigException @@ -70,10 +69,8 @@ class PusherPool: """ def __init__(self, hs: "HomeServer"): - self.hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.pusher_factory = PusherFactory(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() @@ -112,9 +109,7 @@ class PusherPool: if not self._should_start_pushers: logger.info("Not starting pushers because they are disabled in the config") return - run_as_background_process( - "start_pushers", self.server_name, self._start_pushers - ) + self.hs.run_as_background_process("start_pushers", self._start_pushers) async def add_or_update_pusher( self, diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index 974d83bb8b..94981e22eb 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -185,46 +185,6 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): return 200, multi_user_devices -# FIXME(2025-07-22): Remove this on the next release, this will only get used -# during rollout to Synapse 1.135 and can be removed after that release. -class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): - """Unused endpoint, kept for backwards compatibility during rollout.""" - - NAME = "upload_keys_for_user" - PATH_ARGS = () - CACHE = False - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self.e2e_keys_handler = hs.get_e2e_keys_handler() - self.store = hs.get_datastores().main - self.clock = hs.get_clock() - - @staticmethod - async def _serialize_payload( # type: ignore[override] - user_id: str, device_id: str, keys: JsonDict - ) -> JsonDict: - return { - "user_id": user_id, - "device_id": device_id, - "keys": keys, - } - - async def _handle_request( # type: ignore[override] - self, request: Request, content: JsonDict - ) -> Tuple[int, JsonDict]: - user_id = content["user_id"] - device_id = content["device_id"] - keys = content["keys"] - - results = await self.e2e_keys_handler.upload_keys_for_user( - user_id, device_id, keys - ) - - return 200, results - - class ReplicationHandleNewDeviceUpdateRestServlet(ReplicationEndpoint): """Wake up a device writer to send local device list changes as federation outbound pokes. @@ -291,5 +251,4 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ReplicationNotifyUserSignatureUpdateRestServlet(hs).register(http_server) ReplicationMultiUserDevicesResyncRestServlet(hs).register(http_server) ReplicationHandleNewDeviceUpdateRestServlet(hs).register(http_server) - ReplicationUploadKeysForUserRestServlet(hs).register(http_server) ReplicationDeviceHandleRoomUnPartialStated(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 7a86b2e65e..f2561bc0c5 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -32,7 +32,6 @@ from synapse.api.constants import EventTypes, Membership, ReceiptTypes from synapse.federation import send_queue from synapse.federation.sender import FederationSender from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.streams import ( AccountDataStream, DeviceListsStream, @@ -344,7 +343,9 @@ class ReplicationDataHandler: # to wedge here forever. deferred: "Deferred[None]" = Deferred() deferred = timeout_deferred( - deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._reactor + deferred=deferred, + timeout=_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, + clock=self._clock, ) waiting_list = self._streams_to_waiters.setdefault( @@ -436,7 +437,9 @@ class FederationSenderHandler: # to. This is always set before we use it. self.federation_position: Optional[int] = None - self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer") + self._fed_position_linearizer = Linearizer( + name="_fed_position_linearizer", clock=hs.get_clock() + ) async def process_replication_rows( self, stream_name: str, token: int, rows: list @@ -511,8 +514,8 @@ class FederationSenderHandler: # no need to queue up another task. return - run_as_background_process( - "_save_and_send_ack", self.server_name, self._save_and_send_ack + self._hs.run_as_background_process( + "_save_and_send_ack", self._save_and_send_ack ) async def _save_and_send_ack(self) -> None: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index dd7e38dd78..4d0d3d44ab 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -41,7 +41,6 @@ from prometheus_client import Counter from twisted.internet.protocol import ReconnectingClientFactory from synapse.metrics import SERVER_NAME_LABEL, LaterGauge -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import ( ClearUserSyncsCommand, Command, @@ -132,6 +131,7 @@ class ReplicationCommandHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self._replication_data_handler = hs.get_replication_data_handler() self._presence_handler = hs.get_presence_handler() self._store = hs.get_datastores().main @@ -361,9 +361,8 @@ class ReplicationCommandHandler: return # fire off a background process to start processing the queue. - run_as_background_process( + self.hs.run_as_background_process( "process-replication-data", - self.server_name, self._unsafe_process_queue, stream_name, ) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 25a7868cd7..bcfc65c2c0 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -42,7 +42,6 @@ from synapse.logging.context import PreserveLoggingContext from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, - run_as_background_process, ) from synapse.replication.tcp.commands import ( VALID_CLIENT_COMMANDS, @@ -140,9 +139,14 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): max_line_buffer = 10000 def __init__( - self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" + self, + hs: "HomeServer", + server_name: str, + clock: Clock, + handler: "ReplicationCommandHandler", ): self.server_name = server_name + self.hs = hs self.clock = clock self.command_handler = handler @@ -290,9 +294,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): # if so. if isawaitable(res): - run_as_background_process( + self.hs.run_as_background_process( "replication-" + cmd.get_logcontext_id(), - self.server_name, lambda: res, ) @@ -470,9 +473,13 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): VALID_OUTBOUND_COMMANDS = VALID_SERVER_COMMANDS def __init__( - self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" + self, + hs: "HomeServer", + server_name: str, + clock: Clock, + handler: "ReplicationCommandHandler", ): - super().__init__(server_name, clock, handler) + super().__init__(hs, server_name, clock, handler) self.server_name = server_name @@ -497,7 +504,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): clock: Clock, command_handler: "ReplicationCommandHandler", ): - super().__init__(server_name, clock, command_handler) + super().__init__(hs, server_name, clock, command_handler) self.client_name = client_name self.server_name = server_name diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index aba79b2378..caffb2913e 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -40,7 +40,6 @@ from synapse.logging.context import PreserveLoggingContext, make_deferred_yielda from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.commands import ( @@ -109,6 +108,7 @@ class RedisSubscriber(SubscriberProtocol): """ server_name: str + hs: "HomeServer" synapse_handler: "ReplicationCommandHandler" synapse_stream_prefix: str synapse_channel_names: List[str] @@ -146,9 +146,7 @@ class RedisSubscriber(SubscriberProtocol): def connectionMade(self) -> None: logger.info("Connected to redis") super().connectionMade() - run_as_background_process( - "subscribe-replication", self.server_name, self._send_subscribe - ) + self.hs.run_as_background_process("subscribe-replication", self._send_subscribe) async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we @@ -223,8 +221,8 @@ class RedisSubscriber(SubscriberProtocol): # if so. if isawaitable(res): - run_as_background_process( - "replication-" + cmd.get_logcontext_id(), self.server_name, lambda: res + self.hs.run_as_background_process( + "replication-" + cmd.get_logcontext_id(), lambda: res ) def connectionLost(self, reason: Failure) -> None: # type: ignore[override] @@ -245,11 +243,17 @@ class RedisSubscriber(SubscriberProtocol): Args: cmd: The command to send """ - run_as_background_process( + self.hs.run_as_background_process( "send-cmd", - self.server_name, self._async_send_command, cmd, + # We originally started tracing background processes to avoid `There was no + # active span` errors but this change meant we started generating 15x the + # number of spans than before (this is one of the most heavily called + # instances of `run_as_background_process`). + # + # Since we don't log or tag a tracing span in the downstream + # code, we can safely disable this. bg_start_span=False, ) @@ -310,9 +314,8 @@ class SynapseRedisFactory(RedisFactory): convertNumbers=convertNumbers, ) - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname hs.get_clock().looping_call(self._send_ping, 30 * 1000) @@ -390,6 +393,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): ) self.server_name = hs.hostname + self.hs = hs self.synapse_handler = hs.get_replication_command_handler() self.synapse_stream_prefix = hs.hostname self.synapse_channel_names = channel_names @@ -405,6 +409,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): # the base method does some other things than just instantiating the # protocol. p.server_name = self.server_name + p.hs = self.hs p.synapse_handler = self.synapse_handler p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection p.synapse_stream_prefix = self.synapse_stream_prefix diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index d800cfe6f6..ef72a0a532 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -30,7 +30,6 @@ from twisted.internet.interfaces import IAddress from twisted.internet.protocol import ServerFactory from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import PositionCommand from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol from synapse.replication.tcp.streams import EventsStream @@ -55,6 +54,7 @@ class ReplicationStreamProtocolFactory(ServerFactory): def __init__(self, hs: "HomeServer"): self.command_handler = hs.get_replication_command_handler() self.clock = hs.get_clock() + self.hs = hs self.server_name = hs.config.server.server_name # If we've created a `ReplicationStreamProtocolFactory` then we're @@ -69,7 +69,7 @@ class ReplicationStreamProtocolFactory(ServerFactory): def buildProtocol(self, addr: IAddress) -> ServerReplicationStreamProtocol: return ServerReplicationStreamProtocol( - self.server_name, self.clock, self.command_handler + self.hs, self.server_name, self.clock, self.command_handler ) @@ -82,6 +82,7 @@ class ReplicationStreamer: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() @@ -147,8 +148,8 @@ class ReplicationStreamer: logger.debug("Notifier poke loop already running") return - run_as_background_process( - "replication_notifier", self.server_name, self._run_notifier_loop + self.hs.run_as_background_process( + "replication_notifier", self._run_notifier_loop ) async def _run_notifier_loop(self) -> None: diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 25c15e5d48..87ac0a5ae1 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -77,6 +77,7 @@ STREAMS_MAP = { __all__ = [ "STREAMS_MAP", "Stream", + "EventsStream", "BackfillStream", "PresenceStream", "PresenceFederationStream", @@ -87,6 +88,7 @@ __all__ = [ "CachesStream", "DeviceListsStream", "ToDeviceStream", + "FederationStream", "AccountDataStream", "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", diff --git a/synapse/res/providers.json b/synapse/res/providers.json index 2dc9fec8e3..48ad5dabda 100644 --- a/synapse/res/providers.json +++ b/synapse/res/providers.json @@ -1,14 +1,15 @@ [ { - "provider_name": "Twitter", - "provider_url": "http://www.twitter.com/", + "provider_name": "X", + "provider_url": "https://x.com/", "endpoints": [ { "schemes": [ - "https://twitter.com/*/moments/*", - "https://*.twitter.com/*/moments/*" + "https://x.com/*", + "https://x.com/*/status/*", + "https://*.x.com/*/status/*" ], - "url": "https://publish.twitter.com/oembed" + "url": "https://publish.x.com/oembed" } ] }, diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index c20de89bf7..c1939a9b57 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -65,7 +65,7 @@ class PushRuleRestServlet(RestServlet): hs.get_instance_name() in hs.config.worker.writers.push_rules ) self._push_rules_handler = hs.get_push_rules_handler() - self._push_rule_linearizer = Linearizer(name="push_rules") + self._push_rule_linearizer = Linearizer(name="push_rules", clock=hs.get_clock()) async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if not self._is_push_worker: diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 64deae7650..1084139df0 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -66,7 +66,6 @@ from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client._base import client_patterns from synapse.rest.client.transactions import HttpTransactionCache from synapse.state import CREATE_KEY, POWER_KEY @@ -1225,6 +1224,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.server_name = hs.hostname + self.hs = hs self.event_creation_handler = hs.get_event_creation_handler() self.auth = hs.get_auth() self._store = hs.get_datastores().main @@ -1307,9 +1307,8 @@ class RoomRedactEventRestServlet(TransactionRestServlet): ) if with_relations: - run_as_background_process( + self.hs.run_as_background_process( "redact_related_events", - self.server_name, self._relation_handler.redact_events_related_to, requester=requester, event_id=event_id, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index bb63b51599..01868de60b 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -126,6 +126,7 @@ class SyncRestServlet(RestServlet): self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, + clock=self.clock, cache_name="sync_valid_filter", server_name=self.server_name, ) @@ -363,9 +364,6 @@ class SyncRestServlet(RestServlet): # https://github.com/matrix-org/matrix-doc/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md # states that this field should always be included, as long as the server supports the feature. - response["org.matrix.msc2732.device_unused_fallback_key_types"] = ( - sync_result.device_unused_fallback_key_types - ) response["device_unused_fallback_key_types"] = ( sync_result.device_unused_fallback_key_types ) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 1a57996aec..571ba2fa62 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -56,7 +56,7 @@ class HttpTransactionCache: ] = {} # Try to clean entries every 30 mins. This means entries will exist # for at *LEAST* 30 mins, and at *MOST* 60 mins. - self.cleaner = self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS) + self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS) def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable: """A helper function which returns a transaction key that can be used diff --git a/synapse/server.py b/synapse/server.py index 118dee7051..cc0d3a427b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,10 +28,27 @@ import abc import functools import logging -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, TypeVar, cast +from threading import Thread +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, + cast, +) +from wsgiref.simple_server import WSGIServer +from attr import dataclass from typing_extensions import TypeAlias +from twisted.internet import defer +from twisted.internet.base import _SystemEventID from twisted.internet.interfaces import IOpenSSLContextFactory from twisted.internet.tcp import Port from twisted.python.threadpool import ThreadPool @@ -44,6 +61,7 @@ from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.auth_blocking import AuthBlocking from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter, RequestRatelimiter +from synapse.app._base import unregister_sighups from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler from synapse.config.homeserver import HomeServerConfig @@ -133,6 +151,7 @@ from synapse.metrics import ( all_later_gauges_to_clean_up_on_shutdown, register_threadpool, ) +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager from synapse.module_api import ModuleApi from synapse.module_api.callbacks import ModuleApiCallbacks @@ -156,6 +175,7 @@ from synapse.storage.controllers import StorageControllers from synapse.streams.events import EventSources from synapse.synapse_rust.rendezvous import RendezvousHandler from synapse.types import DomainSpecificString, ISynapseReactor +from synapse.util.caches import CACHE_METRIC_REGISTRY from synapse.util.clock import Clock from synapse.util.distributor import Distributor from synapse.util.macaroons import MacaroonGenerator @@ -166,7 +186,9 @@ from synapse.util.task_scheduler import TaskScheduler logger = logging.getLogger(__name__) if TYPE_CHECKING: + # Old Python versions don't have `LiteralString` from txredisapi import ConnectionHandler + from typing_extensions import LiteralString from synapse.handlers.jwt import JwtHandler from synapse.handlers.oidc import OidcHandler @@ -196,6 +218,7 @@ if TYPE_CHECKING: T: TypeAlias = object F = TypeVar("F", bound=Callable[["HomeServer"], T]) +R = TypeVar("R") def cache_in_self(builder: F) -> F: @@ -219,7 +242,8 @@ def cache_in_self(builder: F) -> F: @functools.wraps(builder) def _get(self: "HomeServer") -> T: try: - return getattr(self, depname) + dep = getattr(self, depname) + return dep except AttributeError: pass @@ -239,6 +263,22 @@ def cache_in_self(builder: F) -> F: return cast(F, _get) +@dataclass +class ShutdownInfo: + """Information for callable functions called at time of shutdown. + + Attributes: + func: the object to call before shutdown. + trigger_id: an ID returned when registering this event trigger. + args: the arguments to call the function with. + kwargs: the keyword arguments to call the function with. + """ + + func: Callable[..., Any] + trigger_id: _SystemEventID + kwargs: Dict[str, object] + + class HomeServer(metaclass=abc.ABCMeta): """A basic homeserver object without lazy component builders. @@ -289,6 +329,7 @@ class HomeServer(metaclass=abc.ABCMeta): hostname : The hostname for the server. config: The full config for the homeserver. """ + if not reactor: from twisted.internet import reactor as _reactor @@ -300,6 +341,7 @@ class HomeServer(metaclass=abc.ABCMeta): self.signing_key = config.key.signing_key[0] self.config = config self._listening_services: List[Port] = [] + self._metrics_listeners: List[Tuple[WSGIServer, Thread]] = [] self.start_time: Optional[int] = None self._instance_id = random_string(5) @@ -315,6 +357,211 @@ class HomeServer(metaclass=abc.ABCMeta): # This attribute is set by the free function `refresh_certificate`. self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None + self._is_shutdown = False + self._async_shutdown_handlers: List[ShutdownInfo] = [] + self._sync_shutdown_handlers: List[ShutdownInfo] = [] + self._background_processes: set[defer.Deferred[Optional[Any]]] = set() + + def run_as_background_process( + self, + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[R]]], + *args: Any, + **kwargs: Any, + ) -> "defer.Deferred[Optional[R]]": + """Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + It returns a Deferred which completes when the function completes, but it doesn't + follow the synapse logcontext rules, which makes it appropriate for passing to + clock.looping_call and friends (or for firing-and-forgetting in the middle of a + normal synapse async function). + + Because the returned Deferred does not follow the synapse logcontext rules, awaiting + the result of this function will result in the log context being cleared (bad). In + order to properly await the result of this function and maintain the current log + context, use `make_deferred_yieldable`. + + Args: + desc: a description for this background process type + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). + func: a function, which may return a Deferred or a coroutine + bg_start_span: Whether to start an opentracing span. Defaults to True. + Should only be disabled for processes that will not log to or tag + a span. + args: positional args for func + kwargs: keyword args for func + + Returns: + Deferred which returns the result of func, or `None` if func raises. + Note that the returned Deferred does not follow the synapse logcontext + rules. + """ + if self._is_shutdown: + raise Exception( + f"Cannot start background process. HomeServer has been shutdown {len(self._background_processes)} {len(self.get_clock()._looping_calls)} {len(self.get_clock()._call_id_to_delayed_call)}" + ) + + # Ignore linter error as this is the one location this should be called. + deferred = run_as_background_process(desc, self.hostname, func, *args, **kwargs) # type: ignore[untracked-background-process] + self._background_processes.add(deferred) + + def on_done(res: R) -> R: + try: + self._background_processes.remove(deferred) + except KeyError: + # If the background process isn't being tracked anymore we can just move on. + pass + return res + + deferred.addBoth(on_done) + return deferred + + async def shutdown(self) -> None: + """ + Cleanly stops all aspects of the HomeServer and removes any references that + have been handed out in order to allow the HomeServer object to be garbage + collected. + + You must ensure the HomeServer object to not be frozen in the garbage collector + in order for it to be cleaned up. By default, Synapse freezes the HomeServer + object in the garbage collector. + """ + + self._is_shutdown = True + + logger.info( + "Received shutdown request for %s (%s).", + self.hostname, + self.get_instance_id(), + ) + + # Unregister sighups first. If a shutdown was requested we shouldn't be responding + # to things like config changes. So it would be best to stop listening to these first. + unregister_sighups(self._instance_id) + + # TODO: It would be desireable to be able to report an error if the HomeServer + # object is frozen in the garbage collector as that would prevent it from being + # collected after being shutdown. + # In theory the following should work, but it doesn't seem to make a difference + # when I test it locally. + # + # if gc.is_tracked(self): + # logger.error("HomeServer object is tracked by garbage collection so cannot be fully cleaned up") + + for listener in self._listening_services: + # During unit tests, an incomplete `twisted.pair.testing._FakePort` is used + # for listeners so check listener type here to ensure shutdown procedure is + # only applied to actual `Port` instances. + if type(listener) is Port: + port_shutdown = listener.stopListening() + if port_shutdown is not None: + await port_shutdown + self._listening_services.clear() + + for server, thread in self._metrics_listeners: + server.shutdown() + thread.join() + self._metrics_listeners.clear() + + # TODO: Cleanup replication pieces + + self.get_keyring().shutdown() + + # Cleanup metrics associated with the homeserver + for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): + later_gauge.unregister_hooks_for_homeserver_instance_id( + self.get_instance_id() + ) + + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver( + self.config.server.server_name + ) + + for db in self.get_datastores().databases: + db.stop_background_updates() + + if self.should_send_federation(): + try: + self.get_federation_sender().shutdown() + except Exception: + pass + + for shutdown_handler in self._async_shutdown_handlers: + try: + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + defer.ensureDeferred(shutdown_handler.func(**shutdown_handler.kwargs)) + except Exception as e: + logger.error("Error calling shutdown async handler: %s", e) + self._async_shutdown_handlers.clear() + + for shutdown_handler in self._sync_shutdown_handlers: + try: + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + shutdown_handler.func(**shutdown_handler.kwargs) + except Exception as e: + logger.error("Error calling shutdown sync handler: %s", e) + self._sync_shutdown_handlers.clear() + + self.get_clock().shutdown() + + for background_process in list(self._background_processes): + try: + background_process.cancel() + except Exception: + pass + self._background_processes.clear() + + for db in self.get_datastores().databases: + db._db_pool.close() + + def register_async_shutdown_handler( + self, + *, + phase: str, + eventType: str, + shutdown_func: Callable[..., Any], + **kwargs: object, + ) -> None: + """ + Register a system event trigger with the HomeServer so it can be cleanly + removed when the HomeServer is shutdown. + """ + id = self.get_clock().add_system_event_trigger( + phase, + eventType, + shutdown_func, + **kwargs, + ) + self._async_shutdown_handlers.append( + ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs) + ) + + def register_sync_shutdown_handler( + self, + *, + phase: str, + eventType: str, + shutdown_func: Callable[..., Any], + **kwargs: object, + ) -> None: + """ + Register a system event trigger with the HomeServer so it can be cleanly + removed when the HomeServer is shutdown. + """ + id = self.get_clock().add_system_event_trigger( + phase, + eventType, + shutdown_func, + **kwargs, + ) + self._sync_shutdown_handlers.append( + ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs) + ) + def register_module_web_resource(self, path: str, resource: Resource) -> None: """Allows a module to register a web resource to be served at the given path. @@ -366,36 +613,25 @@ class HomeServer(metaclass=abc.ABCMeta): self.datastores = Databases(self.DATASTORE_CLASS, self) logger.info("Finished setting up.") - def __del__(self) -> None: - """ - Called when an the homeserver is garbage collected. + # Register background tasks required by this server. This must be done + # somewhat manually due to the background tasks not being registered + # unless handlers are instantiated. + if self.config.worker.run_background_tasks: + self.start_background_tasks() - Make sure we actually do some clean-up, rather than leak data. - """ - self.cleanup() - - def cleanup(self) -> None: - """ - WIP: Clean-up any references to the homeserver and stop any running related - processes, timers, loops, replication stream, etc. - - This should be called wherever you care about the HomeServer being completely - garbage collected like in tests. It's not necessary to call if you plan to just - shut down the whole Python process anyway. - - Can be called multiple times. - """ - logger.info("Received cleanup request for %s.", self.hostname) - - # TODO: Stop background processes, timers, loops, replication stream, etc. - - # Cleanup metrics associated with the homeserver - for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): - later_gauge.unregister_hooks_for_homeserver_instance_id( - self.get_instance_id() - ) - - logger.info("Cleanup complete for %s.", self.hostname) + # def __del__(self) -> None: + # """ + # Called when an the homeserver is garbage collected. + # + # Make sure we actually do some clean-up, rather than leak data. + # """ + # + # # NOTE: This is a chicken and egg problem. + # # __del__ will never be called since the HomeServer cannot be garbage collected + # # until the shutdown function has been called. So it makes no sense to call + # # shutdown inside of __del__, even though that is a logical place to assume it + # # should be called. + # self.shutdown() def start_listening(self) -> None: # noqa: B027 (no-op by design) """Start the HTTP, manhole, metrics, etc listeners @@ -442,7 +678,8 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_clock(self) -> Clock: - return Clock(self._reactor) + # Ignore the linter error since this is the one place the `Clock` should be created. + return Clock(self._reactor, server_name=self.hostname) # type: ignore[multiple-internal-clocks] def get_datastores(self) -> Databases: if not self.datastores: @@ -452,7 +689,7 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_distributor(self) -> Distributor: - return Distributor(server_name=self.hostname) + return Distributor(hs=self) @cache_in_self def get_registration_ratelimiter(self) -> Ratelimiter: @@ -1007,8 +1244,10 @@ class HomeServer(metaclass=abc.ABCMeta): ) media_threadpool.start() - self.get_clock().add_system_event_trigger( - "during", "shutdown", media_threadpool.stop + self.register_sync_shutdown_handler( + phase="during", + eventType="shutdown", + shutdown_func=media_threadpool.stop, ) # Register the threadpool with our metrics. diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 19f86b5a56..73cf4091eb 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -36,6 +36,7 @@ SERVER_NOTICE_ROOM_TAG = "m.server_notice" class ServerNoticesManager: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() # nb must be called this for @cached self._store = hs.get_datastores().main self._config = hs.config self._account_data_handler = hs.get_account_data_handler() diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 3d8016c264..394dc72fa6 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -642,13 +642,16 @@ class StateResolutionHandler: self.server_name = hs.hostname self.clock = hs.get_clock() - self.resolve_linearizer = Linearizer(name="state_resolve_lock") + self.resolve_linearizer = Linearizer( + name="state_resolve_lock", clock=self.clock + ) # dict of set of event_ids -> _StateCacheEntry. self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( cache_name="state_cache", server_name=self.server_name, + hs=hs, clock=self.clock, max_len=100000, expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f214f55897..1fddcc0799 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -56,7 +56,7 @@ class SQLBaseStore(metaclass=ABCMeta): ): self.hs = hs self.server_name = hs.hostname # nb must be called this for @cached - self._clock = hs.get_clock() + self.clock = hs.get_clock() # nb must be called this for @cached self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 9aa9e51aeb..e3e793d5f5 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -41,7 +41,6 @@ from typing import ( import attr from synapse._pydantic_compat import BaseModel -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine from synapse.storage.types import Connection, Cursor from synapse.types import JsonDict, StrCollection @@ -285,6 +284,13 @@ class BackgroundUpdater: self.sleep_duration_ms = hs.config.background_updates.sleep_duration_ms self.sleep_enabled = hs.config.background_updates.sleep_enabled + def shutdown(self) -> None: + """ + Stop any further background updates from happening. + """ + self.enabled = False + self._background_update_handlers.clear() + def get_status(self) -> UpdaterStatus: """An integer summarising the updater status. Used as a metric.""" if self._aborted: @@ -396,9 +402,8 @@ class BackgroundUpdater: # if we start a new background update, not all updates are done. self._all_done = False sleep = self.sleep_enabled - run_as_background_process( + self.hs.run_as_background_process( "background_updates", - self.server_name, self.run_background_updates, sleep, ) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 120934af57..646e2cf115 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -62,7 +62,6 @@ from synapse.logging.opentracing import ( trace, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.controllers.state import StateStorageController from synapse.storage.databases import Databases from synapse.storage.databases.main.events import DeltaState @@ -195,6 +194,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): def __init__( self, + hs: "HomeServer", server_name: str, per_item_callback: Callable[ [str, _EventPersistQueueTask], @@ -207,6 +207,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): and its result will be returned via the Deferreds returned from add_to_queue. """ self.server_name = server_name + self.hs = hs self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} self._currently_persisting_rooms: Set[str] = set() self._per_item_callback = per_item_callback @@ -311,7 +312,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): self._currently_persisting_rooms.discard(room_id) # set handle_queue_loop off in the background - run_as_background_process("persist_events", self.server_name, handle_queue_loop) + self.hs.run_as_background_process("persist_events", handle_queue_loop) def _get_drainining_queue( self, room_id: str @@ -354,7 +355,7 @@ class EventsPersistenceStorageController: self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id self._event_persist_queue = _EventPeristenceQueue( - self.server_name, self._process_event_persist_queue_task + hs, self.server_name, self._process_event_persist_queue_task ) self._state_resolution_handler = hs.get_state_resolution_handler() self._state_controller = state_controller diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 14b37ac543..ded9cb0567 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -46,9 +46,8 @@ class PurgeEventsStorageController: """High level interface for purging rooms and event history.""" def __init__(self, hs: "HomeServer", stores: Databases): - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.stores = stores if hs.config.worker.run_background_tasks: diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index ad90a1be13..76978402b9 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -69,15 +69,17 @@ class StateStorageController: def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() self._is_mine_id = hs.is_mine_id - self._clock = hs.get_clock() self.stores = stores self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) self._partial_state_room_tracker = PartialCurrentStateTracker(stores.main) # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. - self._joined_host_linearizer = Linearizer("_JoinedHostsCache") + self._joined_host_linearizer = Linearizer( + name="_JoinedHostsCache", clock=self.clock + ) def notify_event_un_partial_stated(self, event_id: str) -> None: self._partial_state_events_tracker.notify_un_partial_stated(event_id) @@ -815,9 +817,7 @@ class StateStorageController: state_group = object() assert state_group is not None - with Measure( - self._clock, name="get_joined_hosts", server_name=self.server_name - ): + with Measure(self.clock, name="get_joined_hosts", server_name=self.server_name): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry ) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index aae029f910..a4b2b26795 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -62,7 +62,6 @@ from synapse.logging.context import ( make_deferred_yieldable, ) from synapse.metrics import SERVER_NAME_LABEL, register_threadpool -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdater from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.storage.types import Connection, Cursor, SQLQueryParameters @@ -146,7 +145,7 @@ def make_pool( def _on_new_connection(conn: Connection) -> None: # Ensure we have a logging context so we can correctly track queries, # etc. - with LoggingContext("db.on_new_connection"): + with LoggingContext(name="db.on_new_connection", server_name=server_name): engine.on_new_connection( LoggingDatabaseConnection( conn=conn, @@ -638,12 +637,17 @@ class DatabasePool: # background updates of tables that aren't safe to update. self._clock.call_later( 0.0, - run_as_background_process, + self.hs.run_as_background_process, "upsert_safety_check", - self.server_name, self._check_safe_to_upsert, ) + def stop_background_updates(self) -> None: + """ + Stops the database from running any further background updates. + """ + self.updates.shutdown() + def name(self) -> str: "Return the name of this database" return self._database_config.name @@ -681,9 +685,8 @@ class DatabasePool: if background_update_names: self._clock.call_later( 15.0, - run_as_background_process, + self.hs.run_as_background_process, "upsert_safety_check", - self.server_name, self._check_safe_to_upsert, ) @@ -1043,7 +1046,9 @@ class DatabasePool: assert not self.engine.in_transaction(conn) with LoggingContext( - str(curr_context), parent_context=parent_context + name=str(curr_context), + server_name=self.server_name, + parent_context=parent_context, ) as context: with opentracing.start_active_span( operation_name="db.connection", diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index cad26fefa4..674c6b921e 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -751,7 +751,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): "instance_name": self._instance_name, "cache_func": cache_name, "keys": keys, - "invalidation_ts": self._clock.time_msec(), + "invalidation_ts": self.clock.time_msec(), }, ) @@ -778,7 +778,7 @@ class CacheInvalidationWorkerStore(SQLBaseStore): assert self._cache_id_gen is not None stream_ids = self._cache_id_gen.get_next_mult_txn(txn, len(key_tuples)) - ts = self._clock.time_msec() + ts = self.clock.time_msec() txn.call_after(self.hs.get_notifier().on_new_replication_data) self.db_pool.simple_insert_many_txn( txn, @@ -830,7 +830,8 @@ class CacheInvalidationWorkerStore(SQLBaseStore): next_interval = REGULAR_CLEANUP_INTERVAL_MS self.hs.get_clock().call_later( - next_interval / 1000, self._clean_up_cache_invalidation_wrapper + next_interval / 1000, + self._clean_up_cache_invalidation_wrapper, ) async def _clean_up_batch_of_old_cache_invalidations( diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index 3f9f482add..45cfe97dba 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -77,7 +77,7 @@ class CensorEventsStore(EventsWorkerStore, CacheInvalidationWorkerStore, SQLBase return before_ts = ( - self._clock.time_msec() - self.hs.config.server.redaction_retention_period + self.clock.time_msec() - self.hs.config.server.redaction_retention_period ) # We fetch all redactions that: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index c7a330cc83..dc6ab99a6c 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -438,10 +438,11 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke cache_name="client_ip_last_seen", server_name=self.server_name, max_size=50000, + clock=hs.get_clock(), ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - self._clock.looping_call(self._prune_old_user_ips, 5 * 1000) + self.clock.looping_call(self._prune_old_user_ips, 5 * 1000) if self._update_on_this_worker: # This is the designated worker that can write to the client IP @@ -452,11 +453,11 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke Tuple[str, str, str], Tuple[str, Optional[str], int] ] = {} - self._client_ip_looper = self._clock.looping_call( - self._update_client_ips_batch, 5 * 1000 - ) - self.hs.get_clock().add_system_event_trigger( - "before", "shutdown", self._update_client_ips_batch + self.clock.looping_call(self._update_client_ips_batch, 5 * 1000) + hs.register_async_shutdown_handler( + phase="before", + eventType="shutdown", + shutdown_func=self._update_client_ips_batch, ) @wrap_as_background_process("prune_old_user_ips") @@ -492,7 +493,7 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke ) """ - timestamp = self._clock.time_msec() - self.user_ips_max_age + timestamp = self.clock.time_msec() - self.user_ips_max_age def _prune_old_user_ips_txn(txn: LoggingTransaction) -> None: txn.execute(sql, (timestamp,)) @@ -628,7 +629,7 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke return if not now: - now = int(self._clock.time_msec()) + now = int(self.clock.time_msec()) key = (user_id, access_token, ip) try: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index f6f3c94a0d..a66e11f738 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -96,7 +96,8 @@ class DeviceInboxWorkerStore(SQLBaseStore): ] = ExpiringCache( cache_name="last_device_delete_cache", server_name=self.server_name, - clock=self._clock, + hs=hs, + clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, ) @@ -154,7 +155,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( run_as_background_process, DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, "_delete_old_federation_inbox_rows", @@ -826,7 +827,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): ) async with self._to_device_msg_id_gen.get_next() as stream_id: - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() await self.db_pool.runInteraction( "add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id ) @@ -881,7 +882,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): ) async with self._to_device_msg_id_gen.get_next() as stream_id: - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() await self.db_pool.runInteraction( "add_messages_from_remote_to_device_inbox", add_messages_txn, @@ -1002,7 +1003,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): # We delete at most 100 rows that are older than # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS delete_before_ts = ( - self._clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + self.clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS ) sql = """ WITH to_delete AS ( @@ -1032,7 +1033,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): # We sleep a bit so that we don't hammer the database in a tight # loop first time we run this. - await self._clock.sleep(1) + await self.clock.sleep(1) async def get_devices_with_messages( self, user_id: str, device_ids: StrCollection diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index fc1e1c73f1..d4b9ce0ea0 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -195,7 +195,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self._prune_old_outbound_device_pokes, 60 * 60 * 1000 ) @@ -1390,7 +1390,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): table="device_lists_remote_resync", keyvalues={"user_id": user_id}, values={}, - insertion_values={"added_ts": self._clock.time_msec()}, + insertion_values={"added_ts": self.clock.time_msec()}, ) await self.db_pool.runInteraction( @@ -1601,7 +1601,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): that user when the destination comes back. It doesn't matter which device we keep. """ - yesterday = self._clock.time_msec() - prune_age + yesterday = self.clock.time_msec() - prune_age def _prune_txn(txn: LoggingTransaction) -> None: # look for (user, destination) pairs which have an update older than @@ -2086,7 +2086,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): stream_id, ) - now = self._clock.time_msec() + now = self.clock.time_msec() encoded_context = json_encoder.encode(context) mark_sent = not self.hs.is_mine_id(user_id) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 2e9f62075a..2d3d0c0036 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -1564,7 +1564,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker DELETE FROM e2e_one_time_keys_json WHERE {clause} AND ts_added_ms < ? AND length(key_id) = 6 """ - args.append(self._clock.time_msec() - (7 * 24 * 3600 * 1000)) + args.append(self.clock.time_msec() - (7 * 24 * 3600 * 1000)) txn.execute(sql, args) return users, txn.rowcount @@ -1585,7 +1585,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker None, if there is no such key. Otherwise, the timestamp before which replacement is allowed without UIA. """ - timestamp = self._clock.time_msec() + duration_ms + timestamp = self.clock.time_msec() + duration_ms def impl(txn: LoggingTransaction) -> Optional[int]: txn.execute( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 5c9bd2e848..d77420ff47 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -167,6 +167,7 @@ class EventFederationWorkerStore( # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( max_size=500000, + clock=self.hs.get_clock(), server_name=self.server_name, cache_name="_event_auth_cache", size_callback=len, @@ -176,7 +177,7 @@ class EventFederationWorkerStore( # index. self.tests_allow_no_chain_cover_index = True - self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) + self.clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) if isinstance(self.database_engine, PostgresEngine): self.db_pool.updates.register_background_validate_constraint_and_delete_rows( @@ -1328,7 +1329,7 @@ class EventFederationWorkerStore( ( room_id, current_depth, - self._clock.time_msec(), + self.clock.time_msec(), BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS, limit, @@ -1841,7 +1842,7 @@ class EventFederationWorkerStore( last_cause=EXCLUDED.last_cause; """ - txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec(), cause)) + txn.execute(sql, (room_id, event_id, 1, self.clock.time_msec(), cause)) @trace async def get_event_ids_with_failed_pull_attempts( @@ -1905,7 +1906,7 @@ class EventFederationWorkerStore( ), ) - current_time = self._clock.time_msec() + current_time = self.clock.time_msec() event_ids_with_backoff = {} for event_id, last_attempt_ts, num_attempts in event_failed_pull_attempts: @@ -2025,7 +2026,7 @@ class EventFederationWorkerStore( values={}, insertion_values={ "room_id": event.room_id, - "received_ts": self._clock.time_msec(), + "received_ts": self.clock.time_msec(), "event_json": json_encoder.encode(event.get_dict()), "internal_metadata": json_encoder.encode( event.internal_metadata.get_dict() @@ -2299,7 +2300,7 @@ class EventFederationWorkerStore( # If there is nothing in the staging area default it to 0. age = 0 if received_ts is not None: - age = self._clock.time_msec() - received_ts + age = self.clock.time_msec() - received_ts return count, age diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 4db0230421..ec26aedc6b 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -95,6 +95,8 @@ from typing import ( import attr +from twisted.internet.task import LoopingCall + from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause @@ -254,6 +256,8 @@ def _deserialize_action(actions: str, is_highlight: bool) -> List[Union[dict, st class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBaseStore): + _background_tasks: List[LoopingCall] = [] + def __init__( self, database: DatabasePool, @@ -263,7 +267,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas super().__init__(database, db_conn, hs) # Track when the process started. - self._started_ts = self._clock.time_msec() + self._started_ts = self.clock.time_msec() # These get correctly set by _find_stream_orderings_for_times_txn self.stream_ordering_month_ago: Optional[int] = None @@ -273,18 +277,14 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas self._find_stream_orderings_for_times_txn(cur) cur.close() - self.find_stream_orderings_looping_call = self._clock.looping_call( - self._find_stream_orderings_for_times, 10 * 60 * 1000 - ) + self.clock.looping_call(self._find_stream_orderings_for_times, 10 * 60 * 1000) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self._rotate_notif_loop = self._clock.looping_call( - self._rotate_notifs, 30 * 1000 - ) + self.clock.looping_call(self._rotate_notifs, 30 * 1000) - self._clear_old_staging_loop = self._clock.looping_call( + self.clock.looping_call( self._clear_old_push_actions_staging, 30 * 60 * 1000 ) @@ -1190,7 +1190,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas is_highlight, # highlight column int(count_as_unread), # unread column thread_id, # thread_id column - self._clock.time_msec(), # inserted_ts column + self.clock.time_msec(), # inserted_ts column ) await self.db_pool.simple_insert_many( @@ -1241,14 +1241,14 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas def _find_stream_orderings_for_times_txn(self, txn: LoggingTransaction) -> None: logger.info("Searching for stream ordering 1 month ago") self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn( - txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000 + txn, self.clock.time_msec() - 30 * 24 * 60 * 60 * 1000 ) logger.info( "Found stream ordering 1 month ago: it's %d", self.stream_ordering_month_ago ) logger.info("Searching for stream ordering 1 day ago") self.stream_ordering_day_ago = self._find_first_stream_ordering_after_ts_txn( - txn, self._clock.time_msec() - 24 * 60 * 60 * 1000 + txn, self.clock.time_msec() - 24 * 60 * 60 * 1000 ) logger.info( "Found stream ordering 1 day ago: it's %d", self.stream_ordering_day_ago @@ -1787,7 +1787,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas # We delete anything more than an hour old, on the assumption that we'll # never take more than an hour to persist an event. - delete_before_ts = self._clock.time_msec() - 60 * 60 * 1000 + delete_before_ts = self.clock.time_msec() - 60 * 60 * 1000 if self._started_ts > delete_before_ts: # We need to wait for at least an hour before we started deleting, @@ -1824,7 +1824,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas return # We sleep to ensure that we don't overwhelm the DB. - await self._clock.sleep(1.0) + await self.clock.sleep(1.0) async def get_push_actions_for_user( self, diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 0a0102ee64..37dd8e48d5 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -730,7 +730,7 @@ class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseS WHERE ? <= event_id AND event_id <= ? """ - txn.execute(sql, (self._clock.time_msec(), last_event_id, upper_event_id)) + txn.execute(sql, (self.clock.time_msec(), last_event_id, upper_event_id)) self.db_pool.updates._background_update_progress_txn( txn, "redactions_received_ts", {"last_event_id": upper_event_id} diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 31e2312211..4f9a1a4f78 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -70,7 +70,6 @@ from synapse.logging.opentracing import ( ) from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.streams import BackfillStream, UnPartialStatedEventStream @@ -282,13 +281,14 @@ class EventsWorkerStore(SQLBaseStore): if hs.config.worker.run_background_tasks: # We periodically clean out old transaction ID mappings - self._clock.looping_call( + self.clock.looping_call( self._cleanup_old_transaction_ids, 5 * 60 * 1000, ) self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( + clock=hs.get_clock(), server_name=self.server_name, cache_name="*getEvent*", max_size=hs.config.caches.event_cache_size, @@ -1154,9 +1154,7 @@ class EventsWorkerStore(SQLBaseStore): should_start = False if should_start: - run_as_background_process( - "fetch_events", self.server_name, self._fetch_thread - ) + self.hs.run_as_background_process("fetch_events", self._fetch_thread) async def _fetch_thread(self) -> None: """Services requests for events from `_event_fetch_list`.""" @@ -1276,7 +1274,7 @@ class EventsWorkerStore(SQLBaseStore): were not part of this request. """ with Measure( - self._clock, name="_fetch_event_list", server_name=self.server_name + self.clock, name="_fetch_event_list", server_name=self.server_name ): try: events_to_fetch = { @@ -2278,7 +2276,7 @@ class EventsWorkerStore(SQLBaseStore): """Cleans out transaction id mappings older than 24hrs.""" def _cleanup_old_transaction_ids_txn(txn: LoggingTransaction) -> None: - one_day_ago = self._clock.time_msec() - 24 * 60 * 60 * 1000 + one_day_ago = self.clock.time_msec() - 24 * 60 * 60 * 1000 sql = """ DELETE FROM event_txn_id_device_id WHERE inserted_ts < ? @@ -2633,7 +2631,7 @@ class EventsWorkerStore(SQLBaseStore): keyvalues={"event_id": event_id}, values={ "reason": rejection_reason, - "last_check": self._clock.time_msec(), + "last_check": self.clock.time_msec(), }, ) self.db_pool.simple_update_txn( diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index d0e4a91b59..e2b15eaf6a 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -28,7 +28,6 @@ from twisted.internet import defer from twisted.internet.task import LoopingCall from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.storage._base import SQLBaseStore @@ -99,15 +98,15 @@ class LockStore(SQLBaseStore): # lead to a race, as we may drop the lock while we are still processing. # However, a) it should be a small window, b) the lock is best effort # anyway and c) we want to really avoid leaking locks when we restart. - hs.get_clock().add_system_event_trigger( - "before", - "shutdown", - self._on_shutdown, + hs.register_async_shutdown_handler( + phase="before", + eventType="shutdown", + shutdown_func=self._on_shutdown, ) self._acquiring_locks: Set[Tuple[str, str]] = set() - self._clock.looping_call( + self.clock.looping_call( self._reap_stale_read_write_locks, _LOCK_TIMEOUT_MS / 10.0 ) @@ -153,7 +152,7 @@ class LockStore(SQLBaseStore): if lock and await lock.is_still_valid(): return None - now = self._clock.time_msec() + now = self.clock.time_msec() token = random_string(6) def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: @@ -202,7 +201,8 @@ class LockStore(SQLBaseStore): lock = Lock( self.server_name, self._reactor, - self._clock, + self.hs, + self.clock, self, read_write=False, lock_name=lock_name, @@ -251,7 +251,7 @@ class LockStore(SQLBaseStore): # constraints. If it doesn't then we have acquired the lock, # otherwise we haven't. - now = self._clock.time_msec() + now = self.clock.time_msec() token = random_string(6) self.db_pool.simple_insert_txn( @@ -270,7 +270,8 @@ class LockStore(SQLBaseStore): lock = Lock( self.server_name, self._reactor, - self._clock, + self.hs, + self.clock, self, read_write=True, lock_name=lock_name, @@ -338,7 +339,7 @@ class LockStore(SQLBaseStore): """ def reap_stale_read_write_locks_txn(txn: LoggingTransaction) -> None: - txn.execute(delete_sql, (self._clock.time_msec() - _LOCK_TIMEOUT_MS,)) + txn.execute(delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT_MS,)) if txn.rowcount: logger.info("Reaped %d stale locks", txn.rowcount) @@ -374,6 +375,7 @@ class Lock: self, server_name: str, reactor: ISynapseReactor, + hs: "HomeServer", clock: Clock, store: LockStore, read_write: bool, @@ -387,6 +389,7 @@ class Lock: """ self._server_name = server_name self._reactor = reactor + self._hs = hs self._clock = clock self._store = store self._read_write = read_write @@ -410,6 +413,7 @@ class Lock: _RENEWAL_INTERVAL_MS, self._server_name, self._store, + self._hs, self._clock, self._read_write, self._lock_name, @@ -421,6 +425,7 @@ class Lock: def _renew( server_name: str, store: LockStore, + hs: "HomeServer", clock: Clock, read_write: bool, lock_name: str, @@ -457,9 +462,8 @@ class Lock: desc="renew_lock", ) - return run_as_background_process( + return hs.run_as_background_process( "Lock._renew", - server_name, _internal_renew, store, clock, diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index f726846e57..b8bd0042d7 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -565,7 +565,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): sql, ( user_id.to_string(), - self._clock.time_msec() - self.unused_expiration_time, + self.clock.time_msec() - self.unused_expiration_time, ), ) row = txn.fetchone() @@ -1059,7 +1059,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): txn: LoggingTransaction, ) -> int: # Calculate the timestamp for the start of the time period - start_ts = self._clock.time_msec() - time_period_ms + start_ts = self.clock.time_msec() - time_period_ms txn.execute(sql, (user_id, start_ts)) row = txn.fetchone() if row is None: diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index a3467bff3d..49411ed034 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -78,7 +78,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): # Read the extrems every 60 minutes if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) + self.clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() @@ -224,7 +224,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): """ Counts the number of users who used this homeserver in the last 24 hours. """ - yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24) + yesterday = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24) return await self.db_pool.runInteraction( "count_daily_users", self._count_users, yesterday ) @@ -236,7 +236,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): from the mau figure in synapse.storage.monthly_active_users which, amongst other things, includes a 3 day grace period before a user counts. """ - thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) + thirty_days_ago = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) return await self.db_pool.runInteraction( "count_monthly_users", self._count_users, thirty_days_ago ) @@ -281,7 +281,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): def _count_r30v2_users(txn: LoggingTransaction) -> Dict[str, int]: thirty_days_in_secs = 86400 * 30 - now = int(self._clock.time()) + now = int(self.clock.time()) sixty_days_ago_in_secs = now - 2 * thirty_days_in_secs one_day_from_now_in_secs = now + 86400 @@ -389,7 +389,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): """ Returns millisecond unixtime for start of UTC day. """ - now = time.gmtime(self._clock.time()) + now = time.gmtime(self.clock.time()) today_start = calendar.timegm((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0)) return today_start * 1000 @@ -403,7 +403,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore): logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() a_day_in_milliseconds = 24 * 60 * 60 * 1000 - now = self._clock.time_msec() + now = self.clock.time_msec() # A note on user_agent. Technically a given device can have multiple # user agents, so we need to decide which one to pick. We could have diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index f5a6b98be7..86744f616c 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -49,7 +49,6 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): hs: "HomeServer", ): super().__init__(database, db_conn, hs) - self._clock = hs.get_clock() self.hs = hs if hs.config.redis.redis_enabled: @@ -226,7 +225,7 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): reserved_users: reserved users to preserve """ - thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) + thirty_days_ago = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) in_clause, in_clause_args = make_in_list_sql_clause( self.database_engine, "user_id", reserved_users @@ -328,7 +327,7 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn, table="monthly_active_users", keyvalues={"user_id": user_id}, - values={"timestamp": int(self._clock.time_msec())}, + values={"timestamp": int(self.clock.time_msec())}, ) else: logger.warning("mau limit reserved threepid %s not found in db", tp) @@ -391,7 +390,7 @@ class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): txn, table="monthly_active_users", keyvalues={"user_id": user_id}, - values={"timestamp": int(self._clock.time_msec())}, + values={"timestamp": int(self.clock.time_msec())}, ) self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index ff4eb9acb2..f1dbf68971 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -1073,7 +1073,7 @@ class ReceiptsWorkerStore(SQLBaseStore): if event_ts is None: return None - now = self._clock.time_msec() + now = self.clock.time_msec() logger.debug( "Receipt %s for event %s in %s (%i ms old)", receipt_type, diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 117444e7b7..906d1a91f6 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -212,7 +212,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): ) if hs.config.worker.run_background_tasks: - self._clock.call_later( + self.clock.call_later( 0.0, self._set_expiration_date_when_missing, ) @@ -226,7 +226,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS ) @@ -298,7 +298,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): ) -> None: user_id_obj = UserID.from_string(user_id) - now = int(self._clock.time()) + now = int(self.clock.time()) user_approved = approved or not self._require_approval @@ -457,7 +457,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): if not info: return False - now = self._clock.time_msec() + now = self.clock.time_msec() days = self.config.server.mau_appservice_trial_days.get( info.appservice_id, self.config.server.mau_trial_days ) @@ -640,7 +640,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): return await self.db_pool.runInteraction( "get_users_expiring_soon", select_users_txn, - self._clock.time_msec(), + self.clock.time_msec(), self.config.account_validity.account_validity_renew_at, ) @@ -1084,7 +1084,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): """ def _count_daily_user_type(txn: LoggingTransaction) -> Dict[str, int]: - yesterday = int(self._clock.time()) - (60 * 60 * 24) + yesterday = int(self.clock.time()) - (60 * 60 * 24) sql = """ SELECT user_type, COUNT(*) AS count FROM ( @@ -1496,7 +1496,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): await self.db_pool.runInteraction( "cull_expired_threepid_validation_tokens", cull_expired_threepid_validation_tokens_txn, - self._clock.time_msec(), + self.clock.time_msec(), ) @wrap_as_background_process("account_validity_set_expiration_dates") @@ -1537,7 +1537,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): random value in the [now + period - d ; now + period] range, d being a delta equal to 10% of the validity period. """ - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() assert self._account_validity_period is not None expiration_ts = now_ms + self._account_validity_period @@ -1608,7 +1608,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): Raises: StoreError if there was a problem updating this. """ - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.simple_update_one( "access_tokens", @@ -1639,7 +1639,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): uses_allowed, pending, completed, expiry_time = res # Check if the token has expired - now = self._clock.time_msec() + now = self.clock.time_msec() if expiry_time and expiry_time < now: return False @@ -1771,7 +1771,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): return await self.db_pool.runInteraction( "select_registration_tokens", select_registration_tokens_txn, - self._clock.time_msec(), + self.clock.time_msec(), valid, ) @@ -2251,7 +2251,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): "consume_login_token", self._consume_login_token, token, - self._clock.time_msec(), + self.clock.time_msec(), ) async def invalidate_login_tokens_by_session_id( @@ -2271,7 +2271,7 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): "auth_provider_id": auth_provider_id, "auth_provider_session_id": auth_provider_session_id, }, - updatevalues={"used_ts": self._clock.time_msec()}, + updatevalues={"used_ts": self.clock.time_msec()}, desc="invalidate_login_tokens_by_session_id", ) @@ -2640,7 +2640,6 @@ class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): ): super().__init__(database, db_conn, hs) - self._clock = hs.get_clock() self.config = hs.config self.db_pool.updates.register_background_index_update( @@ -2761,7 +2760,7 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self._delete_expired_login_tokens, THIRTY_MINUTES_IN_MS ) @@ -2790,7 +2789,7 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): The token ID """ next_id = self._access_tokens_id_gen.get_next() - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.simple_insert( "access_tokens", @@ -2874,7 +2873,7 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): keyvalues={"name": user_id}, updatevalues={ "consent_version": consent_version, - "consent_ts": self._clock.time_msec(), + "consent_ts": self.clock.time_msec(), }, ) self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) @@ -2986,7 +2985,7 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): txn, table="threepid_validation_session", keyvalues={"session_id": session_id}, - updatevalues={"validated_at": self._clock.time_msec()}, + updatevalues={"validated_at": self.clock.time_msec()}, ) return next_link @@ -3064,7 +3063,7 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): # We keep the expired tokens for an extra 5 minutes so we can measure how many # times a token is being used after its expiry - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.runInteraction( "delete_expired_login_tokens", _delete_expired_login_tokens_txn, diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9db2e14a06..65caf4b1ea 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1002,7 +1002,7 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): """ with Measure( - self._clock, + self.clock, name="get_joined_user_ids_from_state", server_name=self.server_name, ): diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 8a5fa8386c..1154bb2d59 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -55,7 +55,7 @@ class SessionStore(SQLBaseStore): # Create a background job for culling expired sessions. if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) + self.clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) async def create_session( self, session_type: str, value: JsonDict, expiry_ms: int @@ -133,7 +133,7 @@ class SessionStore(SQLBaseStore): _get_session, session_type, session_id, - self._clock.time_msec(), + self.clock.time_msec(), ) @wrap_as_background_process("delete_expired_sessions") @@ -147,5 +147,5 @@ class SessionStore(SQLBaseStore): await self.db_pool.runInteraction( "delete_expired_sessions", _delete_expired_sessions_txn, - self._clock.time_msec(), + self.clock.time_msec(), ) diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index f7af3e88d3..c0c5087b13 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -201,7 +201,7 @@ class SlidingSyncStore(SQLBaseStore): "user_id": user_id, "effective_device_id": device_id, "conn_id": conn_id, - "created_ts": self._clock.time_msec(), + "created_ts": self.clock.time_msec(), }, returning=("connection_key",), ) @@ -212,7 +212,7 @@ class SlidingSyncStore(SQLBaseStore): table="sliding_sync_connection_positions", values={ "connection_key": connection_key, - "created_ts": self._clock.time_msec(), + "created_ts": self.clock.time_msec(), }, returning=("connection_position",), ) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index bfc324b80d..41c9483927 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -81,11 +81,11 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): super().__init__(database, db_conn, hs) if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) + self.clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: - now = self._clock.time_msec() + now = self.clock.time_msec() day_ago = now - 24 * 60 * 60 * 1000 def _cleanup_transactions_txn(txn: LoggingTransaction) -> None: @@ -160,7 +160,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): insertion_values={ "response_code": code, "response_json": db_binary_type(encode_canonical_json(response_dict)), - "ts": self._clock.time_msec(), + "ts": self.clock.time_msec(), }, desc="set_received_txn_response", ) diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 9b3b7e086f..b62f3e6f5b 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -125,6 +125,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): self._state_group_cache: DictionaryCache[int, StateKey, str] = DictionaryCache( name="*stateGroupCache*", + clock=hs.get_clock(), server_name=self.server_name, # TODO: this hasn't been tuned yet max_entries=50000, @@ -132,6 +133,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): self._state_group_members_cache: DictionaryCache[int, StateKey, str] = ( DictionaryCache( name="*stateGroupMembersCache*", + clock=hs.get_clock(), server_name=self.server_name, max_entries=500000, ) diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 9fb7831e6b..530d2be8e3 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -17,6 +17,10 @@ from twisted.internet.defer import Deferred from synapse.types import ISynapseReactor class HttpClient: + """ + The returned deferreds follow Synapse logcontext rules. + """ + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: ... def get(self, url: str, response_limit: int) -> Deferred[bytes]: ... def post( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 1c343f8d3e..2a167f209c 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -47,7 +47,6 @@ from typing import ( Tuple, TypeVar, Union, - cast, overload, ) @@ -56,7 +55,6 @@ from typing_extensions import Concatenate, ParamSpec, Unpack from twisted.internet import defer from twisted.internet.defer import CancelledError -from twisted.internet.interfaces import IReactorTime from twisted.python.failure import Failure from synapse.logging.context import ( @@ -65,7 +63,6 @@ from synapse.logging.context import ( run_coroutine_in_background, run_in_background, ) -from synapse.types import ISynapseThreadlessReactor from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -551,25 +548,19 @@ class Linearizer: def __init__( self, - name: Optional[str] = None, + name: str, + clock: Clock, max_count: int = 1, - clock: Optional[Clock] = None, ): """ Args: + name: TODO max_count: The maximum number of concurrent accesses + clock: (ideally, the homeserver clock `hs.get_clock()`) """ - if name is None: - self.name: Union[str, int] = id(self) - else: - self.name = name - - if not clock: - from twisted.internet import reactor - - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) - self._clock = clock + self.name = name self.max_count = max_count + self._clock = clock # key_to_defer is a map from the key to a _LinearizerEntry. self.key_to_defer: Dict[Hashable, _LinearizerEntry] = {} @@ -779,7 +770,11 @@ class ReadWriteLock: def timeout_deferred( - deferred: "defer.Deferred[_T]", timeout: float, reactor: IReactorTime + *, + deferred: "defer.Deferred[_T]", + timeout: float, + cancel_on_shutdown: bool = True, + clock: Clock, ) -> "defer.Deferred[_T]": """The in built twisted `Deferred.addTimeout` fails to time out deferreds that have a canceller that throws exceptions. This method creates a new @@ -797,7 +792,13 @@ def timeout_deferred( Args: deferred: The Deferred to potentially timeout. timeout: Timeout in seconds - reactor: The twisted reactor to use + cancel_on_shutdown: Whether this call should be tracked for cleanup during + shutdown. In general, all calls should be tracked. There may be a use case + not to track calls with a `timeout` of 0 (or similarly short) since tracking + them may result in rapid insertions and removals of tracked calls + unnecessarily. But unless a specific instance of tracking proves to be an + issue, we can just track all delayed calls. + clock: The `Clock` instance used to track delayed calls. Returns: @@ -821,7 +822,10 @@ def timeout_deferred( if not new_d.called: new_d.errback(defer.TimeoutError("Timed out after %gs" % (timeout,))) - delayed_call = reactor.callLater(timeout, time_it_out) + # We don't track these calls since they are short. + delayed_call = clock.call_later( + timeout, time_it_out, call_later_cancel_on_shutdown=cancel_on_shutdown + ) def convert_cancelled(value: Failure) -> Failure: # if the original deferred was cancelled, and our timeout has fired, then @@ -963,9 +967,9 @@ class AwakenableSleeper: currently sleeping. """ - def __init__(self, reactor: IReactorTime) -> None: + def __init__(self, clock: Clock) -> None: self._streams: Dict[str, Set[defer.Deferred[None]]] = {} - self._reactor = reactor + self._clock = clock def wake(self, name: str) -> None: """Wake everything related to `name` that is currently sleeping.""" @@ -984,7 +988,11 @@ class AwakenableSleeper: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._reactor.callLater(delay_ms / 1000, sleep_deferred.callback, None) + call = self._clock.call_later( + delay_ms / 1000, + sleep_deferred.callback, + None, + ) # Create a deferred that will get called if `wake` is called with # the same `name`. @@ -1018,8 +1026,8 @@ class AwakenableSleeper: class DeferredEvent: """Like threading.Event but for async code""" - def __init__(self, reactor: IReactorTime) -> None: - self._reactor = reactor + def __init__(self, clock: Clock) -> None: + self._clock = clock self._deferred: "defer.Deferred[None]" = defer.Deferred() def set(self) -> None: @@ -1039,7 +1047,11 @@ class DeferredEvent: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._reactor.callLater(timeout_seconds, sleep_deferred.callback, None) + call = self._clock.call_later( + timeout_seconds, + sleep_deferred.callback, + None, + ) try: await make_deferred_yieldable( diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 4c4037412a..f77301afd8 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -21,6 +21,7 @@ import logging from typing import ( + TYPE_CHECKING, Awaitable, Callable, Dict, @@ -38,9 +39,11 @@ from twisted.internet import defer from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.clock import Clock +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -97,12 +100,13 @@ class BatchingQueue(Generic[V, R]): self, *, name: str, - server_name: str, + hs: "HomeServer", clock: Clock, process_batch_callback: Callable[[List[V]], Awaitable[R]], ): self._name = name - self.server_name = server_name + self.hs = hs + self.server_name = hs.hostname self._clock = clock # The set of keys currently being processed. @@ -127,6 +131,14 @@ class BatchingQueue(Generic[V, R]): name=self._name, **{SERVER_NAME_LABEL: self.server_name} ) + def shutdown(self) -> None: + """ + Prepares the object for garbage collection by removing any handed out + references. + """ + number_queued.remove(self._name, self.server_name) + number_of_keys.remove(self._name, self.server_name) + async def add_to_queue(self, value: V, key: Hashable = ()) -> R: """Adds the value to the queue with the given key, returning the result of the processing function for the batch that included the given value. @@ -145,9 +157,7 @@ class BatchingQueue(Generic[V, R]): # If we're not currently processing the key fire off a background # process to start processing. if key not in self._processing_keys: - run_as_background_process( - self._name, self.server_name, self._process_queue, key - ) + self.hs.run_as_background_process(self._name, self._process_queue, key) with self._number_in_flight_metric.track_inprogress(): return await make_deferred_yieldable(d) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 710a29e3f0..08ff842af0 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -244,7 +244,7 @@ def register_cache( collect_callback=collect_callback, ) metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) - CACHE_METRIC_REGISTRY.register_hook(metric_name, metric.collect) + CACHE_METRIC_REGISTRY.register_hook(server_name, metric_name, metric.collect) return metric diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 92d446ce2a..016acbac71 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -47,6 +47,7 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry +from synapse.util.clock import Clock cache_pending_metric = Gauge( "synapse_util_caches_cache_pending", @@ -82,6 +83,7 @@ class DeferredCache(Generic[KT, VT]): self, *, name: str, + clock: Clock, server_name: str, max_entries: int = 1000, tree: bool = False, @@ -103,6 +105,7 @@ class DeferredCache(Generic[KT, VT]): prune_unread_entries: If True, cache entries that haven't been read recently will be evicted from the cache in the background. Set to False to opt-out of this behaviour. + clock: The homeserver `Clock` instance """ cache_type = TreeCache if tree else dict @@ -120,6 +123,7 @@ class DeferredCache(Generic[KT, VT]): # a Deferred. self.cache: LruCache[KT, VT] = LruCache( max_size=max_entries, + clock=clock, server_name=server_name, cache_name=name, cache_type=cache_type, diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 47b8f4ddc8..6e3c8eada9 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -53,6 +53,7 @@ from synapse.util import unwrapFirstError from synapse.util.async_helpers import delay_cancellation from synapse.util.caches.deferred_cache import DeferredCache from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -154,13 +155,20 @@ class _CacheDescriptorBase: ) -class HasServerName(Protocol): +class HasServerNameAndClock(Protocol): server_name: str """ The homeserver name that this cache is associated with (used to label the metric) (`hs.hostname`). """ + clock: Clock + """ + The homeserver clock instance used to track delayed and looping calls. Important to + be able to fully cleanup the homeserver instance on server shutdown. + (`hs.get_clock()`). + """ + class DeferredCacheDescriptor(_CacheDescriptorBase): """A method decorator that applies a memoizing cache around the function. @@ -239,7 +247,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): self.prune_unread_entries = prune_unread_entries def __get__( - self, obj: Optional[HasServerName], owner: Optional[Type] + self, obj: Optional[HasServerNameAndClock], owner: Optional[Type] ) -> Callable[..., "defer.Deferred[Any]"]: # We need access to instance-level `obj.server_name` attribute assert obj is not None, ( @@ -249,9 +257,13 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): assert obj.server_name is not None, ( "The `server_name` attribute must be set on the object where `@cached` decorator is used." ) + assert obj.clock is not None, ( + "The `clock` attribute must be set on the object where `@cached` decorator is used." + ) cache: DeferredCache[CacheKey, Any] = DeferredCache( name=self.name, + clock=obj.clock, server_name=obj.server_name, max_entries=self.max_entries, tree=self.tree, diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 168ddc51cd..eb5493d322 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -37,6 +37,7 @@ import attr from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -127,10 +128,13 @@ class DictionaryCache(Generic[KT, DKT, DV]): for the '2' dict key. """ - def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): + def __init__( + self, *, name: str, clock: Clock, server_name: str, max_entries: int = 1000 + ): """ Args: name + clock: The homeserver `Clock` instance server_name: The homeserver name that this cache is associated with (used to label the metric) (`hs.hostname`). max_entries @@ -160,6 +164,7 @@ class DictionaryCache(Generic[KT, DKT, DV]): Union[_PerKeyValue, Dict[DKT, DV]], ] = LruCache( max_size=max_entries, + clock=clock, server_name=server_name, cache_name=name, cache_type=TreeCache, diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 305af5051c..29ce6c0a77 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -21,17 +21,29 @@ import logging from collections import OrderedDict -from typing import Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Iterable, + Literal, + Optional, + TypeVar, + Union, + overload, +) import attr from twisted.internet import defer from synapse.config import cache as cache_config -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import EvictionReason, register_cache from synapse.util.clock import Clock +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -49,6 +61,7 @@ class ExpiringCache(Generic[KT, VT]): *, cache_name: str, server_name: str, + hs: "HomeServer", clock: Clock, max_len: int = 0, expiry_ms: int = 0, @@ -99,9 +112,7 @@ class ExpiringCache(Generic[KT, VT]): return def f() -> "defer.Deferred[None]": - return run_as_background_process( - "prune_cache", server_name, self._prune_cache - ) + return hs.run_as_background_process("prune_cache", self._prune_cache) self._clock.looping_call(f, self._expiry_ms / 2) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 187380c433..324acb728a 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -45,14 +45,10 @@ from typing import ( overload, ) -from twisted.internet import defer, reactor +from twisted.internet import defer from synapse.config import cache as cache_config -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.metrics.jemalloc import get_jemalloc_stats -from synapse.types import ISynapseThreadlessReactor from synapse.util import caches from synapse.util.caches import CacheMetric, EvictionReason, register_cache from synapse.util.caches.treecache import ( @@ -123,6 +119,7 @@ GLOBAL_ROOT = ListNode["_Node"].create_root_node() def _expire_old_entries( server_name: str, + hs: "HomeServer", clock: Clock, expiry_seconds: float, autotune_config: Optional[dict], @@ -228,9 +225,8 @@ def _expire_old_entries( logger.info("Dropped %d items from caches", i) - return run_as_background_process( + return hs.run_as_background_process( "LruCache._expire_old_entries", - server_name, _internal_expire_old_entries, clock, expiry_seconds, @@ -261,6 +257,7 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: _expire_old_entries, 30 * 1000, server_name, + hs, clock, expiry_time, hs.config.caches.cache_autotuning, @@ -404,13 +401,13 @@ class LruCache(Generic[KT, VT]): self, *, max_size: int, + clock: Clock, server_name: str, cache_name: str, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): ... @@ -420,13 +417,13 @@ class LruCache(Generic[KT, VT]): self, *, max_size: int, - server_name: Literal[None] = None, + clock: Clock, + server_name: str, cache_name: Literal[None] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): ... @@ -435,13 +432,13 @@ class LruCache(Generic[KT, VT]): self, *, max_size: int, - server_name: Optional[str] = None, + clock: Clock, + server_name: str, cache_name: Optional[str] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): @@ -450,12 +447,10 @@ class LruCache(Generic[KT, VT]): max_size: The maximum amount of entries the cache can hold server_name: The homeserver name that this cache is associated with - (used to label the metric) (`hs.hostname`). Must be set if `cache_name` is - set. If unset, no metrics will be reported on this cache. + (used to label the metric) (`hs.hostname`). - cache_name: The name of this cache, for the prometheus metrics. Must be set - if `server_name` is set. If unset, no metrics will be reported on this - cache. + cache_name: The name of this cache, for the prometheus metrics. If unset, no + metrics will be reported on this cache. cache_type: type of underlying cache to be used. Typically one of dict @@ -494,13 +489,6 @@ class LruCache(Generic[KT, VT]): Note: The new key does not have to be unique. """ - # Default `clock` to something sensible. Note that we rename it to - # `real_clock` so that mypy doesn't think its still `Optional`. - if clock is None: - real_clock = Clock(cast(ISynapseThreadlessReactor, reactor)) - else: - real_clock = clock - cache: Union[Dict[KT, _Node[KT, VT]], TreeCache] = cache_type() self.cache = cache # Used for introspection. self.apply_cache_factor_from_config = apply_cache_factor_from_config @@ -592,7 +580,7 @@ class LruCache(Generic[KT, VT]): key, value, weak_ref_to_self, - real_clock, + clock, callbacks, prune_unread_entries, ) @@ -610,7 +598,7 @@ class LruCache(Generic[KT, VT]): metrics.inc_memory_usage(node.memory) def move_node_to_front(node: _Node[KT, VT]) -> None: - node.move_to_front(real_clock, list_root) + node.move_to_front(clock, list_root) def delete_node(node: _Node[KT, VT]) -> int: node.drop_from_lists() diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 79e34262df..3d39357236 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -198,7 +198,17 @@ class ResponseCache(Generic[KV]): # the should_cache bit, we leave it in the cache for now and schedule # its removal later. if self.timeout_sec and context.should_cache: - self.clock.call_later(self.timeout_sec, self._entry_timeout, key) + self.clock.call_later( + self.timeout_sec, + self._entry_timeout, + key, + # We don't need to track these calls since they don't hold any strong + # references which would keep the `HomeServer` in memory after shutdown. + # We don't want to track these because they can get cancelled really + # quickly and thrash the tracking mechanism, ie. during repeated calls + # to /sync. + call_later_cancel_on_shutdown=False, + ) else: # otherwise, remove the result immediately. self.unset(key) diff --git a/synapse/util/clock.py b/synapse/util/clock.py index d28dbac357..5e65cf32a4 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -17,10 +17,12 @@ from typing import ( Any, Callable, + Dict, + List, ) -import attr from typing_extensions import ParamSpec +from zope.interface import implementer from twisted.internet import defer, task from twisted.internet.defer import Deferred @@ -34,23 +36,54 @@ from synapse.util import log_failure P = ParamSpec("P") -@attr.s(slots=True) class Clock: """ A Clock wraps a Twisted reactor and provides utilities on top of it. + This clock should be used in place of calls to the base reactor wherever `LoopingCall` + or `DelayedCall` are made (such as when calling `reactor.callLater`. This is to + ensure the calls made by this `HomeServer` instance are tracked and can be cleaned + up during `HomeServer.shutdown()`. + + We enforce usage of this clock instead of using the reactor directly via lints in + `scripts-dev/mypy_synapse_plugin.py`. + + Args: reactor: The Twisted reactor to use. """ - _reactor: ISynapseThreadlessReactor = attr.ib() + _reactor: ISynapseThreadlessReactor + + def __init__(self, reactor: ISynapseThreadlessReactor, server_name: str) -> None: + self._reactor = reactor + self._server_name = server_name + + self._delayed_call_id: int = 0 + """Unique ID used to track delayed calls""" + + self._looping_calls: List[LoopingCall] = [] + """List of active looping calls""" + + self._call_id_to_delayed_call: Dict[int, IDelayedCall] = {} + """Mapping from unique call ID to delayed call""" + + self._is_shutdown = False + """Whether shutdown has been requested by the HomeServer""" + + def shutdown(self) -> None: + self._is_shutdown = True + self.cancel_all_looping_calls() + self.cancel_all_delayed_calls() async def sleep(self, seconds: float) -> None: d: defer.Deferred[float] = defer.Deferred() # Start task in the `sentinel` logcontext, to avoid leaking the current context # into the reactor once it finishes. with context.PreserveLoggingContext(): - self._reactor.callLater(seconds, d.callback, seconds) + # We can ignore the lint here since this class is the one location callLater should + # be called. + self._reactor.callLater(seconds, d.callback, seconds) # type: ignore[call-later-not-tracked] await d def time(self) -> float: @@ -123,6 +156,9 @@ class Clock: ) -> LoopingCall: """Common functionality for `looping_call` and `looping_call_now`""" + if self._is_shutdown: + raise Exception("Cannot start looping call. Clock has been shutdown") + def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: assert context.current_context() is context.SENTINEL_CONTEXT, ( "Expected `looping_call` callback from the reactor to start with the sentinel logcontext " @@ -144,13 +180,19 @@ class Clock: # this function and yield control back to the reactor to avoid leaking the # current logcontext to the reactor (which would then get picked up and # associated with the next thing the reactor does) - with context.PreserveLoggingContext(context.LoggingContext("looping_call")): + with context.PreserveLoggingContext( + context.LoggingContext( + name="looping_call", server_name=self._server_name + ) + ): # We use `run_in_background` to reset the logcontext after `f` (or the # awaitable returned by `f`) completes to avoid leaking the current # logcontext to the reactor return context.run_in_background(f, *args, **kwargs) - call = task.LoopingCall(wrapped_f, *args, **kwargs) + # We can ignore the lint here since this is the one location LoopingCall's + # should be created. + call = task.LoopingCall(wrapped_f, *args, **kwargs) # type: ignore[prefer-synapse-clock-looping-call] call.clock = self._reactor # If `now=true`, the function will be called here immediately so we need to be # in the sentinel context now. @@ -160,10 +202,32 @@ class Clock: with context.PreserveLoggingContext(): d = call.start(msec / 1000.0, now=now) d.addErrback(log_failure, "Looping call died", consumeErrors=False) + self._looping_calls.append(call) return call + def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: + """ + Stop all running looping calls. + + Args: + consumeErrors: Whether to re-raise errors encountered when cancelling the + scheduled call. + """ + for call in self._looping_calls: + try: + call.stop() + except Exception: + if not consumeErrors: + raise + self._looping_calls.clear() + def call_later( - self, delay: float, callback: Callable, *args: Any, **kwargs: Any + self, + delay: float, + callback: Callable, + *args: Any, + call_later_cancel_on_shutdown: bool = True, + **kwargs: Any, ) -> IDelayedCall: """Call something later @@ -175,37 +239,78 @@ class Clock: delay: How long to wait in seconds. callback: Function to call *args: Postional arguments to pass to function. + call_later_cancel_on_shutdown: Whether this call should be tracked for cleanup during + shutdown. In general, all calls should be tracked. There may be a use case + not to track calls with a `timeout` of 0 (or similarly short) since tracking + them may result in rapid insertions and removals of tracked calls + unnecessarily. But unless a specific instance of tracking proves to be an + issue, we can just track all delayed calls. **kwargs: Key arguments to pass to function. """ - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - assert context.current_context() is context.SENTINEL_CONTEXT, ( - "Expected `call_later` callback from the reactor to start with the sentinel logcontext " - f"but saw {context.current_context()}. In other words, another task shouldn't have " - "leaked their logcontext to us." - ) + if self._is_shutdown: + raise Exception("Cannot start delayed call. Clock has been shutdown") - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `call_later` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `loop_call` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - with context.PreserveLoggingContext(context.LoggingContext("call_later")): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(callback, *args, **kwargs) + def create_wrapped_callback( + track_for_shutdown_cancellation: bool, + ) -> Callable[P, None]: + def wrapped_callback(*args: Any, **kwargs: Any) -> None: + assert context.current_context() is context.SENTINEL_CONTEXT, ( + "Expected `call_later` callback from the reactor to start with the sentinel logcontext " + f"but saw {context.current_context()}. In other words, another task shouldn't have " + "leaked their logcontext to us." + ) - return self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) + # Because this is a callback from the reactor, we will be using the + # `sentinel` log context at this point. We want the function to log with + # some logcontext as we want to know which server the logs came from. + # + # We use `PreserveLoggingContext` to prevent our new `call_later` + # logcontext from finishing as soon as we exit this function, in case `f` + # returns an awaitable/deferred which would continue running and may try to + # restore the `loop_call` context when it's done (because it's trying to + # adhere to the Synapse logcontext rules.) + # + # This also ensures that we return to the `sentinel` context when we exit + # this function and yield control back to the reactor to avoid leaking the + # current logcontext to the reactor (which would then get picked up and + # associated with the next thing the reactor does) + try: + with context.PreserveLoggingContext( + context.LoggingContext( + name="call_later", server_name=self._server_name + ) + ): + # We use `run_in_background` to reset the logcontext after `f` (or the + # awaitable returned by `f`) completes to avoid leaking the current + # logcontext to the reactor + context.run_in_background(callback, *args, **kwargs) + finally: + if track_for_shutdown_cancellation: + # We still want to remove the call from the tracking map. Even if + # the callback raises an exception. + self._call_id_to_delayed_call.pop(call_id) + + return wrapped_callback + + if call_later_cancel_on_shutdown: + call_id = self._delayed_call_id + self._delayed_call_id = self._delayed_call_id + 1 + + # We can ignore the lint here since this class is the one location callLater + # should be called. + call = self._reactor.callLater( + delay, create_wrapped_callback(True), *args, **kwargs + ) # type: ignore[call-later-not-tracked] + call = DelayedCallWrapper(call, call_id, self) + self._call_id_to_delayed_call[call_id] = call + return call + else: + # We can ignore the lint here since this class is the one location callLater should + # be called. + return self._reactor.callLater( + delay, create_wrapped_callback(False), *args, **kwargs + ) # type: ignore[call-later-not-tracked] def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: try: @@ -214,6 +319,24 @@ class Clock: if not ignore_errs: raise + def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: + """ + Stop all scheduled calls that were marked with `cancel_on_shutdown` when they were created. + + Args: + ignore_errs: Whether to re-raise errors encountered when cancelling the + scheduled call. + """ + # We make a copy here since calling `cancel()` on a delayed_call + # will result in the call removing itself from the map mid-iteration. + for call in list(self._call_id_to_delayed_call.values()): + try: + call.cancel() + except Exception: + if not ignore_errs: + raise + self._call_id_to_delayed_call.clear() + def call_when_running( self, callback: Callable[P, object], @@ -258,7 +381,9 @@ class Clock: # current logcontext to the reactor (which would then get picked up and # associated with the next thing the reactor does) with context.PreserveLoggingContext( - context.LoggingContext("call_when_running") + context.LoggingContext( + name="call_when_running", server_name=self._server_name + ) ): # We use `run_in_background` to reset the logcontext after `f` (or the # awaitable returned by `f`) completes to avoid leaking the current @@ -276,7 +401,7 @@ class Clock: callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs, - ) -> None: + ) -> Any: """ Add a function to be called when a system event occurs. @@ -290,6 +415,9 @@ class Clock: callback: Function to call *args: Postional arguments to pass to function. **kwargs: Key arguments to pass to function. + + Returns: + an ID that can be used to remove this call with `reactor.removeSystemEventTrigger`. """ def wrapped_callback(*args: Any, **kwargs: Any) -> None: @@ -313,7 +441,11 @@ class Clock: # this function and yield control back to the reactor to avoid leaking the # current logcontext to the reactor (which would then get picked up and # associated with the next thing the reactor does) - with context.PreserveLoggingContext(context.LoggingContext("system_event")): + with context.PreserveLoggingContext( + context.LoggingContext( + name="system_event", server_name=self._server_name + ) + ): # We use `run_in_background` to reset the logcontext after `f` (or the # awaitable returned by `f`) completes to avoid leaking the current # logcontext to the reactor @@ -321,6 +453,50 @@ class Clock: # We can ignore the lint here since this class is the one location # `addSystemEventTrigger` should be called. - self._reactor.addSystemEventTrigger( + return self._reactor.addSystemEventTrigger( phase, event_type, wrapped_callback, *args, **kwargs ) # type: ignore[prefer-synapse-clock-add-system-event-trigger] + + +@implementer(IDelayedCall) +class DelayedCallWrapper: + """Wraps an `IDelayedCall` so that we can intercept the call to `cancel()` and + properly cleanup the delayed call from the tracking map of the `Clock`. + + args: + delayed_call: The actual `IDelayedCall` + call_id: Unique identifier for this delayed call + clock: The clock instance tracking this call + """ + + def __init__(self, delayed_call: IDelayedCall, call_id: int, clock: Clock): + self.delayed_call = delayed_call + self.call_id = call_id + self.clock = clock + + def cancel(self) -> None: + """Remove the call from the tracking map and propagate the call to the + underlying delayed_call. + """ + self.delayed_call.cancel() + try: + self.clock._call_id_to_delayed_call.pop(self.call_id) + except KeyError: + # If the delayed call isn't being tracked anymore we can just move on. + pass + + def getTime(self) -> float: + """Propagate the call to the underlying delayed_call.""" + return self.delayed_call.getTime() + + def delay(self, secondsLater: float) -> None: + """Propagate the call to the underlying delayed_call.""" + self.delayed_call.delay(secondsLater) + + def reset(self, secondsFromNow: float) -> None: + """Propagate the call to the underlying delayed_call.""" + self.delayed_call.reset(secondsFromNow) + + def active(self) -> bool: + """Propagate the call to the underlying delayed_call.""" + return self.delayed_call.active() diff --git a/synapse/util/constants.py b/synapse/util/constants.py index 9986017147..7a3d073df5 100644 --- a/synapse/util/constants.py +++ b/synapse/util/constants.py @@ -18,3 +18,5 @@ # readability and catching bugs. ONE_MINUTE_SECONDS = 60 ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS + +MILLISECONDS_PER_SECOND = 1000 diff --git a/synapse/util/daemonize.py b/synapse/util/daemonize.py index e653abff97..dba815040d 100644 --- a/synapse/util/daemonize.py +++ b/synapse/util/daemonize.py @@ -32,6 +32,7 @@ from typing import NoReturn, Optional, Type from synapse.logging.context import ( LoggingContext, PreserveLoggingContext, + current_context, ) @@ -149,9 +150,12 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") - signal.signal(signal.SIGTERM, sigterm) + # Copy the `server_name` from the current logcontext + server_name = current_context().server_name + # Cleanup pid file at exit. def exit() -> None: - with LoggingContext("atexit"): + with LoggingContext(name="atexit", server_name=server_name): logger.warning("Stopping daemon.") os.remove(pid_file) sys.exit(0) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index f48ae3373c..dec6536e4e 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -20,6 +20,7 @@ # import logging from typing import ( + TYPE_CHECKING, Any, Awaitable, Callable, @@ -36,10 +37,13 @@ from typing_extensions import ParamSpec from twisted.internet import defer from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import UserID from synapse.util.async_helpers import maybe_awaitable +if TYPE_CHECKING: + from synapse.server import HomeServer + + logger = logging.getLogger(__name__) @@ -58,13 +62,13 @@ class Distributor: model will do for today. """ - def __init__(self, server_name: str) -> None: + def __init__(self, hs: "HomeServer") -> None: """ Args: server_name: The homeserver name of the server (used to label metrics) (this should be `hs.hostname`). """ - self.server_name = server_name + self.hs = hs self.signals: Dict[str, Signal] = {} self.pre_registration: Dict[str, List[Callable]] = {} @@ -97,8 +101,8 @@ class Distributor: if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) - run_as_background_process( - name, self.server_name, self.signals[name].fire, *args, **kwargs + self.hs.run_as_background_process( + name, self.signals[name].fire, *args, **kwargs ) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 09ccdb8ca3..7b6ad0e459 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -217,7 +217,11 @@ class Measure: else: assert isinstance(curr_context, LoggingContext) parent_context = curr_context - self._logging_context = LoggingContext(str(curr_context), parent_context) + self._logging_context = LoggingContext( + name=str(curr_context), + server_name=self.server_name, + parent_context=parent_context, + ) self.start: Optional[float] = None def __enter__(self) -> "Measure": @@ -289,21 +293,46 @@ class DynamicCollectorRegistry(CollectorRegistry): def __init__(self) -> None: super().__init__() - self._pre_update_hooks: Dict[str, Callable[[], None]] = {} + self._server_name_to_pre_update_hooks: Dict[ + str, Dict[str, Callable[[], None]] + ] = {} + """ + Mapping of server name to a mapping of metric name to metric pre-update + hook + """ def collect(self) -> Generator[Metric, None, None]: """ Collects metrics, calling pre-update hooks first. """ - for pre_update_hook in self._pre_update_hooks.values(): - pre_update_hook() + for pre_update_hooks in self._server_name_to_pre_update_hooks.values(): + for pre_update_hook in pre_update_hooks.values(): + pre_update_hook() yield from super().collect() - def register_hook(self, metric_name: str, hook: Callable[[], None]) -> None: + def register_hook( + self, server_name: str, metric_name: str, hook: Callable[[], None] + ) -> None: """ Registers a hook that is called before metric collection. """ - self._pre_update_hooks[metric_name] = hook + server_hooks = self._server_name_to_pre_update_hooks.setdefault(server_name, {}) + if server_hooks.get(metric_name) is not None: + # TODO: This should be an `assert` since registering the same metric name + # multiple times will clobber the old metric. + # We currently rely on this behaviour as we instantiate multiple + # `SyncRestServlet`, one per listener, and in the `__init__` we setup a new + # LruCache. + # Once the above behaviour is changed, this should be changed to an `assert`. + logger.error( + "Metric named %s already registered for server %s", + metric_name, + server_name, + ) + server_hooks[metric_name] = hook + + def unregister_hooks_for_homeserver(self, server_name: str) -> None: + self._server_name_to_pre_update_hooks.pop(server_name, None) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 695eb462bf..756677fe6c 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -419,4 +419,7 @@ class _PerHostRatelimiter: except KeyError: pass - self.clock.call_later(0.0, start_next_request) + self.clock.call_later( + 0.0, + start_next_request, + ) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 42a0cc7aa8..96fe2bd566 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -24,7 +24,6 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Optional, Type from synapse.api.errors import CodeMessageException -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage import DataStore from synapse.types import StrCollection from synapse.util.clock import Clock @@ -32,6 +31,7 @@ from synapse.util.clock import Clock if TYPE_CHECKING: from synapse.notifier import Notifier from synapse.replication.tcp.handler import ReplicationCommandHandler + from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -62,6 +62,7 @@ async def get_retry_limiter( *, destination: str, our_server_name: str, + hs: "HomeServer", clock: Clock, store: DataStore, ignore_backoff: bool = False, @@ -124,6 +125,7 @@ async def get_retry_limiter( return RetryDestinationLimiter( destination=destination, our_server_name=our_server_name, + hs=hs, clock=clock, store=store, failure_ts=failure_ts, @@ -163,6 +165,7 @@ class RetryDestinationLimiter: *, destination: str, our_server_name: str, + hs: "HomeServer", clock: Clock, store: DataStore, failure_ts: Optional[int], @@ -181,6 +184,7 @@ class RetryDestinationLimiter: Args: destination our_server_name: Our homeserver name (used to label metrics) (`hs.hostname`) + hs: The homeserver instance clock store failure_ts: when this destination started failing (in ms since @@ -197,6 +201,7 @@ class RetryDestinationLimiter: error code. """ self.our_server_name = our_server_name + self.hs = hs self.clock = clock self.store = store self.destination = destination @@ -331,6 +336,4 @@ class RetryDestinationLimiter: logger.exception("Failed to store destination_retry_timings") # we deliberately do this in the background. - run_as_background_process( - "store_retry_timings", self.our_server_name, store_retry_timings - ) + self.hs.run_as_background_process("store_retry_timings", store_retry_timings) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 0539989320..8dd6f12feb 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -27,12 +27,11 @@ from twisted.python.failure import Failure from synapse.logging.context import ( ContextResourceUsage, LoggingContext, + PreserveLoggingContext, nested_logging_context, - set_current_context, ) from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.types import JsonMapping, ScheduledTask, TaskStatus @@ -107,10 +106,8 @@ class TaskScheduler: OCCASIONAL_REPORT_INTERVAL_MS = 5 * 60 * 1000 # 5 minutes def __init__(self, hs: "HomeServer"): - self._hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() self._running_tasks: Set[str] = set() @@ -215,7 +212,7 @@ class TaskScheduler: if self._run_background_tasks: self._launch_scheduled_tasks() else: - self._hs.get_replication_command_handler().send_new_active_task(task.id) + self.hs.get_replication_command_handler().send_new_active_task(task.id) return task.id @@ -362,7 +359,7 @@ class TaskScheduler: finally: self._launching_new_tasks = False - run_as_background_process("launch_scheduled_tasks", self.server_name, inner) + self.hs.run_as_background_process("launch_scheduled_tasks", inner) @wrap_as_background_process("clean_scheduled_tasks") async def _clean_scheduled_tasks(self) -> None: @@ -425,14 +422,11 @@ class TaskScheduler: """ current_time = self._clock.time() - calling_context = set_current_context(task_log_context) - try: + with PreserveLoggingContext(task_log_context): usage = task_log_context.get_resource_usage() TaskScheduler._log_task_usage( "continuing", task, usage, current_time - start_time ) - finally: - set_current_context(calling_context) async def wrapper() -> None: with nested_logging_context(task.id) as log_context: @@ -473,7 +467,10 @@ class TaskScheduler: occasional_status_call.stop() # Try launch a new task since we've finished with this one. - self._clock.call_later(0.1, self._launch_scheduled_tasks) + self._clock.call_later( + 0.1, + self._launch_scheduled_tasks, + ) if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: return @@ -493,4 +490,4 @@ class TaskScheduler: self._running_tasks.add(task.id) await self.update_task(task.id, status=TaskStatus.ACTIVE) - run_as_background_process(f"task-{task.action}", self.server_name, wrapper) + self.hs.run_as_background_process(f"task-{task.action}", wrapper) diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py index 03bf5a94a7..cf9c836e06 100644 --- a/synmark/suites/logging.py +++ b/synmark/suites/logging.py @@ -86,7 +86,9 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: hs_config = Config() # To be able to sleep. - clock = Clock(reactor) + # Ignore linter error here since we are running outside of the context of a + # Synapse `HomeServer`. + clock = Clock(reactor, server_name=hs_config.server.server_name) # type: ignore[multiple-internal-clocks] errors = StringIO() publisher = LogPublisher() diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index d109441e55..830a3daa8f 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -23,13 +23,20 @@ from pyperf import perf_counter from synapse.types import ISynapseReactor from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ - cache: LruCache[int, bool] = LruCache(max_size=loops) + # Ignore linter error here since we are running outside of the context of a + # Synapse `HomeServer`. + cache: LruCache[int, bool] = LruCache( + max_size=loops, + clock=Clock(reactor, server_name="synmark_benchmark"), # type: ignore[multiple-internal-clocks] + server_name="synmark_benchmark", + ) start = perf_counter() diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index 00cfdd0447..c67e0c9001 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -23,6 +23,7 @@ from pyperf import perf_counter from synapse.types import ISynapseReactor from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock async def main(reactor: ISynapseReactor, loops: int) -> float: @@ -30,7 +31,13 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ - cache: LruCache[int, bool] = LruCache(max_size=loops // 2) + # Ignore linter error here since we are running outside of the context of a + # Synapse `HomeServer`. + cache: LruCache[int, bool] = LruCache( + max_size=loops // 2, + clock=Clock(reactor, server_name="synmark_benchmark"), # type: ignore[multiple-internal-clocks] + server_name="synmark_benchmark", + ) start = perf_counter() diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py new file mode 100644 index 0000000000..d8119ba310 --- /dev/null +++ b/tests/app/test_homeserver_shutdown.py @@ -0,0 +1,193 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import gc +import weakref + +from synapse.app.homeserver import SynapseHomeServer +from synapse.storage.background_updates import UpdaterStatus + +from tests.server import ( + cleanup_test_reactor_system_event_triggers, + get_clock, + setup_test_homeserver, +) +from tests.unittest import HomeserverTestCase + + +class HomeserverCleanShutdownTestCase(HomeserverTestCase): + def setUp(self) -> None: + pass + + # NOTE: ideally we'd have another test to ensure we properly shutdown with + # real in-flight HTTP requests since those result in additional resources being + # setup that hold strong references to the homeserver. + # Mainly, the HTTP channel created by a real TCP connection from client to server + # is held open between requests and care needs to be taken in Twisted to ensure it is properly + # closed in a timely manner during shutdown. Simulating this behaviour in a unit test + # won't be as good as a proper integration test in complement. + + def test_clean_homeserver_shutdown(self) -> None: + """Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected""" + self.reactor, self.clock = get_clock() + self.hs = setup_test_homeserver( + cleanup_func=self.addCleanup, + reactor=self.reactor, + homeserver_to_use=SynapseHomeServer, + clock=self.clock, + ) + self.wait_for_background_updates() + + hs_ref = weakref.ref(self.hs) + + # Run the reactor so any `callWhenRunning` functions can be cleared out. + self.reactor.run() + # This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor` + # we use in tests doesn't handle this properly (see doc comment) + cleanup_test_reactor_system_event_triggers(self.reactor) + + # Cleanup the homeserver. + self.get_success(self.hs.shutdown()) + + # Cleanup the internal reference in our test case + del self.hs + + # Force garbage collection. + gc.collect() + + # Ensure the `HomeServer` hs been garbage collected by attempting to use the + # weakref to it. + if hs_ref() is not None: + self.fail("HomeServer reference should not be valid at this point") + + # To help debug this test when it fails, it is useful to leverage the + # `objgraph` module. + # The following code serves as an example of what I have found to be useful + # when tracking down references holding the `SynapseHomeServer` in memory: + # + # all_objects = gc.get_objects() + # for obj in all_objects: + # try: + # # These are a subset of types that are typically involved with + # # holding the `HomeServer` in memory. You may want to inspect + # # other types as well. + # if isinstance(obj, DataStore): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # db_obj = obj + # if isinstance(obj, SynapseHomeServer): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # synapse_hs = obj + # if isinstance(obj, SynapseSite): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # sysite = obj + # if isinstance(obj, DatabasePool): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # dbpool = obj + # except Exception: + # pass + # + # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) + # + # # The following values for `max_depth` and `too_many` have been found to + # # render a useful amount of information without taking an overly long time + # # to generate the result. + # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) + + def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: + """Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected + before background updates have completed""" + self.reactor, self.clock = get_clock() + self.hs = setup_test_homeserver( + cleanup_func=self.addCleanup, + reactor=self.reactor, + homeserver_to_use=SynapseHomeServer, + clock=self.clock, + ) + + # Pump the background updates by a single iteration, just to ensure any extra + # resources it uses have been started. + store = weakref.proxy(self.hs.get_datastores().main) + self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) + + hs_ref = weakref.ref(self.hs) + + # Run the reactor so any `callWhenRunning` functions can be cleared out. + self.reactor.run() + # This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor` + # we use in tests doesn't handle this properly (see doc comment) + cleanup_test_reactor_system_event_triggers(self.reactor) + + # Ensure the background updates are not complete. + self.assertNotEqual(store.db_pool.updates.get_status(), UpdaterStatus.COMPLETE) + + # Cleanup the homeserver. + self.get_success(self.hs.shutdown()) + + # Cleanup the internal reference in our test case + del self.hs + + # Force garbage collection. + gc.collect() + + # Ensure the `HomeServer` hs been garbage collected by attempting to use the + # weakref to it. + if hs_ref() is not None: + self.fail("HomeServer reference should not be valid at this point") + + # To help debug this test when it fails, it is useful to leverage the + # `objgraph` module. + # The following code serves as an example of what I have found to be useful + # when tracking down references holding the `SynapseHomeServer` in memory: + # + # all_objects = gc.get_objects() + # for obj in all_objects: + # try: + # # These are a subset of types that are typically involved with + # # holding the `HomeServer` in memory. You may want to inspect + # # other types as well. + # if isinstance(obj, DataStore): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # db_obj = obj + # if isinstance(obj, SynapseHomeServer): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # synapse_hs = obj + # if isinstance(obj, SynapseSite): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # sysite = obj + # if isinstance(obj, DatabasePool): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # dbpool = obj + # except Exception: + # pass + # + # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) + # + # # The following values for `max_depth` and `too_many` have been found to + # # render a useful amount of information without taking an overly long time + # # to generate the result. + # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 9498ea1279..f4490a1a79 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import List, Optional, Sequence, Tuple, cast +from typing import List, Optional, Sequence, Tuple from unittest.mock import AsyncMock, Mock from typing_extensions import TypeAlias @@ -44,13 +44,12 @@ from synapse.types import DeviceListUpdates, JsonDict from synapse.util.clock import Clock from tests import unittest - -from ..utils import MockClock +from tests.server import get_clock class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): def setUp(self) -> None: - self.clock = MockClock() + self.reactor, self.clock = get_clock() self.store = Mock() self.as_api = Mock() @@ -168,16 +167,18 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): ) -class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): +class ApplicationServiceSchedulerRecovererTestCase(unittest.HomeserverTestCase): def setUp(self) -> None: - self.clock = MockClock() + super().setUp() + self.reactor, self.clock = get_clock() self.as_api = Mock() self.store = Mock() self.service = Mock() self.callback = AsyncMock() self.recoverer = _Recoverer( server_name="test_server", - clock=cast(Clock, self.clock), + hs=self.hs, + clock=self.clock, as_api=self.as_api, store=self.store, service=self.service, @@ -202,7 +203,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): txn.send = AsyncMock(return_value=True) txn.complete = AsyncMock(return_value=None) # wait for exp backoff - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(1, txn.complete.call_count) # 2 because it needs to get None to know there are no more txns @@ -229,21 +230,21 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) txn.send = AsyncMock(return_value=False) txn.complete = AsyncMock(return_value=None) - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) - self.clock.advance_time(4) + self.reactor.advance(4) self.assertEqual(2, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) - self.clock.advance_time(8) + self.reactor.advance(8) self.assertEqual(3, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) txn.send = AsyncMock(return_value=True) # successfully send the txn pop_txn = True # returns the txn the first time, then no more. - self.clock.advance_time(16) + self.reactor.advance(16) self.assertEqual(1, txn.send.call_count) # new mock reset call count self.assertEqual(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) @@ -268,7 +269,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) txn.send = AsyncMock(return_value=False) txn.complete = AsyncMock(return_value=None) - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index deb6bade46..74db2dab08 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -24,6 +24,7 @@ from synapse.config.cache import CacheConfig, add_resizable_cache from synapse.types import JsonDict from synapse.util.caches.lrucache import LruCache +from tests.server import get_clock from tests.unittest import TestCase @@ -32,6 +33,7 @@ class CacheConfigTests(TestCase): # Reset caches before each test since there's global state involved. self.config = CacheConfig(RootConfig()) self.config.reset() + _, self.clock = get_clock() def tearDown(self) -> None: # Also reset the caches after each test to leave state pristine. @@ -75,7 +77,9 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized once the config is loaded. """ - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -96,7 +100,9 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 200) @@ -106,7 +112,9 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized to the new default cache size once the config is loaded. """ - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -126,7 +134,9 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 150) @@ -145,15 +155,21 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache_a: LruCache = LruCache(max_size=100) + cache_a: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) self.assertEqual(cache_a.max_size, 200) - cache_b: LruCache = LruCache(max_size=100) + cache_b: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("*Cache_b*", cache_resize_callback=cache_b.set_cache_factor) self.assertEqual(cache_b.max_size, 300) - cache_c: LruCache = LruCache(max_size=100) + cache_c: LruCache = LruCache( + max_size=100, clock=self.clock, server_name="test_server" + ) add_resizable_cache("*cache_c*", cache_resize_callback=cache_c.set_cache_factor) self.assertEqual(cache_c.max_size, 200) @@ -168,7 +184,9 @@ class CacheConfigTests(TestCase): cache: LruCache = LruCache( max_size=self.config.event_cache_size, + clock=self.clock, apply_cache_factor_from_config=False, + server_name="test_server", ) add_resizable_cache("event_cache", cache_resize_callback=cache.set_cache_factor) diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index 833cfe628b..85e0a3b6b6 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -231,7 +231,10 @@ class MSC3861OAuthDelegation(TestCase): reactor, clock = get_clock() with self.assertRaises(ConfigError): setup_test_homeserver( - self.addCleanup, reactor=reactor, clock=clock, config=config + cleanup_func=self.addCleanup, + config=config, + reactor=reactor, + clock=clock, ) def test_jwt_auth_cannot_be_enabled(self) -> None: @@ -395,7 +398,10 @@ class MasAuthDelegation(TestCase): reactor, clock = get_clock() with self.assertRaises(ConfigError): setup_test_homeserver( - self.addCleanup, reactor=reactor, clock=clock, config=config + cleanup_func=self.addCleanup, + config=config, + reactor=reactor, + clock=clock, ) @skip_unless(HAS_AUTHLIB, "requires authlib") diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 1dc0de73fd..93ae24628a 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -121,7 +121,9 @@ class KeyringTestCase(unittest.HomeserverTestCase): async def first_lookup() -> None: with LoggingContext( - "context_11", request=cast(ContextRequest, FakeRequest("context_11")) + name="context_11", + server_name=self.hs.hostname, + request=cast(ContextRequest, FakeRequest("context_11")), ): res_deferreds = kr.verify_json_objects_for_server( [("server10", json1, 0), ("server11", {}, 0)] @@ -161,7 +163,9 @@ class KeyringTestCase(unittest.HomeserverTestCase): async def second_lookup() -> None: with LoggingContext( - "context_12", request=cast(ContextRequest, FakeRequest("context_12")) + name="context_12", + server_name=self.hs.hostname, + request=cast(ContextRequest, FakeRequest("context_12")), ): res_deferreds_2 = kr.verify_json_objects_for_server( [ diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 262f94367c..509f1f1e82 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -229,7 +229,10 @@ class MessageAcceptTests(unittest.FederatingHomeserverTestCase): room_version=RoomVersions.V10, ) - with LoggingContext("test-context"): + with LoggingContext( + name="test-context", + server_name=self.hs.hostname, + ): failure = self.get_failure( self.federation_event_handler.on_receive_pdu( self.OTHER_SERVER_NAME, lying_event diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 999d7f5e6c..df36185b99 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -19,7 +19,17 @@ # # -from typing import Dict, Iterable, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + Iterable, + List, + Optional, + TypeVar, +) from unittest.mock import AsyncMock, Mock from parameterized import parameterized @@ -36,6 +46,7 @@ from synapse.appservice import ( TransactionUnusedFallbackKeys, ) from synapse.handlers.appservice import ApplicationServicesHandler +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client import login, receipts, register, room, sendtodevice from synapse.server import HomeServer from synapse.types import ( @@ -49,9 +60,14 @@ from synapse.util.clock import Clock from synapse.util.stringutils import random_string from tests import unittest +from tests.server import get_clock from tests.test_utils import event_injection from tests.unittest import override_config -from tests.utils import MockClock + +if TYPE_CHECKING: + from typing_extensions import LiteralString + +R = TypeVar("R") class AppServiceHandlerTestCase(unittest.TestCase): @@ -61,14 +77,27 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_store = Mock() self.mock_as_api = AsyncMock() self.mock_scheduler = Mock() + self.reactor, self.clock = get_clock() + hs = Mock() + + def test_run_as_background_process( + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[R]]], + *args: Any, + **kwargs: Any, + ) -> "defer.Deferred[Optional[R]]": + # Ignore linter error as this is used only for testing purposes (i.e. outside of Synapse). + return run_as_background_process(desc, "test_server", func, *args, **kwargs) # type: ignore[untracked-background-process] + + hs.run_as_background_process = test_run_as_background_process hs.get_datastores.return_value = Mock(main=self.mock_store) self.mock_store.get_appservice_last_pos = AsyncMock(return_value=None) self.mock_store.set_appservice_last_pos = AsyncMock(return_value=None) self.mock_store.set_appservice_stream_type_pos = AsyncMock(return_value=None) hs.get_application_service_api.return_value = self.mock_as_api hs.get_application_service_scheduler.return_value = self.mock_scheduler - hs.get_clock.return_value = MockClock() + hs.get_clock.return_value = self.clock self.handler = ApplicationServicesHandler(hs) self.event_source = hs.get_event_sources() diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 4f0b1574b3..fca1f2cc44 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -410,7 +410,6 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): device_id = "xyz" fallback_key = {"alg1:k1": "fallback_key1"} fallback_key2 = {"alg1:k2": "fallback_key2"} - fallback_key3 = {"alg1:k2": "fallback_key3"} otk = {"alg1:k2": "key2"} # we shouldn't have any unused fallback keys yet @@ -531,28 +530,6 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): {"failures": {}, "one_time_keys": {local_user: {device_id: fallback_key2}}}, ) - # using the unstable prefix should also set the fallback key - self.get_success( - self.handler.upload_keys_for_user( - local_user, - device_id, - {"org.matrix.msc2732.fallback_keys": fallback_key3}, - ) - ) - - claim_res = self.get_success( - self.handler.claim_one_time_keys( - {local_user: {device_id: {"alg1": 1}}}, - self.requester, - timeout=None, - always_include_fallback_keys=False, - ) - ) - self.assertEqual( - claim_res, - {"failures": {}, "one_time_keys": {local_user: {device_id: fallback_key3}}}, - ) - def test_fallback_key_bulk(self) -> None: """Like test_fallback_key, but claims multiple keys in one handler call.""" alice = f"@alice:{self.hs.hostname}" diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 910c24c167..5085a0309b 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -21,7 +21,6 @@ # import copy -from unittest import mock from twisted.internet.testing import MemoryReactor @@ -50,7 +49,7 @@ room_keys = { class E2eRoomKeysHandlerTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - return self.setup_test_homeserver(replication_layer=mock.Mock()) + return self.setup_test_homeserver() def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.handler = hs.get_e2e_room_keys_handler() diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 77ea03e67d..c9ece68729 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -318,6 +318,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): current_depth = 1 limit = 100 + # Make sure backfill still works self.get_success( self.hs.get_federation_handler().maybe_backfill( @@ -485,6 +486,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): # the auth code requires that a signature exists, but doesn't check that # signature... go figure. join_event.signatures[other_server] = {"x": "y"} + self.get_success( self.hs.get_federation_event_handler().on_send_membership_event( other_server, join_event diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 3b3f12796e..5771699a62 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -224,7 +224,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): self.reactor.advance(60000) # Finally, the call under test: send the pulled event into _process_pulled_event - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, pulled_event, backfilled=False @@ -321,7 +324,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) # The function under test: try to process the pulled event - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, pulled_event, backfilled=True @@ -339,7 +345,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(backfill_num_attempts, 1) # The function under test: try to process the pulled event again - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, pulled_event, backfilled=True @@ -447,7 +456,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(backfill_num_attempts, 1) # The function under test: try to process the pulled event - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, pulled_event, backfilled=True @@ -602,7 +614,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): ) # The function under test: try to backfill and process the pulled event - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler().backfill( self.OTHER_SERVER_NAME, @@ -742,7 +757,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): } # The function under test: try to backfill and process the pulled event - with LoggingContext("test"): + with LoggingContext( + name="test", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler().backfill( self.OTHER_SERVER_NAME, @@ -887,7 +905,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): next_depth += 1 next_timestamp += 100 - with LoggingContext("send_rejected_power_levels_event"): + with LoggingContext( + name="send_rejected_power_levels_event", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, @@ -969,7 +990,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): "during state resolution. The test setup is incorrect.", ) - with LoggingContext("send_rejected_kick_event"): + with LoggingContext( + name="send_rejected_kick_event", + server_name=self.hs.hostname, + ): self.get_success( self.hs.get_federation_event_handler()._process_pulled_event( self.OTHER_SERVER_NAME, rejected_kick_event, backfilled=False @@ -1085,7 +1109,10 @@ class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): # We have to bump the clock a bit, to keep the retry logic in # `FederationClient.get_pdu` happy self.reactor.advance(60000) - with LoggingContext("send_pulled_event"): + with LoggingContext( + name="send_pulled_event", + server_name=self.hs.hostname, + ): async def get_event( destination: str, event_id: str, timeout: Optional[int] = None diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index d1d0c484fa..00da1d942f 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -15,11 +15,17 @@ from typing import Optional from unittest import mock +import signedjson +from signedjson.key import encode_verify_key_base64, get_verify_key + from twisted.internet.testing import MemoryReactor +from synapse.api.errors import SynapseError +from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase, make_event_from_dict +from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM @@ -36,16 +42,24 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): admin.register_servlets, login.register_servlets, room.register_servlets, + filter.register_servlets, + sync.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: # mock out the federation transport client self.mock_federation_transport_client = mock.Mock( - spec=["get_policy_recommendation_for_pdu"] + spec=[ + "get_policy_recommendation_for_pdu", + "ask_policy_server_to_sign_event", + ] ) self.mock_federation_transport_client.get_policy_recommendation_for_pdu = ( mock.AsyncMock() ) + self.mock_federation_transport_client.ask_policy_server_to_sign_event = ( + mock.AsyncMock() + ) return super().setup_test_homeserver( federation_transport_client=self.mock_federation_transport_client ) @@ -62,6 +76,8 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): room_creator=self.creator, tok=self.creator_token ) room_version = self.get_success(main_store.get_room_version(self.room_id)) + self.room_version = room_version + self.signing_key = signedjson.key.generate_signing_key("policy_server") # Create some sample events self.spammy_event = make_event_from_dict( @@ -110,7 +126,48 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): self.mock_federation_transport_client.get_policy_recommendation_for_pdu.side_effect = get_policy_recommendation_for_pdu - def _add_policy_server_to_room(self) -> None: + # Mock policy server actions on signing events + async def policy_server_signs_event( + destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> Optional[JsonDict]: + sigs = compute_event_signature( + pdu.room_version, + pdu.get_dict(), + self.OTHER_SERVER_NAME, + self.signing_key, + ) + return sigs + + async def policy_server_signs_event_with_wrong_key( + destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> Optional[JsonDict]: + sk = signedjson.key.generate_signing_key("policy_server") + sigs = compute_event_signature( + pdu.room_version, + pdu.get_dict(), + self.OTHER_SERVER_NAME, + sk, + ) + return sigs + + async def policy_server_refuses_to_sign_event( + destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> Optional[JsonDict]: + return {} + + async def policy_server_event_sign_error( + destination: str, pdu: EventBase, timeout: Optional[int] = None + ) -> Optional[JsonDict]: + return None + + self.policy_server_signs_event = policy_server_signs_event + self.policy_server_refuses_to_sign_event = policy_server_refuses_to_sign_event + self.policy_server_event_sign_error = policy_server_event_sign_error + self.policy_server_signs_event_with_wrong_key = ( + policy_server_signs_event_with_wrong_key + ) + + def _add_policy_server_to_room(self, public_key: Optional[str] = None) -> None: # Inject a member event into the room policy_user_id = f"@policy:{self.OTHER_SERVER_NAME}" self.get_success( @@ -118,12 +175,15 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): self.hs, self.room_id, policy_user_id, "join" ) ) + content = { + "via": self.OTHER_SERVER_NAME, + } + if public_key is not None: + content["public_key"] = public_key self.helper.send_state( self.room_id, "org.matrix.msc4284.policy", - { - "via": self.OTHER_SERVER_NAME, - }, + content, tok=self.creator_token, state_key="", ) @@ -218,9 +278,192 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(ok, False) self.assertEqual(self.call_count, 1) - def test_not_spammy_event_is_not_spam(self) -> None: - self._add_policy_server_to_room() + def test_signed_event_is_not_spam(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is a signed event.", + }, + }, + ) - ok = self.get_success(self.handler.is_event_allowed(self.not_spammy_event)) + # We're going to sign the event and check it marks the event as not-spam, without hitting the + # policy server + sigs = compute_event_signature( + event.room_version, + event.get_dict(), + self.OTHER_SERVER_NAME, + self.signing_key, + ) + event.signatures.update(sigs) + + ok = self.get_success(self.handler.is_event_allowed(event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 1) + # Make sure we did not make an HTTP hit to get_policy_recommendation_for_pdu + self.assertEqual(self.call_count, 0) + + def test_ask_policy_server_to_sign_event_ok(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is another signed event.", + }, + }, + ) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + self.assertEqual(len(event.signatures), 1) + + def test_ask_policy_server_to_sign_event_refuses(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is spam and is refused.", + }, + }, + ) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_refuses_to_sign_event + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + self.assertEqual(len(event.signatures), 0) + + def test_ask_policy_server_to_sign_event_cannot_reach(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is spam and is refused.", + }, + }, + ) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_event_sign_error + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + self.assertEqual(len(event.signatures), 0) + + def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event_with_wrong_key + unverified_event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is signed but with the wrong key.", + }, + }, + ) + # verify=False so it passes + self.get_success( + self.handler.ask_policy_server_to_sign_event(unverified_event, verify=False) + ) + self.assertEqual(len(unverified_event.signatures), 1) + + verified_event = make_event_from_dict( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:example.org", + "content": { + "msgtype": "m.text", + "body": "This is signed but with the wrong key.", + }, + }, + ) + # verify=True so it fails + self.get_failure( + self.handler.ask_policy_server_to_sign_event(verified_event, verify=True), + SynapseError, + ) + + def test_policy_server_signatures_end_to_end(self) -> None: + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event + # Send an event and ensure we get a policy server signature on it. + resp = self.helper.send_event( + self.room_id, + "m.room.message", + {"body": "honk", "msgtype": "m.text"}, + tok=self.creator_token, + ) + ev = self._fetch_federation_event(resp["event_id"]) + assert ev is not None + sig = ( + ev.get("signatures", {}) + .get(self.OTHER_SERVER_NAME, {}) + .get(POLICY_SERVER_KEY_ID, None) + ) + self.assertNotEquals( + sig, + None, + f"event did not include policy server signature, signature block = {ev.get('signatures', None)}", + ) + + def _fetch_federation_event(self, event_id: str) -> Optional[JsonDict]: + # Request federation events to see the signatures + channel = self.make_request( + "POST", + "/_matrix/client/v3/user/%s/filter" % (self.creator), + {"event_format": "federation"}, + self.creator_token, + ) + self.assertEqual(channel.code, 200) + filter_id = channel.json_body["filter_id"] + # Note: we could use `/context`, but given we don't test that neutral events are + # delivered over `/sync` anywhere else, might as well implicitly test it here. + channel = self.make_request( + "GET", + "/sync?filter=%s" % filter_id, + access_token=self.creator_token, + ) + self.assertEqual(channel.code, 200, channel.result) + + for ev in channel.json_body["rooms"]["join"][self.room_id]["timeline"][ + "events" + ]: + if ev["event_id"] == event_id: + return ev + return None diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 4d2807151e..90c185bc3d 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -79,15 +79,17 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): ) -> HomeServer: # we mock out the keyring so as to skip the authentication check on the # federation API call. - mock_keyring = Mock(spec=["verify_json_for_server"]) + mock_keyring = Mock(spec=["verify_json_for_server", "shutdown"]) mock_keyring.verify_json_for_server = AsyncMock(return_value=True) + mock_keyring.shutdown = Mock() # we mock out the federation client too self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", - reactor=reactor, + reactor=self.reactor, + clock=self.clock, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, @@ -96,7 +98,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): ) # the tests assume that we are starting at unix time 1000 - reactor.pump((1000,)) + self.reactor.pump((1000,)) self.mock_hs_notifier = Mock() hs = self.setup_test_homeserver( diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 12428e64a9..c66ca489a4 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -65,7 +65,7 @@ from synapse.util.caches.ttlcache import TTLCache from tests import unittest from tests.http import dummy_address, get_test_ca_cert_file, wrap_server_factory_for_tls -from tests.server import FakeTransport, ThreadedMemoryReactorClock +from tests.server import FakeTransport, get_clock from tests.utils import checked_cast, default_config logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ logger = logging.getLogger(__name__) class MatrixFederationAgentTests(unittest.TestCase): def setUp(self) -> None: - self.reactor = ThreadedMemoryReactorClock() + self.reactor, self.clock = get_clock() self.mock_resolver = AsyncMock(spec=SrvResolver) @@ -98,6 +98,7 @@ class MatrixFederationAgentTests(unittest.TestCase): self.well_known_resolver = WellKnownResolver( server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, + clock=self.clock, agent=Agent(self.reactor, contextFactory=self.tls_factory), user_agent=b"test-agent", well_known_cache=self.well_known_cache, @@ -200,7 +201,10 @@ class MatrixFederationAgentTests(unittest.TestCase): """ Sends a simple GET request via the agent, and checks its logcontext management """ - with LoggingContext("one") as context: + with LoggingContext( + name="one", + server_name="test_server", + ) as context: fetch_d: Deferred[IResponse] = self.agent.request(b"GET", uri) # Nothing happened yet @@ -277,6 +281,7 @@ class MatrixFederationAgentTests(unittest.TestCase): return MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), + clock=self.clock, tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. ip_allowlist=IPSet(), @@ -1021,6 +1026,7 @@ class MatrixFederationAgentTests(unittest.TestCase): agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, + clock=self.clock, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. ip_allowlist=IPSet(), @@ -1030,6 +1036,7 @@ class MatrixFederationAgentTests(unittest.TestCase): _well_known_resolver=WellKnownResolver( server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), + clock=self.clock, agent=Agent(self.reactor, contextFactory=tls_factory), user_agent=b"test-agent", well_known_cache=self.well_known_cache, diff --git a/tests/http/federation/test_srv_resolver.py b/tests/http/federation/test_srv_resolver.py index ffcbf4b3ca..a359b0a141 100644 --- a/tests/http/federation/test_srv_resolver.py +++ b/tests/http/federation/test_srv_resolver.py @@ -30,7 +30,7 @@ from synapse.http.federation.srv_resolver import Server, SrvResolver from synapse.logging.context import LoggingContext, current_context from tests import unittest -from tests.utils import MockClock +from tests.server import get_clock class SrvResolverTestCase(unittest.TestCase): @@ -52,7 +52,10 @@ class SrvResolverTestCase(unittest.TestCase): @defer.inlineCallbacks def do_lookup() -> Generator["Deferred[object]", object, List[Server]]: - with LoggingContext("one") as ctx: + with LoggingContext( + name="one", + server_name="test_server", + ) as ctx: resolve_d = resolver.resolve_service(service_name) result: List[Server] result = yield defer.ensureDeferred(resolve_d) # type: ignore[assignment] @@ -102,7 +105,7 @@ class SrvResolverTestCase(unittest.TestCase): @defer.inlineCallbacks def test_from_cache(self) -> Generator["Deferred[object]", object, None]: - clock = MockClock() + reactor, clock = get_clock() dns_client_mock = Mock(spec_set=["lookupService"]) dns_client_mock.lookupService = Mock(spec_set=[]) diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py index 393f3ab0bd..8eec4329fe 100644 --- a/tests/http/server/_base.py +++ b/tests/http/server/_base.py @@ -502,7 +502,7 @@ def _log_for_request(request_number: int, message: str) -> None: """Logs a message for an iteration of `make_request_with_cancellation_test`.""" # We want consistent alignment when logging stack traces, so ensure the logging # context has a fixed width name. - with LoggingContext(name=f"request-{request_number:<2}"): + with LoggingContext(name=f"request-{request_number:<2}", server_name="test_server"): logger.info(message) diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py index 44752d69cb..6d87541888 100644 --- a/tests/http/test_matrixfederationclient.py +++ b/tests/http/test_matrixfederationclient.py @@ -63,10 +63,6 @@ def check_logcontext(context: LoggingContextOrSentinel) -> None: class FederationClientTests(HomeserverTestCase): - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver(reactor=reactor, clock=clock) - return hs - def prepare( self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer ) -> None: @@ -80,7 +76,10 @@ class FederationClientTests(HomeserverTestCase): @defer.inlineCallbacks def do_request() -> Generator["Deferred[Any]", object, object]: - with LoggingContext("one") as context: + with LoggingContext( + name="one", + server_name=self.hs.hostname, + ) as context: fetch_d = defer.ensureDeferred( self.cl.get_json("testserv:8008", "foo/bar") ) diff --git a/tests/logging/test_opentracing.py b/tests/logging/test_opentracing.py index d102d50138..2f389f7f44 100644 --- a/tests/logging/test_opentracing.py +++ b/tests/logging/test_opentracing.py @@ -19,7 +19,7 @@ # # -from typing import Awaitable, Dict, cast +from typing import Awaitable, Optional, cast from twisted.internet import defer from twisted.internet.testing import MemoryReactorClock @@ -35,20 +35,25 @@ from synapse.logging.opentracing import ( tag_args, trace_with_opname, ) +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.clock import Clock -try: - import opentracing - from opentracing.scope_managers.contextvars import ContextVarsScopeManager -except ImportError: - opentracing = None # type: ignore - ContextVarsScopeManager = None # type: ignore +from tests.server import get_clock try: import jaeger_client except ImportError: jaeger_client = None # type: ignore + +try: + import opentracing + + from synapse.logging.scopecontextmanager import LogContextScopeManager +except ImportError: + opentracing = None # type: ignore + LogContextScopeManager = None # type: ignore + import logging from tests.unittest import TestCase @@ -56,7 +61,7 @@ from tests.unittest import TestCase logger = logging.getLogger(__name__) -class TracingScopeTestCase(TestCase): +class LogContextScopeManagerTestCase(TestCase): """ Test that our tracing machinery works well in a variety of situations (especially with Twisted's runtime and deferreds). @@ -67,7 +72,7 @@ class TracingScopeTestCase(TestCase): opentracing backend is Jaeger. """ - if opentracing is None: + if opentracing is None or LogContextScopeManager is None: skip = "Requires opentracing" # type: ignore[unreachable] if jaeger_client is None: skip = "Requires jaeger_client" # type: ignore[unreachable] @@ -77,9 +82,8 @@ class TracingScopeTestCase(TestCase): # global variables that power opentracing. We create our own tracer instance # and test with it. - scope_manager = ContextVarsScopeManager() config = jaeger_client.config.Config( - config={}, service_name="test", scope_manager=scope_manager + config={}, service_name="test", scope_manager=LogContextScopeManager() ) self._reporter = jaeger_client.reporter.InMemoryReporter() @@ -91,7 +95,7 @@ class TracingScopeTestCase(TestCase): def test_start_active_span(self) -> None: # the scope manager assumes a logging context of some sort. - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): self.assertIsNone(self._tracer.active_span) # start_active_span should start and activate a span. @@ -115,7 +119,7 @@ class TracingScopeTestCase(TestCase): def test_nested_spans(self) -> None: """Starting two spans off inside each other should work""" - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): with start_active_span("root span", tracer=self._tracer) as root_scope: self.assertEqual(self._tracer.active_span, root_scope.span) root_context = cast(jaeger_client.SpanContext, root_scope.span.context) @@ -163,8 +167,11 @@ class TracingScopeTestCase(TestCase): # implements `ISynapseThreadlessReactor` (combination of the normal Twisted # Reactor/Clock interfaces), via inheritance from # `twisted.internet.testing.MemoryReactor` and `twisted.internet.testing.Clock` - clock = Clock( - reactor # type: ignore[arg-type] + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes. + clock = Clock( # type: ignore[multiple-internal-clocks] + reactor, # type: ignore[arg-type] + server_name="test_server", ) scopes = [] @@ -200,7 +207,7 @@ class TracingScopeTestCase(TestCase): self.assertEqual(self._tracer.active_span, root_scope.span) - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): # start the test off d1 = defer.ensureDeferred(root()) @@ -217,147 +224,12 @@ class TracingScopeTestCase(TestCase): [scopes[1].span, scopes[2].span, scopes[0].span], ) - def test_run_in_background_active_scope_still_available(self) -> None: - """ - Test that tasks running via `run_in_background` still have access to the - active tracing scope. - - This is a regression test for a previous Synapse issue where the tracing scope - would `__exit__` and close before the `run_in_background` task completed and our - own previous custom `_LogContextScope.close(...)` would clear - `LoggingContext.scope` preventing further tracing spans from having the correct - parent. - """ - reactor = MemoryReactorClock() - # type-ignore: mypy-zope doesn't seem to recognise that `MemoryReactorClock` - # implements `ISynapseThreadlessReactor` (combination of the normal Twisted - # Reactor/Clock interfaces), via inheritance from - # `twisted.internet.testing.MemoryReactor` and `twisted.internet.testing.Clock` - clock = Clock( - reactor # type: ignore[arg-type] - ) - - scope_map: Dict[str, opentracing.Scope] = {} - - async def async_task() -> None: - root_scope = scope_map["root"] - root_context = cast(jaeger_client.SpanContext, root_scope.span.context) - - self.assertEqual( - self._tracer.active_span, - root_scope.span, - "expected to inherit the root tracing scope from where this was run", - ) - - # Return control back to the reactor thread and wait an arbitrary amount - await clock.sleep(4) - - # This is a key part of what we're testing! In a previous version of - # Synapse, we would lose the active span at this point. - self.assertEqual( - self._tracer.active_span, - root_scope.span, - "expected to still have a root tracing scope/span active", - ) - - # For complete-ness sake, let's also trace more sub-tasks here and assert - # they have the correct span parents as well (root) - - # Start tracing some other sub-task. - # - # This is a key part of what we're testing! In a previous version of - # Synapse, it would have the incorrect span parents. - scope = start_active_span( - "task1", - tracer=self._tracer, - ) - scope_map["task1"] = scope - - # Ensure the span parent is pointing to the root scope - context = cast(jaeger_client.SpanContext, scope.span.context) - self.assertEqual( - context.parent_id, - root_context.span_id, - "expected task1 parent to be the root span", - ) - - # Ensure that the active span is our new sub-task now - self.assertEqual(self._tracer.active_span, scope.span) - # Return control back to the reactor thread and wait an arbitrary amount - await clock.sleep(4) - # We should still see the active span as the scope wasn't closed yet - self.assertEqual(self._tracer.active_span, scope.span) - scope.close() - - async def root() -> None: - with start_active_span( - "root span", - tracer=self._tracer, - # We will close this off later. We're basically just mimicking the same - # pattern for how we handle requests. We pass the span off to the - # request for it to finish. - finish_on_close=False, - ) as root_scope: - scope_map["root"] = root_scope - self.assertEqual(self._tracer.active_span, root_scope.span) - - # Fire-and-forget a task - # - # XXX: The root scope context manager will `__exit__` before this task - # completes. - run_in_background(async_task) - - # Because we used `run_in_background`, the active span should still be - # the root. - self.assertEqual(self._tracer.active_span, root_scope.span) - - # We shouldn't see any active spans outside of the scope - self.assertIsNone(self._tracer.active_span) - - with LoggingContext("root context"): - # Start the test off - d_root = defer.ensureDeferred(root()) - - # Let the tasks complete - reactor.pump((2,) * 8) - self.successResultOf(d_root) - - # After we see all of the tasks are done (like a request when it - # `_finished_processing`), let's finish our root span - scope_map["root"].span.finish() - - # Sanity check again: We shouldn't see any active spans leftover in this - # this context. - self.assertIsNone(self._tracer.active_span) - - # The spans should be reported in order of their finishing: task 1, task 2, - # root. - # - # We use `assertIncludes` just as an easier way to see if items are missing or - # added. We assert the order just below - self.assertIncludes( - set(self._reporter.get_spans()), - { - scope_map["task1"].span, - scope_map["root"].span, - }, - exact=True, - ) - # This is where we actually assert the correct order - self.assertEqual( - self._reporter.get_spans(), - [ - scope_map["task1"].span, - scope_map["root"].span, - ], - ) - def test_trace_decorator_sync(self) -> None: """ Test whether we can use `@trace_with_opname` (`@trace`) and `@tag_args` with sync functions """ - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): @trace_with_opname("fixture_sync_func", tracer=self._tracer) @tag_args @@ -378,7 +250,7 @@ class TracingScopeTestCase(TestCase): Test whether we can use `@trace_with_opname` (`@trace`) and `@tag_args` with functions that return deferreds """ - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): @trace_with_opname("fixture_deferred_func", tracer=self._tracer) @tag_args @@ -402,7 +274,7 @@ class TracingScopeTestCase(TestCase): Test whether we can use `@trace_with_opname` (`@trace`) and `@tag_args` with async functions """ - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): @trace_with_opname("fixture_async_func", tracer=self._tracer) @tag_args @@ -424,7 +296,7 @@ class TracingScopeTestCase(TestCase): Test whether we can use `@trace_with_opname` (`@trace`) and `@tag_args` with functions that return an awaitable (e.g. a coroutine) """ - with LoggingContext("root context"): + with LoggingContext(name="root context", server_name="test_server"): # Something we can return without `await` to get a coroutine async def fixture_async_func() -> str: return "foo" @@ -449,3 +321,203 @@ class TracingScopeTestCase(TestCase): [span.operation_name for span in self._reporter.get_spans()], ["fixture_awaitable_return_func"], ) + + async def test_run_as_background_process_standalone(self) -> None: + """ + Test to make sure that the background process work starts its own trace. + """ + reactor, clock = get_clock() + + callback_finished = False + active_span_in_callback: Optional[jaeger_client.Span] = None + + async def bg_task() -> None: + nonlocal callback_finished, active_span_in_callback + try: + assert isinstance(self._tracer.active_span, jaeger_client.Span) + active_span_in_callback = self._tracer.active_span + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + # type-ignore: We ignore because the point is to test the bare function + run_as_background_process( # type: ignore[untracked-background-process] + desc="some-bg-task", + server_name="test_server", + func=bg_task, + test_only_tracer=self._tracer, + ) + + # Now wait for the background process to finish + while not callback_finished: + await clock.sleep(0) + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + self.assertEqual( + active_span_in_callback.operation_name if active_span_in_callback else None, + "bgproc.some-bg-task", + "expected a new span to be started for the background task", + ) + + # The spans should be reported in order of their finishing. + # + # We use `assertIncludes` just as an easier way to see if items are missing or + # added. We assert the order just below + actual_spans = [span.operation_name for span in self._reporter.get_spans()] + expected_spans = ["bgproc.some-bg-task"] + self.assertIncludes( + set(actual_spans), + set(expected_spans), + exact=True, + ) + # This is where we actually assert the correct order + self.assertEqual( + actual_spans, + expected_spans, + ) + + async def test_run_as_background_process_cross_link(self) -> None: + """ + Test to make sure that the background process work has its own trace and is + disconnected from any currently active trace (like a request). But we still have + cross-links between the two traces if there was already an active trace/span when + we kicked off the background process. + """ + reactor, clock = get_clock() + + callback_finished = False + active_span_in_callback: Optional[jaeger_client.Span] = None + + async def bg_task() -> None: + nonlocal callback_finished, active_span_in_callback + try: + assert isinstance(self._tracer.active_span, jaeger_client.Span) + active_span_in_callback = self._tracer.active_span + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + with LoggingContext(name="some-request", server_name="test_server"): + with start_active_span( + "some-request", + tracer=self._tracer, + ): + # type-ignore: We ignore because the point is to test the bare function + run_as_background_process( # type: ignore[untracked-background-process] + desc="some-bg-task", + server_name="test_server", + func=bg_task, + test_only_tracer=self._tracer, + ) + + # Now wait for the background process to finish + while not callback_finished: + await clock.sleep(0) + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + # We start `bgproc.some-bg-task` and `bgproc_child.some-bg-task` (see + # `run_as_background_process` implementation for why). Either is fine but for + # now we expect the child as its the innermost one that was started. + self.assertEqual( + active_span_in_callback.operation_name if active_span_in_callback else None, + "bgproc_child.some-bg-task", + "expected a new span to be started for the background task", + ) + + # The spans should be reported in order of their finishing. + # + # We use `assertIncludes` just as an easier way to see if items are missing or + # added. We assert the order just below + actual_spans = [span.operation_name for span in self._reporter.get_spans()] + expected_spans = [ + "start_bgproc.some-bg-task", + "bgproc_child.some-bg-task", + "bgproc.some-bg-task", + "some-request", + ] + self.assertIncludes( + set(actual_spans), + set(expected_spans), + exact=True, + ) + # This is where we actually assert the correct order + self.assertEqual( + actual_spans, + expected_spans, + ) + + span_map = {span.operation_name: span for span in self._reporter.get_spans()} + span_id_to_friendly_name = { + span.span_id: span.operation_name for span in self._reporter.get_spans() + } + + def get_span_friendly_name(span_id: Optional[int]) -> str: + if span_id is None: + return "None" + + return span_id_to_friendly_name.get(span_id, f"unknown span {span_id}") + + # Ensure the background process trace/span is disconnected from the request + # trace/span. + self.assertNotEqual( + get_span_friendly_name(span_map["bgproc.some-bg-task"].parent_id), + get_span_friendly_name(span_map["some-request"].span_id), + ) + + # We should see a cross-link in the request trace pointing to the background + # process trace. + # + # Make sure `start_bgproc.some-bg-task` is part of the request trace + self.assertEqual( + get_span_friendly_name(span_map["start_bgproc.some-bg-task"].parent_id), + get_span_friendly_name(span_map["some-request"].span_id), + ) + # And has some references to the background process trace + self.assertIncludes( + { + f"{reference.type}:{get_span_friendly_name(reference.referenced_context.span_id)}" + if isinstance(reference.referenced_context, jaeger_client.SpanContext) + else f"{reference.type}:None" + for reference in ( + span_map["start_bgproc.some-bg-task"].references or [] + ) + }, + { + f"follows_from:{get_span_friendly_name(span_map['bgproc.some-bg-task'].span_id)}" + }, + exact=True, + ) + + # We should see a cross-link in the background process trace pointing to the + # request trace that kicked off the work. + # + # Make sure `start_bgproc.some-bg-task` is part of the request trace + self.assertEqual( + get_span_friendly_name(span_map["bgproc_child.some-bg-task"].parent_id), + get_span_friendly_name(span_map["bgproc.some-bg-task"].span_id), + ) + # And has some references to the background process trace + self.assertIncludes( + { + f"{reference.type}:{get_span_friendly_name(reference.referenced_context.span_id)}" + if isinstance(reference.referenced_context, jaeger_client.SpanContext) + else f"{reference.type}:None" + for reference in ( + span_map["bgproc_child.some-bg-task"].references or [] + ) + }, + { + f"follows_from:{get_span_friendly_name(span_map['some-request'].span_id)}" + }, + exact=True, + ) diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index 60de8d786f..a857737ddf 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -63,13 +63,13 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): log = self.get_log_line() # The terse logger should give us these keys. - expected_log_keys = [ + expected_log_keys = { "log", "time", "level", "namespace", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) self.assertEqual(log["log"], "Hello there, wally!") def test_extra_data(self) -> None: @@ -87,7 +87,7 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): log = self.get_log_line() # The terse logger should give us these keys. - expected_log_keys = [ + expected_log_keys = { "log", "time", "level", @@ -96,8 +96,8 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): "foo", "int", "bool", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) # Check the values of the extra fields. self.assertEqual(log["foo"], "bar") @@ -117,12 +117,12 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): log = self.get_log_line() # The terse logger should give us these keys. - expected_log_keys = [ + expected_log_keys = { "log", "level", "namespace", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) self.assertEqual(log["log"], "Hello there, wally!") def test_with_context(self) -> None: @@ -134,19 +134,20 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): handler.addFilter(LoggingContextFilter()) logger = self.get_logger(handler) - with LoggingContext("name"): + with LoggingContext(name="name", server_name="test_server"): logger.info("Hello there, %s!", "wally") log = self.get_log_line() # The terse logger should give us these keys. - expected_log_keys = [ + expected_log_keys = { "log", "level", "namespace", "request", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + "server_name", + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) self.assertEqual(log["log"], "Hello there, wally!") self.assertEqual(log["request"], "name") @@ -187,14 +188,16 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): request.requester = "@foo:test" with LoggingContext( - request.get_request_id(), parent_context=request.logcontext + name=request.get_request_id(), + server_name="test_server", + parent_context=request.logcontext, ): logger.info("Hello there, %s!", "wally") log = self.get_log_line() # The terse logger includes additional request information, if possible. - expected_log_keys = [ + expected_log_keys = { "log", "level", "namespace", @@ -207,8 +210,9 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): "url", "protocol", "user_agent", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + "server_name", + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) self.assertEqual(log["log"], "Hello there, wally!") self.assertTrue(log["request"].startswith("POST-")) self.assertEqual(log["ip_address"], "127.0.0.1") @@ -236,14 +240,14 @@ class TerseJsonTestCase(LoggerCleanupMixin, TestCase): log = self.get_log_line() # The terse logger should give us these keys. - expected_log_keys = [ + expected_log_keys = { "log", "level", "namespace", "exc_type", "exc_value", - ] - self.assertCountEqual(log.keys(), expected_log_keys) + } + self.assertIncludes(log.keys(), expected_log_keys, exact=True) self.assertEqual(log["log"], "Hello there, wally!") self.assertEqual(log["exc_type"], "ValueError") self.assertEqual(log["exc_value"], "That's wrong, you wally!") diff --git a/tests/media/test_media_retention.py b/tests/media/test_media_retention.py index aec1adb040..6dba214514 100644 --- a/tests/media/test_media_retention.py +++ b/tests/media/test_media_retention.py @@ -37,7 +37,6 @@ from synapse.util.stringutils import ( from tests import unittest from tests.unittest import override_config -from tests.utils import MockClock class MediaRetentionTestCase(unittest.HomeserverTestCase): @@ -51,12 +50,6 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase): admin.register_servlets_for_client_rest_resource, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - # We need to be able to test advancing time in the homeserver, so we - # replace the test homeserver's default clock with a MockClock, which - # supports advancing time. - return self.setup_test_homeserver(clock=MockClock()) - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.remote_server_name = "remote.homeserver" self.store = hs.get_datastores().main diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 832e991730..b3f42c76f1 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -164,7 +164,10 @@ class CacheMetricsTests(unittest.HomeserverTestCase): """ CACHE_NAME = "cache_metrics_test_fgjkbdfg" cache: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name=self.hs.hostname, max_entries=777 + name=CACHE_NAME, + clock=self.hs.get_clock(), + server_name=self.hs.hostname, + max_entries=777, ) metrics_map = get_latest_metrics() @@ -212,10 +215,10 @@ class CacheMetricsTests(unittest.HomeserverTestCase): """ CACHE_NAME = "cache_metric_multiple_servers_test" cache1: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name="hs1", max_entries=777 + name=CACHE_NAME, clock=self.clock, server_name="hs1", max_entries=777 ) cache2: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name="hs2", max_entries=777 + name=CACHE_NAME, clock=self.clock, server_name="hs2", max_entries=777 ) metrics_map = get_latest_metrics() diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py index 3f4d1d9d5f..d461d6cea2 100644 --- a/tests/module_api/test_spamchecker.py +++ b/tests/module_api/test_spamchecker.py @@ -16,6 +16,7 @@ from typing import Literal, Union from twisted.internet.testing import MemoryReactor +from synapse.api.constants import EventContentFields, EventTypes from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.rest import admin, login, room, room_upgrade_rest_servlet from synapse.server import HomeServer @@ -51,8 +52,8 @@ class SpamCheckerTestCase(HomeserverTestCase): return channel - def test_may_user_create_room(self) -> None: - """Test that the may_user_create_room callback is called when a user + def test_user_may_create_room(self) -> None: + """Test that the user_may_create_room callback is called when a user creates a room, and that it receives the correct parameters. """ @@ -67,16 +68,50 @@ class SpamCheckerTestCase(HomeserverTestCase): user_may_create_room=user_may_create_room ) - channel = self.create_room({"foo": "baa"}) + expected_room_config = {"foo": "baa"} + channel = self.create_room(expected_room_config) + self.assertEqual(channel.code, 200) self.assertEqual(self.last_user_id, self.user_id) - self.assertEqual(self.last_room_config["foo"], "baa") + self.assertEqual(self.last_room_config, expected_room_config) - def test_may_user_create_room_on_upgrade(self) -> None: - """Test that the may_user_create_room callback is called when a room is upgraded.""" + def test_user_may_create_room_with_initial_state(self) -> None: + """Test that the user_may_create_room callback is called when a user + creates a room with some initial state events, and that it receives the correct parameters. + """ + + async def user_may_create_room( + user_id: str, room_config: JsonDict + ) -> Union[Literal["NOT_SPAM"], Codes]: + self.last_room_config = room_config + self.last_user_id = user_id + return "NOT_SPAM" + + self._module_api.register_spam_checker_callbacks( + user_may_create_room=user_may_create_room + ) + + expected_room_config = { + "foo": "baa", + "initial_state": [ + { + "type": EventTypes.Topic, + "content": {EventContentFields.TOPIC: "foo"}, + } + ], + } + channel = self.create_room(expected_room_config) + + self.assertEqual(channel.code, 200) + self.assertEqual(self.last_user_id, self.user_id) + self.assertEqual(self.last_room_config, expected_room_config) + + def test_user_may_create_room_on_upgrade(self) -> None: + """Test that the user_may_create_room callback is called when a room is upgraded.""" # First, create a room to upgrade. - channel = self.create_room({"topic": "foo"}) + channel = self.create_room({EventContentFields.TOPIC: "foo"}) + self.assertEqual(channel.code, 200) room_id = channel.json_body["room_id"] @@ -107,13 +142,15 @@ class SpamCheckerTestCase(HomeserverTestCase): # Check that the initial state received by callback contains the topic event. self.assertTrue( any( - event[0][0] == "m.room.topic" and event[1].get("topic") == "foo" + event.get("type") == EventTypes.Topic + and event.get("state_key") == "" + and event.get("content").get(EventContentFields.TOPIC) == "foo" for event in self.last_room_config["initial_state"] ) ) - def test_may_user_create_room_disallowed(self) -> None: - """Test that the codes response from may_user_create_room callback is respected + def test_user_may_create_room_disallowed(self) -> None: + """Test that the codes response from user_may_create_room callback is respected and returned via the API. """ @@ -128,14 +165,16 @@ class SpamCheckerTestCase(HomeserverTestCase): user_may_create_room=user_may_create_room ) - channel = self.create_room({"foo": "baa"}) + expected_room_config = {"foo": "baa"} + channel = self.create_room(expected_room_config) + self.assertEqual(channel.code, 403) self.assertEqual(channel.json_body["errcode"], Codes.UNAUTHORIZED) self.assertEqual(self.last_user_id, self.user_id) - self.assertEqual(self.last_room_config["foo"], "baa") + self.assertEqual(self.last_room_config, expected_room_config) - def test_may_user_create_room_compatibility(self) -> None: - """Test that the may_user_create_room callback is called when a user + def test_user_may_create_room_compatibility(self) -> None: + """Test that the user_may_create_room callback is called when a user creates a room for a module that uses the old callback signature (without the `room_config` parameter) """ @@ -151,6 +190,7 @@ class SpamCheckerTestCase(HomeserverTestCase): ) channel = self.create_room({"foo": "baa"}) + self.assertEqual(channel.code, 200) self.assertEqual(self.last_user_id, self.user_id) @@ -178,6 +218,7 @@ class SpamCheckerTestCase(HomeserverTestCase): ) channel = self.create_room({}) + self.assertEqual(channel.code, 200) room_id = channel.json_body["room_id"] @@ -222,6 +263,7 @@ class SpamCheckerTestCase(HomeserverTestCase): ) channel = self.create_room({}) + self.assertEqual(channel.code, 200) room_id = channel.json_body["room_id"] diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 36d3213908..1a2dab4c7d 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -173,7 +173,13 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - channel = self.site.buildProtocol((host, port)) + # The type ignore is here because mypy doesn't think the host/port tuple is of + # the correct type, even though it is the exact example given for + # `twisted.internet.interfaces.IAddress`. + # Mypy was happy with the type before we overrode `buildProtocol` in + # `SynapseSite`, probably because there was enough inheritance indirection before + # withe the argument not having a type associated with it. + channel = self.site.buildProtocol((host, port)) # type: ignore[arg-type] # hook into the channel's request factory so that we can keep a record # of the requests @@ -185,7 +191,7 @@ class BaseStreamTestCase(unittest.HomeserverTestCase): requests.append(request) return request - channel.requestFactory = request_factory + channel.requestFactory = request_factory # type: ignore[method-assign] # Connect client to server and vice versa. client_to_server_transport = FakeTransport( @@ -427,7 +433,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - channel = self._hs_to_site[hs].buildProtocol((host, port)) + channel = self._hs_to_site[hs].buildProtocol((host, port)) # type: ignore[arg-type] # Connect client to server and vice versa. client_to_server_transport = FakeTransport( diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 92259f2542..3896e0ce8a 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -66,10 +66,11 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): def setUp(self) -> None: super().setUp() - reactor, _ = get_clock() + reactor, clock = get_clock() self.matrix_federation_agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=reactor, + clock=clock, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/replication/test_module_cache_invalidation.py b/tests/replication/test_module_cache_invalidation.py index 8d5d0cce9a..1cb898673b 100644 --- a/tests/replication/test_module_cache_invalidation.py +++ b/tests/replication/test_module_cache_invalidation.py @@ -24,6 +24,7 @@ import synapse from synapse.module_api import cached from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.server import get_clock logger = logging.getLogger(__name__) @@ -36,6 +37,7 @@ KEY = "mykey" class TestCache: current_value = FIRST_VALUE server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() async def cached_function(self, user_id: str) -> str: diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 967f783680..bb83988d76 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -29,16 +29,19 @@ from synapse.logging.context import SENTINEL_CONTEXT, LoggingContext, current_co from synapse.rest.client.transactions import CLEANUP_PERIOD_MS, HttpTransactionCache from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.constants import ( + MILLISECONDS_PER_SECOND, +) from tests import unittest -from tests.utils import MockClock +from tests.server import get_clock reactor = cast(ISynapseReactor, _reactor) class HttpTransactionCacheTestCase(unittest.TestCase): def setUp(self) -> None: - self.clock = MockClock() + self.reactor, self.clock = get_clock() self.hs = Mock() self.hs.get_clock = Mock(return_value=self.clock) self.hs.get_auth = Mock() @@ -90,12 +93,16 @@ class HttpTransactionCacheTestCase(unittest.TestCase): ) -> Generator["defer.Deferred[Any]", object, None]: @defer.inlineCallbacks def cb() -> Generator["defer.Deferred[object]", object, Tuple[int, JsonDict]]: - yield defer.ensureDeferred(Clock(reactor).sleep(0)) + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes. + yield defer.ensureDeferred( + Clock(reactor, server_name="test_server").sleep(0) # type: ignore[multiple-internal-clocks] + ) return 1, {} @defer.inlineCallbacks def test() -> Generator["defer.Deferred[Any]", object, None]: - with LoggingContext("c") as c1: + with LoggingContext(name="c", server_name="test_server") as c1: res = yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb ) @@ -125,7 +132,7 @@ class HttpTransactionCacheTestCase(unittest.TestCase): called[0] = True raise Exception("boo") - with LoggingContext("test") as test_context: + with LoggingContext(name="test", server_name="test_server") as test_context: try: yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb @@ -157,7 +164,7 @@ class HttpTransactionCacheTestCase(unittest.TestCase): called[0] = True return defer.fail(Exception("boo")) - with LoggingContext("test") as test_context: + with LoggingContext(name="test", server_name="test_server") as test_context: try: yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb @@ -178,8 +185,9 @@ class HttpTransactionCacheTestCase(unittest.TestCase): yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb, "an arg" ) - # should NOT have cleaned up yet - self.clock.advance_time_msec(CLEANUP_PERIOD_MS / 2) + # Advance time just under the cleanup period. + # Should NOT have cleaned up yet + self.reactor.advance((CLEANUP_PERIOD_MS - 1) / MILLISECONDS_PER_SECOND) yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb, "an arg" @@ -187,7 +195,8 @@ class HttpTransactionCacheTestCase(unittest.TestCase): # still using cache cb.assert_called_once_with("an arg") - self.clock.advance_time_msec(CLEANUP_PERIOD_MS) + # Advance time just after the cleanup period. + self.reactor.advance(2 / MILLISECONDS_PER_SECOND) yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb, "an arg" diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index cf8241438c..8d2489f718 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -170,7 +170,7 @@ class EndToEndPerspectivesTests(BaseRemoteKeyResourceTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # make a second homeserver, configured to use the first one as a key notary self.http_client2 = Mock() - config = default_config(name="keyclient") + config = default_config(server_name="keyclient") config["trusted_key_servers"] = [ { "server_name": self.hs.hostname, diff --git a/tests/server.py b/tests/server.py index f66ce070c6..a9a53eb8a4 100644 --- a/tests/server.py +++ b/tests/server.py @@ -28,6 +28,7 @@ import sqlite3 import time import uuid import warnings +import weakref from collections import deque from io import SEEK_END, BytesIO from typing import ( @@ -56,7 +57,7 @@ from zope.interface import implementer import twisted from twisted.enterprise import adbapi -from twisted.internet import address, tcp, threads, udp +from twisted.internet import address, defer, tcp, threads, udp from twisted.internet._resolver import SimpleResolverComplexifier from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred, fail, maybeDeferred, succeed @@ -114,7 +115,6 @@ from tests.utils import ( POSTGRES_USER, SQLITE_PERSIST_DB, USE_POSTGRES_FOR_TESTS, - MockClock, default_config, ) @@ -525,6 +525,19 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): # overwrite it again. self.nameResolver = SimpleResolverComplexifier(FakeResolver()) + def run(self) -> None: + """ + Override the call from `MemoryReactorClock` to add an additional step that + cleans up any `whenRunningHooks` that have been called. + This is necessary for a clean shutdown to occur as these hooks can hold + references to the `SynapseHomeServer`. + """ + super().run() + + # `MemoryReactorClock` never clears the hooks that have already been called. + # So manually clear the hooks here after they have been run. + self.whenRunningHooks.clear() + def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver: raise NotImplementedError() @@ -650,6 +663,19 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): super().advance(0) +def cleanup_test_reactor_system_event_triggers( + reactor: ThreadedMemoryReactorClock, +) -> None: + """Cleanup any registered system event triggers. + The `twisted.internet.test.ThreadedMemoryReactor` does not implement + `removeSystemEventTrigger` so won't clean these triggers up on it's own properly. + When trying to override `removeSystemEventTrigger` in `ThreadedMemoryReactorClock` + in order to implement this functionality, twisted complains about the reactor being + unclean and fails some tests. + """ + reactor.triggers.clear() + + def validate_connector(connector: tcp.Connector, expected_ip: str) -> None: """Try to validate the obtained connector as it would happen when synapse is running and the conection will be established. @@ -781,14 +807,19 @@ class ThreadPool: d: "Deferred[None]" = Deferred() d.addCallback(lambda x: function(*args, **kwargs)) d.addBoth(_) - self._reactor.callLater(0, d.callback, True) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + self._reactor.callLater(0, d.callback, True) # type: ignore[call-later-not-tracked] return d def get_clock() -> Tuple[ThreadedMemoryReactorClock, Clock]: - clock = ThreadedMemoryReactorClock() - hs_clock = Clock(clock) - return clock, hs_clock + # Ignore the linter error since this is an expected usage of creating a `Clock` for + # testing purposes. + reactor = ThreadedMemoryReactorClock() + hs_clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] + return reactor, hs_clock @implementer(ITCPTransport) @@ -899,10 +930,16 @@ class FakeTransport: # some implementations of IProducer (for example, FileSender) # don't return a deferred. d = maybeDeferred(self.producer.resumeProducing) - d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: - self._reactor.callLater(0.0, _produce) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + self._reactor.callLater(0.0, _produce) # type: ignore[call-later-not-tracked] def write(self, byt: bytes) -> None: if self.disconnecting: @@ -914,7 +951,10 @@ class FakeTransport: # TLSMemoryBIOProtocol) get very confused if a read comes back while they are # still doing a write. Doing a callLater here breaks the cycle. if self.autoflush: - self._reactor.callLater(0.0, self.flush) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] def writeSequence(self, seq: Iterable[bytes]) -> None: for x in seq: @@ -944,7 +984,10 @@ class FakeTransport: self.buffer = self.buffer[len(to_write) :] if self.buffer and self.autoflush: - self._reactor.callLater(0.0, self.flush) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] if not self.buffer and self.disconnecting: logger.info("FakeTransport: Buffer now empty, completing disconnect") @@ -1020,12 +1063,14 @@ class TestHomeServer(HomeServer): def setup_test_homeserver( - cleanup_func: Callable[[Callable[[], None]], None], - name: str = "test", + *, + cleanup_func: Callable[[Callable[[], Optional["Deferred[None]"]]], None], + server_name: str = "test", config: Optional[HomeServerConfig] = None, reactor: Optional[ISynapseReactor] = None, homeserver_to_use: Type[HomeServer] = TestHomeServer, - **kwargs: Any, + db_txn_limit: Optional[int] = None, + **extra_homeserver_attributes: Any, ) -> HomeServer: """ Setup a homeserver suitable for running tests against. Keyword arguments @@ -1035,28 +1080,44 @@ def setup_test_homeserver( Args: cleanup_func : The function used to register a cleanup routine for - after the test. + after the test. If the function returns a Deferred, the + test case will wait until the Deferred has fired before + proceeding to the next cleanup function. + server_name: Homeserver name + config: Homeserver config + reactor: Twisted reactor + homeserver_to_use: Homeserver class to instantiate. + db_txn_limit: Gives the maximum number of database transactions to run per + connection before reconnecting. 0 means no limit. If unset, defaults to None + here which will default upstream to `0`. + **extra_homeserver_attributes: Additional keyword arguments to install as + `@cache_in_self` attributes on the homeserver. For example, `clock` will be + installed as `hs._clock`. Calling this method directly is deprecated: you should instead derive from HomeserverTestCase. """ if reactor is None: - from twisted.internet import reactor as _reactor - - reactor = cast(ISynapseReactor, _reactor) + reactor = ThreadedMemoryReactorClock() if config is None: - config = default_config(name, parse=True) + config = default_config(server_name, parse=True) + + server_name = config.server.server_name + if not isinstance(server_name, str): + raise ConfigError("Must be a string", ("server_name",)) + + if "clock" not in extra_homeserver_attributes: + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes (i.e. outside of Synapse). + extra_homeserver_attributes["clock"] = Clock(reactor, server_name=server_name) # type: ignore[multiple-internal-clocks] config.caches.resize_all_caches() - if "clock" not in kwargs: - kwargs["clock"] = MockClock() - if USE_POSTGRES_FOR_TESTS: test_db = "synapse_test_%s" % uuid.uuid4().hex - database_config = { + database_config: JsonDict = { "name": "psycopg2", "args": { "dbname": test_db, @@ -1088,10 +1149,6 @@ def setup_test_homeserver( "args": {"database": test_db_location, "cp_min": 1, "cp_max": 1}, } - server_name = config.server.server_name - if not isinstance(server_name, str): - raise ConfigError("Must be a string", ("server_name",)) - # Check if we have set up a DB that we can use as a template. global PREPPED_SQLITE_DB_CONN if PREPPED_SQLITE_DB_CONN is None: @@ -1111,8 +1168,8 @@ def setup_test_homeserver( database_config["_TEST_PREPPED_CONN"] = PREPPED_SQLITE_DB_CONN - if "db_txn_limit" in kwargs: - database_config["txn_limit"] = kwargs["db_txn_limit"] + if db_txn_limit is not None: + database_config["txn_limit"] = db_txn_limit database = DatabaseConnectionConfig("master", database_config) config.database.databases = [database] @@ -1139,17 +1196,30 @@ def setup_test_homeserver( db_conn.close() hs = homeserver_to_use( - name, + server_name, config=config, version_string="Synapse/tests", reactor=reactor, ) - # Register the cleanup hook - cleanup_func(hs.cleanup) + # Capture the `hs` as a `weakref` here to ensure there is no scenario where uncalled + # cleanup functions result in holding the `hs` in memory. + cleanup_hs_ref = weakref.ref(hs) + + def shutdown_hs_on_cleanup() -> "Deferred[None]": + cleanup_hs = cleanup_hs_ref() + deferred: "Deferred[None]" = defer.succeed(None) + if cleanup_hs is not None: + deferred = defer.ensureDeferred(cleanup_hs.shutdown()) + return deferred + + # Register the cleanup hook for the homeserver. + # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting + # strange behaviours. + cleanup_func(shutdown_hs_on_cleanup) # Install @cache_in_self attributes - for key, val in kwargs.items(): + for key, val in extra_homeserver_attributes.items(): setattr(hs, "_" + key, val) # Mock TLS @@ -1175,14 +1245,18 @@ def setup_test_homeserver( hs.get_datastores().main.USE_DEDICATED_DB_THREADS_FOR_EVENT_FETCHING = False if USE_POSTGRES_FOR_TESTS: - database_pool = hs.get_datastores().databases[0] + # Capture the `database_pool` as a `weakref` here to ensure there is no scenario where uncalled + # cleanup functions result in holding the `hs` in memory. + database_pool = weakref.ref(hs.get_datastores().databases[0]) # We need to do cleanup on PostgreSQL def cleanup() -> None: import psycopg2 # Close all the db pools - database_pool._db_pool.close() + db_pool = database_pool() + if db_pool is not None: + db_pool._db_pool.close() dropped = False diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 81a3447dbb..a7c6bdd9b4 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -76,7 +76,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): self.event_ids.append(event.event_id) def test_simple(self) -> None: - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: res = self.get_success( self.store.have_seen_events( self.room_id, [self.event_ids[0], "eventdoesnotexist"] @@ -88,7 +88,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): self.assertEqual(ctx.get_resource_usage().db_txn_count, 1) # a second lookup of the same events should cause no queries - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: res = self.get_success( self.store.have_seen_events( self.room_id, [self.event_ids[0], "eventdoesnotexist"] @@ -113,7 +113,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): ) ) - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: # First, check `have_seen_event` for an event we have not seen yet # to prime the cache with a `false` value. res = self.get_success( @@ -135,7 +135,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): ) ) - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: # Check `have_seen_event` again and we should see the updated fact # that we have now seen the event after persisting it. res = self.get_success( @@ -166,7 +166,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): res = self.store._get_event_cache.get_local((event.event_id,)) self.assertEqual(res, None, "Event was cached when it should not have been.") - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: # Persist the event which should invalidate then prefill the # `_get_event_cache` so we don't return stale values. # Side Note: Apparently, persisting an event isn't a transaction in the @@ -200,7 +200,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): Test to make sure that all events associated with the given `(room_id,)` are invalidated in the `have_seen_event` cache. """ - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: # Prime the cache with some values res = self.get_success( self.store.have_seen_events(self.room_id, self.event_ids) @@ -213,7 +213,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): # Clear the cache with any events associated with the `room_id` self.store.have_seen_event.invalidate((self.room_id,)) - with LoggingContext(name="test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: res = self.get_success( self.store.have_seen_events(self.room_id, self.event_ids) ) @@ -249,7 +249,7 @@ class EventCacheTestCase(unittest.HomeserverTestCase): def test_simple(self) -> None: """Test that we cache events that we pull from the DB.""" - with LoggingContext("test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: self.get_success(self.store.get_event(self.event_id)) # We should have fetched the event from the DB @@ -263,7 +263,7 @@ class EventCacheTestCase(unittest.HomeserverTestCase): # Reset the event cache self.store._get_event_cache.clear() - with LoggingContext("test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: # We keep hold of the event event though we never use it. event = self.get_success(self.store.get_event(self.event_id)) # noqa: F841 @@ -273,7 +273,7 @@ class EventCacheTestCase(unittest.HomeserverTestCase): # Reset the event cache self.store._get_event_cache.clear() - with LoggingContext("test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: self.get_success(self.store.get_event(self.event_id)) # Since the event is still in memory we shouldn't have fetched it @@ -285,7 +285,7 @@ class EventCacheTestCase(unittest.HomeserverTestCase): out once. """ - with LoggingContext("test") as ctx: + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: d = yieldable_gather_results( self.store.get_event, [self.event_id, self.event_id] ) @@ -531,8 +531,8 @@ class GetEventCancellationTestCase(unittest.HomeserverTestCase): "runWithConnection", new=runWithConnection, ): - ctx1 = LoggingContext("get_event1") - ctx2 = LoggingContext("get_event2") + ctx1 = LoggingContext(name="get_event1", server_name=self.hs.hostname) + ctx2 = LoggingContext(name="get_event2", server_name=self.hs.hostname) async def get_event(ctx: LoggingContext) -> None: with ctx: diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py index 4296fcde13..3743a4a386 100644 --- a/tests/storage/databases/main/test_lock.py +++ b/tests/storage/databases/main/test_lock.py @@ -72,15 +72,10 @@ class LockTestCase(unittest.HomeserverTestCase): release_lock.callback(None) # Run the tasks to completion. - # To work around `Linearizer`s using a different reactor to sleep when - # contended (https://github.com/matrix-org/synapse/issues/12841), we call - # `runUntilCurrent` on `twisted.internet.reactor`, which is a different - # reactor to that used by the homeserver. - assert isinstance(reactor, ReactorBase) self.get_success(task1) - reactor.runUntilCurrent() + self.pump() self.get_success(task2) - reactor.runUntilCurrent() + self.pump() self.get_success(task3) # At most one task should have held the lock at a time. @@ -223,15 +218,11 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase): release_lock.callback(None) # Run the tasks to completion. - # To work around `Linearizer`s using a different reactor to sleep when - # contended (https://github.com/matrix-org/synapse/issues/12841), we call - # `runUntilCurrent` on `twisted.internet.reactor`, which is a different - # reactor to that used by the homeserver. assert isinstance(reactor, ReactorBase) self.get_success(task1) - reactor.runUntilCurrent() + self.pump() self.get_success(task2) - reactor.runUntilCurrent() + self.pump() self.get_success(task3) # At most one task should have held the lock at a time. @@ -275,15 +266,11 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase): release_lock.callback(None) # Run the tasks to completion. - # To work around `Linearizer`s using a different reactor to sleep when - # contended (https://github.com/matrix-org/synapse/issues/12841), we call - # `runUntilCurrent` on `twisted.internet.reactor`, which is a different - # reactor to that used by the homeserver. assert isinstance(reactor, ReactorBase) self.get_success(task1) - reactor.runUntilCurrent() + self.pump() self.get_success(task2) - reactor.runUntilCurrent() + self.pump() self.get_success(task3) # At most one task should have held the lock at a time. diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 11313fc933..577229c119 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -86,7 +86,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): conn_pool.runWithConnection = runWithConnection - config = default_config(name="test", parse=True) + config = default_config(server_name="test", parse=True) hs = TestHomeServer("test", config=config) if USE_POSTGRES_FOR_TESTS: diff --git a/tests/synapse_rust/__init__.py b/tests/synapse_rust/__init__.py new file mode 100644 index 0000000000..e056679fd5 --- /dev/null +++ b/tests/synapse_rust/__init__.py @@ -0,0 +1,11 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py new file mode 100644 index 0000000000..032eab77e8 --- /dev/null +++ b/tests/synapse_rust/test_http_client.py @@ -0,0 +1,225 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import json +import logging +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any, Coroutine, Generator, TypeVar, Union + +from twisted.internet.defer import Deferred, ensureDeferred +from twisted.internet.testing import MemoryReactor + +from synapse.logging.context import ( + LoggingContext, + PreserveLoggingContext, + _Sentinel, + current_context, + run_in_background, +) +from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient +from synapse.util.clock import Clock +from synapse.util.json import json_decoder + +from tests.unittest import HomeserverTestCase + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +class StubRequestHandler(BaseHTTPRequestHandler): + server: "StubServer" + + def do_GET(self) -> None: + self.server.calls += 1 + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"ok": True}).encode("utf-8")) + + def log_message(self, format: str, *args: Any) -> None: + # Don't log anything; by default, the server logs to stderr + pass + + +class StubServer(HTTPServer): + """A stub HTTP server that we can send requests to for testing. + + This opens a real HTTP server on a random port, on a separate thread. + """ + + calls: int = 0 + """How many times has the endpoint been requested.""" + + _thread: threading.Thread + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), StubRequestHandler) + + self._thread = threading.Thread( + target=self.serve_forever, + name="StubServer", + kwargs={"poll_interval": 0.01}, + daemon=True, + ) + self._thread.start() + + def shutdown(self) -> None: + super().shutdown() + self._thread.join() + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self.server_port}/" + + +class HttpClientTestCase(HomeserverTestCase): + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + self._rust_http_client = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.server = StubServer() + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def till_deferred_has_result( + self, + awaitable: Union[ + "Coroutine[Deferred[Any], Any, T]", + "Generator[Deferred[Any], Any, T]", + "Deferred[T]", + ], + ) -> "Deferred[T]": + """Wait until a deferred has a result. + + This is useful because the Rust HTTP client will resolve the deferred + using reactor.callFromThread, which are only run when we call + reactor.advance. + """ + deferred = ensureDeferred(awaitable) + tries = 0 + while not deferred.called: + time.sleep(0.1) + self.reactor.advance(0) + tries += 1 + if tries > 100: + raise Exception("Timed out waiting for deferred to resolve") + + return deferred + + def _check_current_logcontext(self, expected_logcontext_string: str) -> None: + context = current_context() + assert isinstance(context, LoggingContext) or isinstance(context, _Sentinel), ( + f"Expected LoggingContext({expected_logcontext_string}) but saw {context}" + ) + self.assertEqual( + str(context), + expected_logcontext_string, + f"Expected LoggingContext({expected_logcontext_string}) but saw {context}", + ) + + def test_request_response(self) -> None: + """ + Test to make sure we can make a basic request and get the expected + response. + """ + + async def do_request() -> None: + resp_body = await self._rust_http_client.get( + url=self.server.endpoint, + response_limit=1 * 1024 * 1024, + ) + raw_response = json_decoder.decode(resp_body.decode("utf-8")) + self.assertEqual(raw_response, {"ok": True}) + + self.get_success(self.till_deferred_has_result(do_request())) + self.assertEqual(self.server.calls, 1) + + async def test_logging_context(self) -> None: + """ + Test to make sure the `LoggingContext` (logcontext) is handled correctly + when making requests. + """ + # Sanity check that we start in the sentinel context + self._check_current_logcontext("sentinel") + + callback_finished = False + + async def do_request() -> None: + nonlocal callback_finished + try: + # Should have the same logcontext as the caller + self._check_current_logcontext("foo") + + with LoggingContext(name="competing", server_name="test_server"): + # Make the actual request + await self._rust_http_client.get( + url=self.server.endpoint, + response_limit=1 * 1024 * 1024, + ) + self._check_current_logcontext("competing") + + # Back to the caller's context outside of the `LoggingContext` block + self._check_current_logcontext("foo") + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + with LoggingContext(name="foo", server_name="test_server"): + # Fire off the function, but don't wait on it. + run_in_background(do_request) + + # Now wait for the function under test to have run + with PreserveLoggingContext(): + while not callback_finished: + # await self.hs.get_clock().sleep(0) + time.sleep(0.1) + self.reactor.advance(0) + + # check that the logcontext is left in a sane state. + self._check_current_logcontext("foo") + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + # Back to the sentinel context + self._check_current_logcontext("sentinel") diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 19dafe64ed..2dd26833c8 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -26,9 +26,10 @@ from synapse.util.distributor import Distributor from . import unittest -class DistributorTestCase(unittest.TestCase): +class DistributorTestCase(unittest.HomeserverTestCase): def setUp(self) -> None: - self.dist = Distributor(server_name="test_server") + super().setUp() + self.dist = Distributor(hs=self.hs) def test_signal_dispatch(self) -> None: self.dist.declare("alert") diff --git a/tests/test_server.py b/tests/test_server.py index 69efceafe8..66c5cf9e37 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -55,9 +55,9 @@ class JsonResourceTests(unittest.TestCase): reactor, clock = get_clock() self.reactor = reactor self.homeserver = setup_test_homeserver( - self.addCleanup, - clock=clock, + cleanup_func=self.addCleanup, reactor=self.reactor, + clock=clock, ) def test_handler_for_request(self) -> None: @@ -217,9 +217,9 @@ class OptionsResourceTests(unittest.TestCase): reactor, clock = get_clock() self.reactor = reactor self.homeserver = setup_test_homeserver( - self.addCleanup, - clock=clock, + cleanup_func=self.addCleanup, reactor=self.reactor, + clock=clock, ) class DummyResource(Resource): diff --git a/tests/test_state.py b/tests/test_state.py index 16446c16bc..ab7b52e90c 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -29,7 +29,6 @@ from typing import ( Optional, Set, Tuple, - cast, ) from unittest.mock import AsyncMock, Mock @@ -43,12 +42,11 @@ from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry from synapse.types import MutableStateMap, StateMap from synapse.types.state import StateFilter -from synapse.util.clock import Clock from synapse.util.macaroons import MacaroonGenerator from tests import unittest - -from .utils import MockClock, default_config +from tests.server import get_clock +from tests.utils import default_config _next_event_id = 1000 @@ -248,7 +246,7 @@ class StateTestCase(unittest.TestCase): "hostname", ] ) - clock = cast(Clock, MockClock()) + reactor, clock = get_clock() hs.config = default_config("tesths", True) hs.get_datastores.return_value = Mock( main=self.dummy_store, diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py deleted file mode 100644 index c52f963a7e..0000000000 --- a/tests/test_test_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2014-2016 OpenMarket Ltd -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from tests import unittest -from tests.utils import MockClock - - -class MockClockTestCase(unittest.TestCase): - def setUp(self) -> None: - self.clock = MockClock() - - def test_advance_time(self) -> None: - start_time = self.clock.time() - - self.clock.advance_time(20) - - self.assertEqual(20, self.clock.time() - start_time) - - def test_later(self) -> None: - invoked = [0, 0] - - def _cb0() -> None: - invoked[0] = 1 - - self.clock.call_later(10, _cb0) - - def _cb1() -> None: - invoked[1] = 1 - - self.clock.call_later(20, _cb1) - - self.assertFalse(invoked[0]) - - self.clock.advance_time(15) - - self.assertTrue(invoked[0]) - self.assertFalse(invoked[1]) - - self.clock.advance_time(5) - - self.assertTrue(invoked[1]) - - def test_cancel_later(self) -> None: - invoked = [0, 0] - - def _cb0() -> None: - invoked[0] = 1 - - t0 = self.clock.call_later(10, _cb0) - - def _cb1() -> None: - invoked[1] = 1 - - self.clock.call_later(20, _cb1) - - self.clock.cancel_call_later(t0) - - self.clock.advance_time(30) - - self.assertFalse(invoked[0]) - self.assertTrue(invoked[1]) diff --git a/tests/unittest.py b/tests/unittest.py index 7d46f84496..9ab052e7c0 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -80,7 +80,7 @@ from synapse.logging.context import ( from synapse.rest import RegisterServletsFunc from synapse.server import HomeServer from synapse.storage.keys import FetchKeyResult -from synapse.types import JsonDict, Requester, UserID, create_requester +from synapse.types import ISynapseReactor, JsonDict, Requester, UserID, create_requester from synapse.util.clock import Clock from synapse.util.httpresourcetree import create_resource_tree @@ -99,6 +99,8 @@ from tests.utils import checked_cast, default_config, setupdb setupdb() setup_logging() +logger = logging.getLogger(__name__) + TV = TypeVar("TV") _ExcType = TypeVar("_ExcType", bound=BaseException, covariant=True) @@ -135,7 +137,7 @@ def around(target: TV) -> Callable[[Callable[Concatenate[S, P], R]], None]: return _around -_TConfig = TypeVar("_TConfig", Config, RootConfig) +_TConfig = TypeVar("_TConfig", Config, HomeServerConfig) def deepcopy_config(config: _TConfig) -> _TConfig: @@ -161,13 +163,13 @@ def deepcopy_config(config: _TConfig) -> _TConfig: @functools.lru_cache(maxsize=8) -def _parse_config_dict(config: str) -> RootConfig: +def _parse_config_dict(config: str) -> HomeServerConfig: config_obj = HomeServerConfig() config_obj.parse_config_dict(json.loads(config), "", "") return config_obj -def make_homeserver_config_obj(config: Dict[str, Any]) -> RootConfig: +def make_homeserver_config_obj(config: Dict[str, Any]) -> HomeServerConfig: """Creates a :class:`HomeServerConfig` instance with the given configuration dict. This is equivalent to:: @@ -392,8 +394,8 @@ class HomeserverTestCase(TestCase): hijacking the authentication system to return a fixed user, and then calling the prepare function. """ + # We need to share the reactor between the homeserver and all of our test utils. self.reactor, self.clock = get_clock() - self._hs_args = {"clock": self.clock, "reactor": self.reactor} self.hs = self.make_homeserver(self.reactor, self.clock) self.hs.get_datastores().main.tests_allow_no_chain_cover_index = False @@ -511,7 +513,7 @@ class HomeserverTestCase(TestCase): Function to be overridden in subclasses. """ - hs = self.setup_test_homeserver() + hs = self.setup_test_homeserver(reactor=reactor, clock=clock) return hs def create_test_resource(self) -> Resource: @@ -634,7 +636,12 @@ class HomeserverTestCase(TestCase): ) def setup_test_homeserver( - self, name: Optional[str] = None, **kwargs: Any + self, + server_name: Optional[str] = None, + config: Optional[JsonDict] = None, + reactor: Optional[ISynapseReactor] = None, + clock: Optional[Clock] = None, + **extra_homeserver_attributes: Any, ) -> HomeServer: """ Set up the test homeserver, meant to be called by the overridable @@ -647,32 +654,41 @@ class HomeserverTestCase(TestCase): Returns: synapse.server.HomeServer """ - kwargs = dict(kwargs) - kwargs.update(self._hs_args) - if "config" not in kwargs: + if config is None: config = self.default_config() - else: - config = kwargs["config"] + + # The sane default is to use the same reactor and clock as our other test utils + if reactor is None: + reactor = self.reactor + + if clock is None: + clock = self.clock # The server name can be specified using either the `name` argument or a config # override. The `name` argument takes precedence over any config overrides. - if name is not None: - config["server_name"] = name + if server_name is not None: + config["server_name"] = server_name # Parse the config from a config dict into a HomeServerConfig config_obj = make_homeserver_config_obj(config) - kwargs["config"] = config_obj # The server name in the config is now `name`, if provided, or the `server_name` # from a config override, or the default of "test". Whichever it is, we # construct a homeserver with a matching name. - kwargs["name"] = config_obj.server.server_name + server_name = config_obj.server.server_name async def run_bg_updates() -> None: - with LoggingContext("run_bg_updates"): + with LoggingContext(name="run_bg_updates", server_name=server_name): self.get_success(stor.db_pool.updates.run_background_updates(False)) - hs = setup_test_homeserver(self.addCleanup, **kwargs) + hs = setup_test_homeserver( + cleanup_func=self.addCleanup, + server_name=server_name, + config=config_obj, + reactor=reactor, + clock=clock, + **extra_homeserver_attributes, + ) stor = hs.get_datastores().main # Run the database background updates, when running against "master". diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py index 7017d6d70a..f0deb1554e 100644 --- a/tests/util/caches/test_deferred_cache.py +++ b/tests/util/caches/test_deferred_cache.py @@ -26,20 +26,26 @@ from twisted.internet import defer from synapse.util.caches.deferred_cache import DeferredCache +from tests.server import get_clock from tests.unittest import TestCase class DeferredCacheTestCase(TestCase): + def setUp(self) -> None: + super().setUp() + + _, self.clock = get_clock() + def test_empty(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) with self.assertRaises(KeyError): cache.get("foo") def test_hit(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) cache.prefill("foo", 123) @@ -47,7 +53,7 @@ class DeferredCacheTestCase(TestCase): def test_hit_deferred(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) origin_d: "defer.Deferred[int]" = defer.Deferred() set_d = cache.set("k1", origin_d) @@ -72,7 +78,7 @@ class DeferredCacheTestCase(TestCase): def test_callbacks(self) -> None: """Invalidation callbacks are called at the right time""" cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) callbacks = set() @@ -107,7 +113,7 @@ class DeferredCacheTestCase(TestCase): def test_set_fail(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) callbacks = set() @@ -146,7 +152,7 @@ class DeferredCacheTestCase(TestCase): def test_get_immediate(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) d1: "defer.Deferred[int]" = defer.Deferred() cache.set("key1", d1) @@ -164,7 +170,7 @@ class DeferredCacheTestCase(TestCase): def test_invalidate(self) -> None: cache: DeferredCache[Tuple[str], int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) cache.prefill(("foo",), 123) cache.invalidate(("foo",)) @@ -174,7 +180,7 @@ class DeferredCacheTestCase(TestCase): def test_invalidate_all(self) -> None: cache: DeferredCache[str, str] = DeferredCache( - name="testcache", server_name="test_server" + name="testcache", clock=self.clock, server_name="test_server" ) callback_record = [False, False] @@ -220,6 +226,7 @@ class DeferredCacheTestCase(TestCase): def test_eviction(self) -> None: cache: DeferredCache[int, str] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=2, apply_cache_factor_from_config=False, @@ -238,6 +245,7 @@ class DeferredCacheTestCase(TestCase): def test_eviction_lru(self) -> None: cache: DeferredCache[int, str] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=2, apply_cache_factor_from_config=False, @@ -260,6 +268,7 @@ class DeferredCacheTestCase(TestCase): def test_eviction_iterable(self) -> None: cache: DeferredCache[int, List[str]] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=3, apply_cache_factor_from_config=False, diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 7865a67709..0e3b6ae36b 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -49,6 +49,7 @@ from synapse.util.caches import descriptors from synapse.util.caches.descriptors import _CacheContext, cached, cachedList from tests import unittest +from tests.server import get_clock from tests.test_utils import get_awaitable_result logger = logging.getLogger(__name__) @@ -56,7 +57,10 @@ logger = logging.getLogger(__name__) def run_on_reactor() -> "Deferred[int]": d: "Deferred[int]" = Deferred() - cast(IReactorTime, reactor).callLater(0, d.callback, 0) + # mypy ignored here because: + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. + cast(IReactorTime, reactor).callLater(0, d.callback, 0) # type: ignore[call-later-not-tracked] return make_deferred_yieldable(d) @@ -67,6 +71,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> str: @@ -102,6 +107,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached(num_args=1) def fn(self, arg1: int, arg2: int) -> str: @@ -148,6 +154,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached obj = Cls() obj.mock.return_value = "fish" @@ -179,6 +186,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, kwarg1: int = 2) -> str: @@ -214,6 +222,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> NoReturn: @@ -239,6 +248,7 @@ class DescriptorTestCase(unittest.TestCase): result: Optional[Deferred] = None call_count = 0 server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> Deferred: @@ -293,6 +303,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> "Deferred[int]": @@ -306,7 +317,7 @@ class DescriptorTestCase(unittest.TestCase): @defer.inlineCallbacks def do_lookup() -> Generator["Deferred[Any]", object, int]: - with LoggingContext("c1") as c1: + with LoggingContext(name="c1", server_name="test_server") as c1: r = yield obj.fn(1) self.assertEqual(current_context(), c1) return cast(int, r) @@ -337,6 +348,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> Deferred: @@ -350,7 +362,7 @@ class DescriptorTestCase(unittest.TestCase): @defer.inlineCallbacks def do_lookup() -> Generator["Deferred[object]", object, None]: - with LoggingContext("c1") as c1: + with LoggingContext(name="c1", server_name="test_server") as c1: try: d = obj.fn(1) self.assertEqual( @@ -381,6 +393,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int = 2, arg3: int = 3) -> str: @@ -419,6 +432,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached(iterable=True) def fn(self, arg1: int, arg2: int) -> Tuple[str, ...]: @@ -453,6 +467,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached(iterable=True) def fn(self, arg1: int) -> NoReturn: @@ -476,6 +491,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(cache_context=True) async def func1(self, key: str, cache_context: _CacheContext) -> int: @@ -504,6 +520,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -537,6 +554,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -547,7 +565,7 @@ class DescriptorTestCase(unittest.TestCase): obj = Cls() async def do_lookup() -> None: - with LoggingContext("c1") as c1: + with LoggingContext(name="c1", server_name="test_server") as c1: try: await obj.fn(123) self.fail("No CancelledError thrown") @@ -583,6 +601,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): def test_passthrough(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -599,6 +618,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -619,6 +639,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -639,6 +660,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): def test_invalidate_missing(self) -> None: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -652,6 +674,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(max_entries=10) def func(self, key: int) -> int: @@ -681,6 +704,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> "Deferred[int]": @@ -701,6 +725,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -736,6 +761,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(max_entries=2) def func(self, key: str) -> str: @@ -775,6 +801,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -824,6 +851,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -843,7 +871,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): assert context.name == "c1" return self.mock(args1, arg2) - with LoggingContext("c1") as c1: + with LoggingContext(name="c1", server_name="test_server") as c1: obj = Cls() obj.mock.return_value = {10: "fish", 20: "chips"} @@ -890,6 +918,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> None: @@ -934,6 +963,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -975,6 +1005,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1011,6 +1042,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1025,7 +1057,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): obj = Cls() async def do_lookup() -> None: - with LoggingContext("c1") as c1: + with LoggingContext(name="c1", server_name="test_server") as c1: try: await obj.list_fn([123]) self.fail("No CancelledError thrown") @@ -1055,6 +1087,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached(tree=True) def fn(self, room_id: str, event_id: str) -> None: diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index cfd2882410..fd8d576aea 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -17,14 +17,14 @@ # [This file includes modifications made by New Vector Limited] # # +import logging import traceback -from typing import Any, Coroutine, Generator, List, NoReturn, Optional, Tuple, TypeVar +from typing import Any, Coroutine, List, NoReturn, Optional, Tuple, TypeVar from parameterized import parameterized_class from twisted.internet import defer from twisted.internet.defer import CancelledError, Deferred, ensureDeferred -from twisted.internet.task import Clock from twisted.python.failure import Failure from synapse.logging.context import ( @@ -47,6 +47,8 @@ from synapse.util.async_helpers import ( from tests.server import get_clock from tests.unittest import TestCase +logger = logging.getLogger(__name__) + T = TypeVar("T") @@ -149,7 +151,7 @@ class ObservableDeferredTest(TestCase): class TimeoutDeferredTest(TestCase): def setUp(self) -> None: - self.clock = Clock() + self.reactor, self.clock = get_clock() def test_times_out(self) -> None: """Basic test case that checks that the original deferred is cancelled and that @@ -162,12 +164,16 @@ class TimeoutDeferredTest(TestCase): cancelled = True non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + timing_out_d = timeout_deferred( + deferred=non_completing_d, + timeout=1.0, + clock=self.clock, + ) self.assertNoResult(timing_out_d) self.assertFalse(cancelled, "deferred was cancelled prematurely") - self.clock.pump((1.0,)) + self.reactor.pump((1.0,)) self.assertTrue(cancelled, "deferred was not cancelled by timeout") self.failureResultOf(timing_out_d, defer.TimeoutError) @@ -180,57 +186,92 @@ class TimeoutDeferredTest(TestCase): raise Exception("can't cancel this deferred") non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + timing_out_d = timeout_deferred( + deferred=non_completing_d, + timeout=1.0, + clock=self.clock, + ) self.assertNoResult(timing_out_d) - self.clock.pump((1.0,)) + self.reactor.pump((1.0,)) self.failureResultOf(timing_out_d, defer.TimeoutError) - def test_logcontext_is_preserved_on_cancellation(self) -> None: - blocking_was_cancelled = False + async def test_logcontext_is_preserved_on_cancellation(self) -> None: + # Sanity check that we start in the sentinel context + self.assertEqual(current_context(), SENTINEL_CONTEXT) - @defer.inlineCallbacks - def blocking() -> Generator["Deferred[object]", object, None]: - nonlocal blocking_was_cancelled + incomplete_deferred_was_cancelled = False - non_completing_d: Deferred = Deferred() - with PreserveLoggingContext(): - try: - yield non_completing_d - except CancelledError: - blocking_was_cancelled = True - raise + def mark_was_cancelled(res: Failure) -> None: + """ + A passthrough errback which sets `incomplete_deferred_was_cancelled`. - with LoggingContext("one") as context_one: - # the errbacks should be run in the test logcontext - def errback(res: Failure, deferred_name: str) -> Failure: - self.assertIs( - current_context(), - context_one, - "errback %s run in unexpected logcontext %s" - % (deferred_name, current_context()), + This means we re-raise any exception and allows further errbacks (in + `timeout_deferred(...)`) to do their thing. Just trying to be a transparent + proxy of any exception while doing our internal test book-keeping. + """ + nonlocal incomplete_deferred_was_cancelled + if res.check(CancelledError): + incomplete_deferred_was_cancelled = True + else: + logger.error( + "Expected incomplete_d to fail with `CancelledError` because our " + "`timeout_deferred(...)` utility canceled it but saw %s", + res, ) - return res - original_deferred = blocking() - original_deferred.addErrback(errback, "orig") - timing_out_d = timeout_deferred(original_deferred, 1.0, self.clock) - self.assertNoResult(timing_out_d) - self.assertIs(current_context(), SENTINEL_CONTEXT) - timing_out_d.addErrback(errback, "timingout") + # Re-raise the exception so that any further errbacks can do their thing as + # normal + res.raiseException() - self.clock.pump((1.0,)) + # Create a deferred which we will never complete + incomplete_d: Deferred = Deferred() + incomplete_d.addErrback(mark_was_cancelled) - self.assertTrue( - blocking_was_cancelled, "non-completing deferred was not cancelled" + with LoggingContext(name="one", server_name="test_server") as context_one: + timing_out_d = timeout_deferred( + deferred=incomplete_d, + timeout=1.0, + clock=self.clock, ) - self.failureResultOf(timing_out_d, defer.TimeoutError) + self.assertNoResult(timing_out_d) + # We should still be in the logcontext we started in self.assertIs(current_context(), context_one) + # Pump the reactor until we trigger the timeout + # + # We're manually pumping the reactor (and causing any pending callbacks to + # be called) so we need to be in the sentinel logcontext to avoid leaking + # our current logcontext into the reactor (which would then get picked up + # and associated with the next thing the reactor does). `with + # PreserveLoggingContext()` will reset the logcontext to the sentinel while + # we're pumping the reactor in the block and return us back to our current + # logcontext after the block. + with PreserveLoggingContext(): + self.reactor.pump( + # We only need to pump `1.0` (seconds) as we set + # `timeout_deferred(timeout=1.0)` above + (1.0,) + ) -class _TestException(Exception): + # We expect the incomplete deferred to have been cancelled because of the + # timeout by this point + self.assertTrue( + incomplete_deferred_was_cancelled, + "incomplete deferred was not cancelled", + ) + # We should see the `TimeoutError` (instead of a `CancelledError`) + self.failureResultOf(timing_out_d, defer.TimeoutError) + # We're still in the same logcontext + self.assertIs(current_context(), context_one) + + # Back to the sentinel context + self.assertEqual(current_context(), SENTINEL_CONTEXT) + + +class _TestException(Exception): # pass @@ -502,7 +543,7 @@ class DelayCancellationTests(TestCase): await make_deferred_yieldable(blocking_d) async def outer() -> None: - with LoggingContext("c") as c: + with LoggingContext(name="c", server_name="test_server") as c: try: await delay_cancellation(inner()) self.fail("`CancelledError` was not raised") @@ -526,8 +567,8 @@ class AwakenableSleeperTests(TestCase): "Tests AwakenableSleeper" def test_sleep(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -541,8 +582,8 @@ class AwakenableSleeperTests(TestCase): self.assertTrue(d.called) def test_explicit_wake(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -558,8 +599,8 @@ class AwakenableSleeperTests(TestCase): reactor.advance(0.6) def test_multiple_sleepers_timeout(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -578,8 +619,8 @@ class AwakenableSleeperTests(TestCase): self.assertTrue(d2.called) def test_multiple_sleepers_wake(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -617,7 +658,7 @@ class GatherCoroutineTests(TestCase): def test_single(self) -> None: "Test passing in a single coroutine works" - with LoggingContext("test_ctx") as text_ctx: + with LoggingContext(name="test_ctx", server_name="test_server") as text_ctx: deferred: "defer.Deferred[None]" coroutine, deferred = self.make_coroutine() @@ -643,7 +684,7 @@ class GatherCoroutineTests(TestCase): def test_multiple_resolve(self) -> None: "Test passing in multiple coroutine that all resolve works" - with LoggingContext("test_ctx") as test_ctx: + with LoggingContext(name="test_ctx", server_name="test_server") as test_ctx: deferred1: "defer.Deferred[int]" coroutine1, deferred1 = self.make_coroutine() deferred2: "defer.Deferred[str]" @@ -676,7 +717,7 @@ class GatherCoroutineTests(TestCase): def test_multiple_fail(self) -> None: "Test passing in multiple coroutine where one fails does the right thing" - with LoggingContext("test_ctx") as test_ctx: + with LoggingContext(name="test_ctx", server_name="test_server") as test_ctx: deferred1: "defer.Deferred[int]" coroutine1, deferred1 = self.make_coroutine() deferred2: "defer.Deferred[str]" diff --git a/tests/util/test_batching_queue.py b/tests/util/test_batching_queue.py index 532582cf87..60bfdf38aa 100644 --- a/tests/util/test_batching_queue.py +++ b/tests/util/test_batching_queue.py @@ -32,13 +32,12 @@ from synapse.util.batching_queue import ( number_queued, ) -from tests.server import get_clock -from tests.unittest import TestCase +from tests.unittest import HomeserverTestCase -class BatchingQueueTestCase(TestCase): +class BatchingQueueTestCase(HomeserverTestCase): def setUp(self) -> None: - self.clock, hs_clock = get_clock() + super().setUp() # We ensure that we remove any existing metrics for "test_queue". try: @@ -51,8 +50,8 @@ class BatchingQueueTestCase(TestCase): self._pending_calls: List[Tuple[List[str], defer.Deferred]] = [] self.queue: BatchingQueue[str, str] = BatchingQueue( name="test_queue", - server_name="test_server", - clock=hs_clock, + hs=self.hs, + clock=self.clock, process_batch_callback=self._process_queue, ) @@ -108,7 +107,7 @@ class BatchingQueueTestCase(TestCase): self.assertFalse(queue_d.called) # We should see a call to `_process_queue` after a reactor tick. - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo"]) @@ -134,7 +133,7 @@ class BatchingQueueTestCase(TestCase): self._assert_metrics(queued=2, keys=1, in_flight=2) - self.clock.pump([0]) + self.reactor.pump([0]) # We should see only *one* call to `_process_queue` self.assertEqual(len(self._pending_calls), 1) @@ -158,7 +157,7 @@ class BatchingQueueTestCase(TestCase): self.assertFalse(self._pending_calls) queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1")) - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) @@ -185,7 +184,7 @@ class BatchingQueueTestCase(TestCase): self._assert_metrics(queued=2, keys=1, in_flight=2) # We should now see a second call to `_process_queue` - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo2", "foo3"]) self.assertFalse(queue_d2.called) @@ -206,9 +205,9 @@ class BatchingQueueTestCase(TestCase): self.assertFalse(self._pending_calls) queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1", key=1)) - self.clock.pump([0]) + self.reactor.pump([0]) queue_d2 = defer.ensureDeferred(self.queue.add_to_queue("foo2", key=2)) - self.clock.pump([0]) + self.reactor.pump([0]) # We queue up another item with key=2 to check that we will keep taking # things off the queue. @@ -240,7 +239,7 @@ class BatchingQueueTestCase(TestCase): self.assertFalse(queue_d3.called) # We should now see a call `_pending_calls` for `foo3` - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo3"]) self.assertFalse(queue_d3.called) diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py index 246e18fd15..16e096a4b2 100644 --- a/tests/util/test_dict_cache.py +++ b/tests/util/test_dict_cache.py @@ -23,12 +23,14 @@ from synapse.util.caches.dictionary_cache import DictionaryCache from tests import unittest +from tests.server import get_clock class DictCacheTestCase(unittest.TestCase): def setUp(self) -> None: + _, clock = get_clock() self.cache: DictionaryCache[str, str, str] = DictionaryCache( - name="foobar", server_name="test_server", max_entries=10 + name="foobar", clock=clock, server_name="test_server", max_entries=10 ) def test_simple_cache_hit_full(self) -> None: diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index bfcc6cd12f..35c0f02e3f 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -19,23 +19,23 @@ # # -from typing import List, cast +from typing import List from synapse.util.caches.expiringcache import ExpiringCache -from synapse.util.clock import Clock -from tests.utils import MockClock +from tests.server import get_clock from .. import unittest class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: - clock = MockClock() + reactor, clock = get_clock() cache: ExpiringCache[str, str] = ExpiringCache( cache_name="test", server_name="testserver", - clock=cast(Clock, clock), + hs=self.hs, + clock=clock, max_len=1, ) @@ -44,11 +44,12 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache["key"], "value") def test_eviction(self) -> None: - clock = MockClock() + reactor, clock = get_clock() cache: ExpiringCache[str, str] = ExpiringCache( cache_name="test", server_name="testserver", - clock=cast(Clock, clock), + hs=self.hs, + clock=clock, max_len=2, ) @@ -63,11 +64,12 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key3"), "value3") def test_iterable_eviction(self) -> None: - clock = MockClock() + reactor, clock = get_clock() cache: ExpiringCache[str, List[int]] = ExpiringCache( cache_name="test", server_name="testserver", - clock=cast(Clock, clock), + hs=self.hs, + clock=clock, max_len=5, iterable=True, ) @@ -87,25 +89,26 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key4"), [6, 7]) def test_time_eviction(self) -> None: - clock = MockClock() + reactor, clock = get_clock() cache: ExpiringCache[str, int] = ExpiringCache( cache_name="test", server_name="testserver", - clock=cast(Clock, clock), + hs=self.hs, + clock=clock, expiry_ms=1000, ) cache["key"] = 1 - clock.advance_time(0.5) + reactor.advance(0.5) cache["key2"] = 2 self.assertEqual(cache.get("key"), 1) self.assertEqual(cache.get("key2"), 2) - clock.advance_time(0.9) + reactor.advance(0.9) self.assertEqual(cache.get("key"), None) self.assertEqual(cache.get("key2"), 2) - clock.advance_time(1) + reactor.advance(1) self.assertEqual(cache.get("key"), None) self.assertEqual(cache.get("key2"), None) diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py index 7510657b85..722ce79dcc 100644 --- a/tests/util/test_linearizer.py +++ b/tests/util/test_linearizer.py @@ -21,14 +21,16 @@ from typing import Hashable, Protocol, Tuple -from twisted.internet import defer, reactor -from twisted.internet.base import ReactorBase +from twisted.internet import defer from twisted.internet.defer import CancelledError, Deferred from synapse.logging.context import LoggingContext, current_context from synapse.util.async_helpers import Linearizer from tests import unittest +from tests.server import ( + get_clock, +) class UnblockFunction(Protocol): @@ -36,6 +38,9 @@ class UnblockFunction(Protocol): class LinearizerTestCase(unittest.TestCase): + def setUp(self) -> None: + self.reactor, self.clock = get_clock() + def _start_task( self, linearizer: Linearizer, key: Hashable ) -> Tuple["Deferred[None]", "Deferred[None]", UnblockFunction]: @@ -73,13 +78,12 @@ class LinearizerTestCase(unittest.TestCase): def _pump(self) -> None: """Pump the reactor to advance `Linearizer`s.""" - assert isinstance(reactor, ReactorBase) - while reactor.getDelayedCalls(): - reactor.runUntilCurrent() + while self.reactor.getDelayedCalls(): + self.reactor.pump([0] * 100) def test_linearizer(self) -> None: """Tests that a task is queued up behind an earlier task.""" - linearizer = Linearizer() + linearizer = Linearizer(name="test_linearizer", clock=self.clock) key = object() @@ -100,7 +104,7 @@ class LinearizerTestCase(unittest.TestCase): Runs through the same scenario as `test_linearizer`. """ - linearizer = Linearizer() + linearizer = Linearizer(name="test_linearizer", clock=self.clock) key = object() @@ -131,11 +135,11 @@ class LinearizerTestCase(unittest.TestCase): The stack should *not* explode when the slow thing completes. """ - linearizer = Linearizer() + linearizer = Linearizer(name="test_linearizer", clock=self.clock) key = "" async def func(i: int) -> None: - with LoggingContext("func(%s)" % i) as lc: + with LoggingContext(name="func(%s)" % i, server_name="test_server") as lc: async with linearizer.queue(key): self.assertEqual(current_context(), lc) @@ -151,24 +155,24 @@ class LinearizerTestCase(unittest.TestCase): def test_multiple_entries(self) -> None: """Tests a `Linearizer` with a concurrency above 1.""" - limiter = Linearizer(max_count=3) + linearizer = Linearizer(name="test_linearizer", max_count=3, clock=self.clock) key = object() - _, acquired_d1, unblock1 = self._start_task(limiter, key) + _, acquired_d1, unblock1 = self._start_task(linearizer, key) self.assertTrue(acquired_d1.called) - _, acquired_d2, unblock2 = self._start_task(limiter, key) + _, acquired_d2, unblock2 = self._start_task(linearizer, key) self.assertTrue(acquired_d2.called) - _, acquired_d3, unblock3 = self._start_task(limiter, key) + _, acquired_d3, unblock3 = self._start_task(linearizer, key) self.assertTrue(acquired_d3.called) # These next two tasks have to wait. - _, acquired_d4, unblock4 = self._start_task(limiter, key) + _, acquired_d4, unblock4 = self._start_task(linearizer, key) self.assertFalse(acquired_d4.called) - _, acquired_d5, unblock5 = self._start_task(limiter, key) + _, acquired_d5, unblock5 = self._start_task(linearizer, key) self.assertFalse(acquired_d5.called) # Once the first task completes, the fourth task can continue. @@ -186,13 +190,13 @@ class LinearizerTestCase(unittest.TestCase): unblock5() # The next task shouldn't have to wait. - _, acquired_d6, unblock6 = self._start_task(limiter, key) + _, acquired_d6, unblock6 = self._start_task(linearizer, key) self.assertTrue(acquired_d6) unblock6() def test_cancellation(self) -> None: """Tests cancellation while waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(name="test_linearizer", clock=self.clock) key = object() @@ -226,7 +230,7 @@ class LinearizerTestCase(unittest.TestCase): def test_cancellation_during_sleep(self) -> None: """Tests cancellation during the sleep just after waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(name="test_linearizer", clock=self.clock) key = object() diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index 651dd84483..ca805bb20a 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -58,7 +58,7 @@ class LoggingContextTestCase(unittest.TestCase): @logcontext_clean def test_with_context(self) -> None: - with LoggingContext("test"): + with LoggingContext(name="test", server_name="test_server"): self._check_test_key("test") @logcontext_clean @@ -66,7 +66,8 @@ class LoggingContextTestCase(unittest.TestCase): """ Test `Clock.sleep` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -80,7 +81,7 @@ class LoggingContextTestCase(unittest.TestCase): # other words, another task shouldn't have leaked their context to us. self._check_test_key("sentinel") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): await clock.sleep(0) self._check_test_key("competing") @@ -90,9 +91,9 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - reactor.callLater(0, lambda: defer.ensureDeferred(competing_callback())) + reactor.callLater(0, lambda: defer.ensureDeferred(competing_callback())) # type: ignore[call-later-not-tracked] - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): await clock.sleep(0) self._check_test_key("foo") await clock.sleep(0) @@ -111,7 +112,8 @@ class LoggingContextTestCase(unittest.TestCase): """ Test `Clock.looping_call` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -125,7 +127,7 @@ class LoggingContextTestCase(unittest.TestCase): # which server spawned this loop and which server the logs came from. self._check_test_key("looping_call") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): await clock.sleep(0) self._check_test_key("competing") @@ -135,7 +137,7 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): lc = clock.looping_call( lambda: defer.ensureDeferred(competing_callback()), 0 ) @@ -161,7 +163,8 @@ class LoggingContextTestCase(unittest.TestCase): """ Test `Clock.looping_call_now` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -175,7 +178,7 @@ class LoggingContextTestCase(unittest.TestCase): # which server spawned this loop and which server the logs came from. self._check_test_key("looping_call") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): await clock.sleep(0) self._check_test_key("competing") @@ -185,7 +188,7 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): lc = clock.looping_call_now( lambda: defer.ensureDeferred(competing_callback()), 0 ) @@ -209,7 +212,8 @@ class LoggingContextTestCase(unittest.TestCase): """ Test `Clock.call_later` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -223,7 +227,7 @@ class LoggingContextTestCase(unittest.TestCase): # which server spawned this loop and which server the logs came from. self._check_test_key("call_later") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): await clock.sleep(0) self._check_test_key("competing") @@ -233,7 +237,7 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): clock.call_later(0, lambda: defer.ensureDeferred(competing_callback())) self._check_test_key("foo") await clock.sleep(0) @@ -249,15 +253,219 @@ class LoggingContextTestCase(unittest.TestCase): # Back to the sentinel context self._check_test_key("sentinel") - async def _test_run_in_background(self, function: Callable[[], object]) -> None: - clock = Clock(reactor) + @logcontext_clean + async def test_deferred_callback_await_in_current_logcontext(self) -> None: + """ + Test that calling the deferred callback in the current logcontext ("foo") and + waiting for it to finish in a logcontext blocks works as expected. + + Works because "always await your awaitables". + + Demonstrates one pattern that we can use fix the naive case where we just call + `d.callback(None)` without anything else. See the *Deferred callbacks* section + of docs/log_contexts.md for more details. + """ + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") callback_finished = False - with LoggingContext("foo"): + async def competing_callback() -> None: + nonlocal callback_finished + try: + # The deferred callback should have the same logcontext as the caller + self._check_test_key("foo") + + with LoggingContext(name="competing", server_name="test_server"): + await clock.sleep(0) + self._check_test_key("competing") + + self._check_test_key("foo") + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + with LoggingContext(name="foo", server_name="test_server"): + d: defer.Deferred[None] = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + self._check_test_key("foo") + d.callback(None) + # The fix for the naive case is here (i.e. things don't work correctly if we + # don't await here). + # + # Wait for `d` to finish before continuing so the "main" logcontext is + # still active. This works because `d` already follows our logcontext + # rules. If not, we would also have to use `make_deferred_yieldable(d)`. + await d + self._check_test_key("foo") + + await clock.sleep(0) + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + # Back to the sentinel context + self._check_test_key("sentinel") + + @logcontext_clean + async def test_deferred_callback_preserve_logging_context(self) -> None: + """ + Test that calling the deferred callback inside `PreserveLoggingContext()` (in + the sentinel context) works as expected. + + Demonstrates one pattern that we can use fix the naive case where we just call + `d.callback(None)` without anything else. See the *Deferred callbacks* section + of docs/log_contexts.md for more details. + """ + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] + + # Sanity check that we start in the sentinel context + self._check_test_key("sentinel") + + callback_finished = False + + async def competing_callback() -> None: + nonlocal callback_finished + try: + # The deferred callback should have the same logcontext as the caller + self._check_test_key("sentinel") + + with LoggingContext(name="competing", server_name="test_server"): + await clock.sleep(0) + self._check_test_key("competing") + + self._check_test_key("sentinel") + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + with LoggingContext(name="foo", server_name="test_server"): + d: defer.Deferred[None] = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + self._check_test_key("foo") + # The fix for the naive case is here (i.e. things don't work correctly if we + # don't `PreserveLoggingContext()` here). + # + # `PreserveLoggingContext` will reset the logcontext to the sentinel before + # calling the callback, and restore the "foo" logcontext afterwards before + # continuing the foo block. This solves the problem because when the + # "competing" logcontext exits, it will restore the sentinel logcontext + # which is never finished by its nature, so there is no warning and no + # leakage into the reactor. + with PreserveLoggingContext(): + d.callback(None) + self._check_test_key("foo") + + await clock.sleep(0) + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + # Back to the sentinel context + self._check_test_key("sentinel") + + @logcontext_clean + async def test_deferred_callback_fire_and_forget_with_current_context(self) -> None: + """ + Test that it's possible to call the deferred callback with the current context + while fire-and-forgetting the callback (no adverse effects like leaking the + logcontext into the reactor or restarting an already finished logcontext). + + Demonstrates one pattern that we can use fix the naive case where we just call + `d.callback(None)` without anything else. See the *Deferred callbacks* section + of docs/log_contexts.md for more details. + """ + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] + + # Sanity check that we start in the sentinel context + self._check_test_key("sentinel") + + callback_finished = False + + async def competing_callback() -> None: + nonlocal callback_finished + try: + # The deferred callback should have the same logcontext as the caller + self._check_test_key("foo") + + with LoggingContext(name="competing", server_name="test_server"): + await clock.sleep(0) + self._check_test_key("competing") + + self._check_test_key("foo") + finally: + # When exceptions happen, we still want to mark the callback as finished + # so that the test can complete and we see the underlying error. + callback_finished = True + + # Part of fix for the naive case is here (i.e. things don't work correctly if we + # don't `PreserveLoggingContext(...)` here). + # + # We can extend the lifetime of the "foo" logcontext is to avoid calling the + # context manager lifetime methods of `LoggingContext` (`__enter__`/`__exit__`). + # And we can still set the current logcontext by using `PreserveLoggingContext` + # and passing in the "foo" logcontext. + with PreserveLoggingContext( + LoggingContext(name="foo", server_name="test_server") + ): + d: defer.Deferred[None] = defer.Deferred() + d.addCallback(lambda _: defer.ensureDeferred(competing_callback())) + self._check_test_key("foo") + # Other part of fix for the naive case is here (i.e. things don't work + # correctly if we don't `run_in_background(...)` here). + # + # `run_in_background(...)` will run the whole lambda in the current + # logcontext and it handles the magic behind the scenes of a) restoring the + # calling logcontext before returning to the caller and b) resetting the + # logcontext to the sentinel after the deferred completes and we yield + # control back to the reactor to avoid leaking the logcontext into the + # reactor. + # + # We're using a lambda here as a little trick so we can still get everything + # to run in the "foo" logcontext, but return the deferred `d` itself so that + # `run_in_background` will wait on that to complete before resetting the + # logcontext to the sentinel. + # + # type-ignore[call-overload]: This appears like a mypy type inference bug. A + # function that returns a deferred is exactly what `run_in_background` + # expects. + # + # type-ignore[func-returns-value]: This appears like a mypy type inference + # bug. We're always returning the deferred `d`. + run_in_background(lambda: (d.callback(None), d)[1]) # type: ignore[call-overload, func-returns-value] + self._check_test_key("foo") + + await clock.sleep(0) + + self.assertTrue( + callback_finished, + "Callback never finished which means the test probably didn't wait long enough", + ) + + # Back to the sentinel context + self._check_test_key("sentinel") + + async def _test_run_in_background(self, function: Callable[[], object]) -> None: + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] + + # Sanity check that we start in the sentinel context + self._check_test_key("sentinel") + + callback_finished = False + + with LoggingContext(name="foo", server_name="test_server"): # Fire off the function, but don't wait on it. deferred = run_in_background(function) self._check_test_key("foo") @@ -292,7 +500,8 @@ class LoggingContextTestCase(unittest.TestCase): @logcontext_clean async def test_run_in_background_with_blocking_fn(self) -> None: async def blocking_function() -> None: - await Clock(reactor).sleep(0) + # Ignore linter error since we are creating a `Clock` for testing purposes. + await Clock(reactor, server_name="test_server").sleep(0) # type: ignore[multiple-internal-clocks] await self._test_run_in_background(blocking_function) @@ -325,7 +534,8 @@ class LoggingContextTestCase(unittest.TestCase): async def testfunc() -> None: self._check_test_key("foo") - d = defer.ensureDeferred(Clock(reactor).sleep(0)) + # Ignore linter error since we are creating a `Clock` for testing purposes. + d = defer.ensureDeferred(Clock(reactor, server_name="test_server").sleep(0)) # type: ignore[multiple-internal-clocks] self.assertIs(current_context(), SENTINEL_CONTEXT) await d self._check_test_key("foo") @@ -354,7 +564,8 @@ class LoggingContextTestCase(unittest.TestCase): This will stress the logic around incomplete deferreds in `run_coroutine_in_background`. """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor, server_name="test_server") # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -367,7 +578,7 @@ class LoggingContextTestCase(unittest.TestCase): # The callback should have the same logcontext as the caller self._check_test_key("foo") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): await clock.sleep(0) self._check_test_key("competing") @@ -377,7 +588,7 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): run_coroutine_in_background(competing_callback()) self._check_test_key("foo") await clock.sleep(0) @@ -410,7 +621,7 @@ class LoggingContextTestCase(unittest.TestCase): # The callback should have the same logcontext as the caller self._check_test_key("foo") - with LoggingContext("competing"): + with LoggingContext(name="competing", server_name="test_server"): # We `await` here but there is nothing to wait for here since the # deferred is already complete so we should immediately continue # executing in the same context. @@ -424,7 +635,7 @@ class LoggingContextTestCase(unittest.TestCase): # so that the test can complete and we see the underlying error. callback_finished = True - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): run_coroutine_in_background(competing_callback()) self._check_test_key("foo") @@ -445,12 +656,12 @@ class LoggingContextTestCase(unittest.TestCase): # the synapse rules. def blocking_function() -> defer.Deferred: d: defer.Deferred = defer.Deferred() - reactor.callLater(0, d.callback, None) + reactor.callLater(0, d.callback, None) # type: ignore[call-later-not-tracked] return d sentinel_context = current_context() - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): d1 = make_deferred_yieldable(blocking_function()) # make sure that the context was reset by make_deferred_yieldable self.assertIs(current_context(), sentinel_context) @@ -467,7 +678,7 @@ class LoggingContextTestCase(unittest.TestCase): ) -> Generator["defer.Deferred[object]", object, None]: sentinel_context = current_context() - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): d1 = make_deferred_yieldable(_chained_deferred_function()) # make sure that the context was reset by make_deferred_yieldable self.assertIs(current_context(), sentinel_context) @@ -479,7 +690,7 @@ class LoggingContextTestCase(unittest.TestCase): @logcontext_clean def test_nested_logging_context(self) -> None: - with LoggingContext("foo"): + with LoggingContext(name="foo", server_name="test_server"): nested_context = nested_logging_context(suffix="bar") self.assertEqual(nested_context.name, "foo-bar") @@ -492,7 +703,7 @@ def _chained_deferred_function() -> defer.Deferred: def cb(res: object) -> defer.Deferred: d2: defer.Deferred = defer.Deferred() - reactor.callLater(0, d2.callback, res) + reactor.callLater(0, d2.callback, res) # type: ignore[call-later-not-tracked] return d2 d.addCallback(cb) diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index b7acf58690..56e9996b00 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -29,18 +29,28 @@ from synapse.util.caches.lrucache import LruCache, setup_expire_lru_cache_entrie from synapse.util.caches.treecache import TreeCache from tests import unittest +from tests.server import get_clock from tests.unittest import override_config class LruCacheTestCase(unittest.HomeserverTestCase): + def setUp(self) -> None: + super().setUp() + + _, self.clock = get_clock() + def test_get_set(self) -> None: - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache["key"] = "value" self.assertEqual(cache.get("key"), "value") self.assertEqual(cache["key"], "value") def test_eviction(self) -> None: - cache: LruCache[int, int] = LruCache(max_size=2) + cache: LruCache[int, int] = LruCache( + max_size=2, clock=self.clock, server_name="test_server" + ) cache[1] = 1 cache[2] = 2 @@ -54,7 +64,9 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get(3), 3) def test_setdefault(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) self.assertEqual(cache.setdefault("key", 1), 1) self.assertEqual(cache.get("key"), 1) self.assertEqual(cache.setdefault("key", 2), 1) @@ -63,7 +75,9 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key"), 2) def test_pop(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache["key"] = 1 self.assertEqual(cache.pop("key"), 1) self.assertEqual(cache.pop("key"), None) @@ -71,7 +85,10 @@ class LruCacheTestCase(unittest.HomeserverTestCase): def test_del_multi(self) -> None: # The type here isn't quite correct as they don't handle TreeCache well. cache: LruCache[Tuple[str, str], str] = LruCache( - max_size=4, cache_type=TreeCache + max_size=4, + clock=self.clock, + cache_type=TreeCache, + server_name="test_server", ) cache[("animal", "cat")] = "mew" cache[("animal", "dog")] = "woof" @@ -91,7 +108,9 @@ class LruCacheTestCase(unittest.HomeserverTestCase): # Man from del_multi say "Yes". def test_clear(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache["key"] = 1 cache.clear() self.assertEqual(len(cache), 0) @@ -99,7 +118,10 @@ class LruCacheTestCase(unittest.HomeserverTestCase): @override_config({"caches": {"per_cache_factors": {"mycache": 10}}}) def test_special_size(self) -> None: cache: LruCache = LruCache( - max_size=10, server_name="test_server", cache_name="mycache" + max_size=10, + clock=self.clock, + server_name="test_server", + cache_name="mycache", ) self.assertEqual(cache.max_size, 100) @@ -107,7 +129,9 @@ class LruCacheTestCase(unittest.HomeserverTestCase): class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache.set("key", "value") self.assertFalse(m.called) @@ -126,7 +150,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_multi_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache.set("key", "value") self.assertFalse(m.called) @@ -145,7 +171,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_set(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -161,7 +189,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_pop(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache( + max_size=1, clock=self.clock, server_name="test_server" + ) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -182,7 +212,10 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m4 = Mock() # The type here isn't quite correct as they don't handle TreeCache well. cache: LruCache[Tuple[str, str], str] = LruCache( - max_size=4, cache_type=TreeCache + max_size=4, + clock=self.clock, + cache_type=TreeCache, + server_name="test_server", ) cache.set(("a", "1"), "value", callbacks=[m1]) @@ -205,7 +238,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_clear(self) -> None: m1 = Mock() m2 = Mock() - cache: LruCache[str, str] = LruCache(max_size=5) + cache: LruCache[str, str] = LruCache( + max_size=5, clock=self.clock, server_name="test_server" + ) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -222,7 +257,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m1 = Mock(name="m1") m2 = Mock(name="m2") m3 = Mock(name="m3") - cache: LruCache[str, str] = LruCache(max_size=2) + cache: LruCache[str, str] = LruCache( + max_size=2, clock=self.clock, server_name="test_server" + ) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -258,7 +295,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: - cache: LruCache[str, List[int]] = LruCache(max_size=5, size_callback=len) + cache: LruCache[str, List[int]] = LruCache( + max_size=5, clock=self.clock, size_callback=len, server_name="test_server" + ) cache["key1"] = [0] cache["key2"] = [1, 2] cache["key3"] = [3] @@ -282,7 +321,10 @@ class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_zero_size_drop_from_cache(self) -> None: """Test that `drop_from_cache` works correctly with 0-sized entries.""" cache: LruCache[str, List[int]] = LruCache( - max_size=5, size_callback=lambda x: 0 + max_size=5, + clock=self.clock, + size_callback=lambda x: 0, + server_name="test_server", ) cache["key1"] = [] @@ -307,7 +349,9 @@ class TimeEvictionTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(max_size=5, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache( + max_size=5, server_name="test_server", clock=self.hs.get_clock() + ) # Check that we evict entries we haven't accessed for 30 minutes. cache["key1"] = 1 @@ -359,7 +403,9 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): mock_jemalloc_class.get_stat.return_value = 924288000 setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(max_size=4, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache( + max_size=4, server_name="test_server", clock=self.hs.get_clock() + ) cache["key1"] = 1 cache["key2"] = 2 @@ -396,7 +442,10 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): def test_invalidate_simple(self) -> None: cache: LruCache[str, int] = LruCache( - max_size=10, extra_index_cb=lambda k, v: str(v) + max_size=10, + clock=self.hs.get_clock(), + server_name="test_server", + extra_index_cb=lambda k, v: str(v), ) cache["key1"] = 1 cache["key2"] = 2 @@ -411,7 +460,10 @@ class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): def test_invalidate_multi(self) -> None: cache: LruCache[str, int] = LruCache( - max_size=10, extra_index_cb=lambda k, v: str(v) + max_size=10, + clock=self.hs.get_clock(), + server_name="test_server", + extra_index_cb=lambda k, v: str(v), ) cache["key1"] = 1 cache["key2"] = 1 diff --git a/tests/util/test_retryutils.py b/tests/util/test_retryutils.py index 82baff5883..593be93ea3 100644 --- a/tests/util/test_retryutils.py +++ b/tests/util/test_retryutils.py @@ -35,6 +35,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -57,6 +58,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -89,6 +91,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), @@ -104,6 +107,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -139,6 +143,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -165,6 +170,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, notifier=notifier, @@ -238,6 +244,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -261,6 +268,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), @@ -273,6 +281,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -297,6 +306,7 @@ class RetryLimiterTestCase(HomeserverTestCase): get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), diff --git a/tests/utils.py b/tests/utils.py index d1b66d4159..051388ee2e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -24,27 +24,19 @@ import os import signal from types import FrameType, TracebackType from typing import ( - Any, - Callable, Dict, - List, Literal, Optional, - Tuple, Type, TypeVar, Union, overload, ) -import attr -from typing_extensions import ParamSpec - from synapse.api.constants import EventTypes from synapse.api.room_versions import RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.config.server import DEFAULT_ROOM_VERSION -from synapse.logging.context import current_context, set_current_context from synapse.server import HomeServer from synapse.storage.database import LoggingDatabaseConnection from synapse.storage.engines import create_engine @@ -140,21 +132,27 @@ def setupdb() -> None: @overload -def default_config(name: str, parse: Literal[False] = ...) -> Dict[str, object]: ... +def default_config( + server_name: str, parse: Literal[False] = ... +) -> Dict[str, object]: ... @overload -def default_config(name: str, parse: Literal[True]) -> HomeServerConfig: ... +def default_config(server_name: str, parse: Literal[True]) -> HomeServerConfig: ... def default_config( - name: str, parse: bool = False + server_name: str, parse: bool = False ) -> Union[Dict[str, object], HomeServerConfig]: """ Create a reasonable test config. + + Args: + server_name: homeserver name + parse: TODO """ config_dict = { - "server_name": name, + "server_name": server_name, # Setting this to an empty list turns off federation sending. "federation_sender_instances": [], "media_store_path": "media", @@ -247,101 +245,6 @@ def mock_getRawHeaders(headers=None): # type: ignore[no-untyped-def] return getRawHeaders -P = ParamSpec("P") - - -@attr.s(slots=True, auto_attribs=True) -class Timer: - absolute_time: float - callback: Callable[[], None] - expired: bool - - -# TODO: Make this generic over a ParamSpec? -@attr.s(slots=True, auto_attribs=True) -class Looper: - func: Callable[..., Any] - interval: float # seconds - last: float - args: Tuple[object, ...] - kwargs: Dict[str, object] - - -class MockClock: - now = 1000.0 - - def __init__(self) -> None: - # Timers in no particular order - self.timers: List[Timer] = [] - self.loopers: List[Looper] = [] - - def time(self) -> float: - return self.now - - def time_msec(self) -> int: - return int(self.time() * 1000) - - def call_later( - self, - delay: float, - callback: Callable[P, object], - *args: P.args, - **kwargs: P.kwargs, - ) -> Timer: - ctx = current_context() - - def wrapped_callback() -> None: - set_current_context(ctx) - callback(*args, **kwargs) - - t = Timer(self.now + delay, wrapped_callback, False) - self.timers.append(t) - - return t - - def looping_call( - self, - function: Callable[P, object], - interval: float, - *args: P.args, - **kwargs: P.kwargs, - ) -> None: - self.loopers.append(Looper(function, interval / 1000.0, self.now, args, kwargs)) - - def cancel_call_later(self, timer: Timer, ignore_errs: bool = False) -> None: - if timer.expired: - if not ignore_errs: - raise Exception("Cannot cancel an expired timer") - - timer.expired = True - self.timers = [t for t in self.timers if t != timer] - - # For unit testing - def advance_time(self, secs: float) -> None: - self.now += secs - - timers = self.timers - self.timers = [] - - for t in timers: - if t.expired: - raise Exception("Timer already expired") - - if self.now >= t.absolute_time: - t.expired = True - t.callback() - else: - self.timers.append(t) - - for looped in self.loopers: - if looped.last + looped.interval < self.now: - looped.func(*looped.args, **looped.kwargs) - looped.last = self.now - - def advance_time_msec(self, ms: float) -> None: - self.advance_time(ms / 1000.0) - - async def create_room(hs: HomeServer, room_id: str, creator_id: str) -> None: """Creates and persist a creation event for the given room"""