From 98f65d9cc4b85a2833ec0afd7bd58d50ad6e4edb Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 11 Aug 2026 20:52:11 +0000 Subject: [PATCH] Don't hang when background updates fail in `update_synapse_database`. `BackgroundUpdater.run_background_updates` raises after five back-to-back failures. The exception was swallowed by the background process wrapper, so `reactor.stop()` was never reached and the script sat in the reactor forever rather than reporting the failure. Catch it, log it, stop the reactor either way and exit non-zero. Signed-off-by: Olivier 'reivilibre --- changelog.d/20102.bugfix | 1 + synapse/_scripts/update_synapse_database.py | 30 ++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 changelog.d/20102.bugfix diff --git a/changelog.d/20102.bugfix b/changelog.d/20102.bugfix new file mode 100644 index 0000000000..9e20e679b3 --- /dev/null +++ b/changelog.d/20102.bugfix @@ -0,0 +1 @@ +Make `update_synapse_database.py --run-background-updates` exit with an error instead of hanging when a background update fails. diff --git a/synapse/_scripts/update_synapse_database.py b/synapse/_scripts/update_synapse_database.py index ce32f47d63..072f3d8deb 100644 --- a/synapse/_scripts/update_synapse_database.py +++ b/synapse/_scripts/update_synapse_database.py @@ -21,6 +21,7 @@ import argparse import logging +import sys from typing import cast import yaml @@ -49,16 +50,28 @@ class MockHomeserver(HomeServer): ) -def run_background_updates(hs: HomeServer) -> None: +def run_background_updates(hs: HomeServer) -> bool: + """Run all pending background updates. Returns True if they all succeeded.""" main = hs.get_datastores().main state = hs.get_datastores().state + succeeded = True async def run_background_updates() -> None: - await main.db_pool.updates.run_background_updates(sleep=False) - if state: - await state.db_pool.updates.run_background_updates(sleep=False) - # Stop the reactor to exit the script once every background update is run. - reactor.stop() + nonlocal succeeded + try: + await main.db_pool.updates.run_background_updates(sleep=False) + if state: + await state.db_pool.updates.run_background_updates(sleep=False) + except Exception: + # `run_background_updates` gives up after repeated failures. Without + # this the exception would be swallowed by the background process + # wrapper, the reactor would never be stopped and the script would + # hang rather than report the failure. + logger.exception("Background updates failed") + succeeded = False + finally: + # Stop the reactor to exit the script once every background update is run. + reactor.stop() def run() -> None: # Apply all background updates on the database. @@ -73,6 +86,8 @@ def run_background_updates(hs: HomeServer) -> None: reactor.run() + return succeeded + def main() -> None: parser = argparse.ArgumentParser( @@ -123,7 +138,8 @@ def main() -> None: hs.get_storage_controllers() if args.run_background_updates: - run_background_updates(hs) + if not run_background_updates(hs): + sys.exit(1) if __name__ == "__main__":