From 2d450d415fe3ce9a913808f85de64f0d6c1eb897 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 1 Apr 2026 13:19:34 +0200 Subject: [PATCH] Update Docker files and complement script to use uv instead of poetry Replace poetry-based dependency management with uv across Docker builds and the complement test script: - Dockerfile: Remove the `poetry export` stage entirely. Use `uv sync` directly in the builder stage following uv best practices (first sync deps with --no-install-project, then copy source and do full sync). Install into a virtualenv and copy it to the runtime image. - editable.Dockerfile: Replace `pip install poetry` + `poetry install` with `uv sync` using the uv base image. Use UV_PROJECT_ENVIRONMENT to install into system Python. - complement.sh: Update lockfile diff check from poetry.lock to uv.lock, remove `poetry run` wrapper, rename TEST_ONLY_IGNORE_POETRY_LOCKFILE to TEST_ONLY_IGNORE_LOCKFILE. - start.py / configure_workers_and_start.py: Change shebang from /usr/local/bin/python to /usr/bin/env python for venv compatibility. - latest_deps.yml: Rename build arg and lockfile references. Part of #19567. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/latest_deps.yml | 8 +- docker/Dockerfile | 101 +++++++++----------------- docker/configure_workers_and_start.py | 2 +- docker/editable.Dockerfile | 23 +++--- docker/start.py | 2 +- scripts-dev/complement.sh | 12 +-- 6 files changed, 61 insertions(+), 87 deletions(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 746223ef4a..6d94a1e0eb 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -160,8 +160,8 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Ensure sytest runs `pip install` - # Delete the lockfile so sytest will `pip install` rather than `poetry install` - run: rm /src/poetry.lock + # Delete the lockfile so sytest will `pip install` rather than `uv sync` + run: rm /src/uv.lock working-directory: /src - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -229,7 +229,7 @@ jobs: env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 + TEST_ONLY_IGNORE_LOCKFILE: 1 - name: Formatted Complement test logs # Always run this step if we attempted to run the Complement tests. @@ -253,7 +253,7 @@ jobs: env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 + TEST_ONLY_IGNORE_LOCKFILE: 1 - name: Formatted in-repo Complement test logs # Always run this step if we attempted to run the Complement tests. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6070d5c355..ba12675caa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,55 +15,11 @@ # DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 . # -# Irritatingly, there is no blessed guide on how to distribute an application with its -# poetry-managed environment in a docker image. We have opted for -# `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in -# in `poetry export` in the past. - ARG DEBIAN_VERSION=trixie ARG PYTHON_VERSION=3.13 -ARG POETRY_VERSION=2.2.1 ### -### Stage 0: generate requirements.txt -### -### This stage is platform-agnostic, so we can use the build platform in case of cross-compilation. -### -FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS requirements - -WORKDIR /synapse - -# Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock /synapse/ - - -# If specified, we won't verify the hashes of dependencies. -# This is only needed if the hashes of dependencies cannot be checked for some -# reason, such as when a git repository is used directly as a dependency. -ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION - -# If specified, we won't use the Poetry lockfile. -# Instead, we'll just install what a regular `pip install` would from PyPI. -ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE - -# This silences a warning as uv isn't able to do hardlinks between its cache -# (mounted as --mount=type=cache) and the target directory. -ENV UV_LINK_MODE=copy - -# Export the dependencies, but only if we're actually going to use the Poetry lockfile. -# Otherwise, just create an empty requirements file so that the Dockerfile can -# proceed. -ARG POETRY_VERSION -RUN --mount=type=cache,target=/root/.cache/uv \ - if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - uvx --with poetry-plugin-export==1.9.0 \ - poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ - else \ - touch /synapse/requirements.txt; \ - fi - -### -### Stage 1: builder +### Stage 0: builder ### FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS builder @@ -84,39 +40,51 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool ARG CARGO_NET_GIT_FETCH_WITH_CLI=false ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI +WORKDIR /synapse + +# If specified, we won't verify the hashes of dependencies. +# This is only needed if the hashes of dependencies cannot be checked for some +# reason, such as when a git repository is used directly as a dependency. +ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION + +# If specified, we won't use the uv lockfile. +# Instead, we'll just install what a regular `pip install` would from PyPI. +ARG TEST_ONLY_IGNORE_LOCKFILE + # To speed up rebuilds, install all of the dependencies before we copy over # the whole synapse project, so that this layer in the Docker cache can be -# used while you develop on the source +# used while you develop on the source. # -# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. -COPY --from=requirements /synapse/requirements.txt /synapse/ +# We use `uv sync --no-install-project` to install only the dependencies +# from the lockfile, without installing the project itself. +COPY pyproject.toml uv.lock /synapse/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install --prefix="/install" --no-deps -r /synapse/requirements.txt + if [ -z "$TEST_ONLY_IGNORE_LOCKFILE" ]; then \ + uv sync --extra all --frozen --no-install-project --no-dev ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--no-verify-hashes}; \ + fi # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ COPY rust /synapse/rust/ -# ... and what we need to `pip install`. -COPY pyproject.toml README.rst build_rust.py Cargo.toml Cargo.lock /synapse/ - -# Repeat of earlier build argument declaration, as this is a new build stage. -ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE +# ... and what we need to build and install the project. +COPY README.rst build_rust.py Cargo.toml Cargo.lock /synapse/ # Install the synapse package itself. -# If we have populated requirements.txt, we don't install any dependencies -# as we should already have those from the previous `pip install` step. +# If we are using the lockfile, we use `uv sync` which will reuse the +# already-installed dependencies from the previous step. +# Otherwise, we fall back to `uv pip install` which resolves freely from PyPI. RUN \ --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/synapse/target,sharing=locked \ --mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \ - if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - uv pip install --prefix="/install" --no-deps /synapse[all]; \ + if [ -z "$TEST_ONLY_IGNORE_LOCKFILE" ]; then \ + uv sync --extra all --frozen --no-dev --no-editable; \ else \ - uv pip install --prefix="/install" /synapse[all]; \ + uv pip install --python=.venv /synapse[all]; \ fi ### -### Stage 2: runtime dependencies download for ARM64 and AMD64 +### Stage 1: runtime dependencies download for ARM64 and AMD64 ### FROM --platform=$BUILDPLATFORM docker.io/library/debian:${DEBIAN_VERSION} AS runtime-deps @@ -164,7 +132,7 @@ RUN \ ### -### Stage 3: runtime +### Stage 2: runtime ### FROM docker.io/library/python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION} @@ -188,11 +156,12 @@ COPY --from=runtime-deps /install-${TARGETARCH}/etc /etc COPY --from=runtime-deps /install-${TARGETARCH}/usr /usr COPY --from=runtime-deps /install-${TARGETARCH}/var /var -# Copy the installed python packages from the builder stage. -# -# uv will generate a `.lock` file when installing packages, which we don't want -# to copy to the final image. -COPY --from=builder --exclude=.lock /install /usr/local +# Copy the virtual environment from the builder stage. +COPY --from=builder /synapse/.venv /synapse/.venv + +# Put the virtualenv on PATH +ENV PATH="/synapse/.venv/bin:$PATH" + COPY ./docker/start.py /start.py COPY ./docker/conf /conf diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 1b8d4f9989..24ee9bfcf7 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/env python # # This file is licensed under the Affero General Public License (AGPL) version 3. # diff --git a/docker/editable.Dockerfile b/docker/editable.Dockerfile index b2aff9cb53..8d7a514298 100644 --- a/docker/editable.Dockerfile +++ b/docker/editable.Dockerfile @@ -6,11 +6,15 @@ ARG PYTHON_VERSION=3.10 ### -### Stage 0: generate requirements.txt +### Stage 0: build editable install ### # We hardcode the use of Debian trixie here because this could change upstream # and other Dockerfiles used for testing are expecting trixie. -FROM docker.io/library/python:${PYTHON_VERSION}-slim-trixie +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-trixie + +# This silences a warning as uv isn't able to do hardlinks between its cache +# (mounted as --mount=type=cache) and the target directory. +ENV UV_LINK_MODE=copy # Install Rust and other dependencies (stolen from normal Dockerfile) # install the OS build deps @@ -49,18 +53,19 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool # at runtime. COPY synapse /editable-src/synapse/ COPY rust /editable-src/rust/ -# ... and what we need to `pip install`. -COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/ +# ... and what we need to install. +COPY pyproject.toml uv.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/ -RUN pip install poetry -RUN poetry config virtualenvs.create false -RUN cd /editable-src && poetry install --extras all +# Install into the system Python (no virtualenv) using the lockfile +ENV UV_PROJECT_ENVIRONMENT=/usr/local +RUN --mount=type=cache,target=/root/.cache/uv \ + cd /editable-src && uv sync --extra all --frozen # Make copies of useful things for inspection: # - the Rust module (must be copied to the editable source tree before startup) -# - poetry.lock is useful for checking if dependencies have changed. +# - uv.lock is useful for checking if dependencies have changed. RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak -RUN cp /editable-src/poetry.lock /poetry.lock.bak +RUN cp /editable-src/uv.lock /uv.lock.bak ### Extra setup from original Dockerfile diff --git a/docker/start.py b/docker/start.py index 19f1ab5075..163102e463 100755 --- a/docker/start.py +++ b/docker/start.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/env python import codecs import glob diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index cca87d42a9..b83361de4b 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -28,8 +28,8 @@ # # ./complement.sh -run "TestOutboundFederation(Profile|Send)" # -# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `poetry export` -# to not emit any hashes when building the Docker image. This then means that +# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `uv sync` +# to not verify hashes when building the Docker image. This then means that # you can use 'unverifiable' sources such as git repositories as dependencies. # Exit if a line returns a non-zero exit code @@ -192,13 +192,13 @@ main() { elif $CONTAINER_RUNTIME inspect "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" &>/dev/null; then # complement-synapse-editable already exists: see if we can still use it: # - The Rust module must still be importable; it will fail to import if the Rust source has changed. - # - The Poetry lock file must be the same (otherwise we assume dependencies have changed) + # - The uv lock file must be the same (otherwise we assume dependencies have changed) # First set up the module in the right place for an editable installation. $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so if ($CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'python' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" -c 'import synapse.synapse_rust' \ - && $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" --brief /editable-src/poetry.lock /poetry.lock.bak); then + && $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" --brief /editable-src/uv.lock /uv.lock.bak); then skip_docker_build=1 else echo "Editable Synapse image is stale. Will rebuild." @@ -234,7 +234,7 @@ main() { # up with our current reality. rm -rf matrix_synapse.egg-info/ # Figure out the Synapse version string in our current checkout - synapse_version_string="$(poetry run python -c 'from synapse.util import SYNAPSE_VERSION; print(SYNAPSE_VERSION)')" + synapse_version_string="$(python -c 'from synapse.util import SYNAPSE_VERSION; print(SYNAPSE_VERSION)')" # Build the base Synapse image from the local checkout echo_if_github "::group::Build Docker image: matrixdotorg/synapse" @@ -242,7 +242,7 @@ main() { -t "$SYNAPSE_IMAGE_PATH" \ --build-arg SYNAPSE_VERSION_STRING="$synapse_version_string" \ --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ - --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ + --build-arg TEST_ONLY_IGNORE_LOCKFILE \ -f "docker/Dockerfile" . echo_if_github "::endgroup::"