mirror of
https://github.com/element-hq/synapse.git
synced 2026-08-28 13:44:24 +00:00
Update Docker files and complement script to use uv instead of poetry
Replace poetry-based dependency management with uv across Docker builds and the complement test script: - Dockerfile: Remove the `poetry export` stage entirely. Use `uv sync` directly in the builder stage following uv best practices (first sync deps with --no-install-project, then copy source and do full sync). Install into a virtualenv and copy it to the runtime image. - editable.Dockerfile: Replace `pip install poetry` + `poetry install` with `uv sync` using the uv base image. Use UV_PROJECT_ENVIRONMENT to install into system Python. - complement.sh: Update lockfile diff check from poetry.lock to uv.lock, remove `poetry run` wrapper, rename TEST_ONLY_IGNORE_POETRY_LOCKFILE to TEST_ONLY_IGNORE_LOCKFILE. - start.py / configure_workers_and_start.py: Change shebang from /usr/local/bin/python to /usr/bin/env python for venv compatibility. - latest_deps.yml: Rename build arg and lockfile references. Part of #19567. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Quentin Gliech
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
72f6464a50
commit
2d450d415f
@@ -160,8 +160,8 @@ jobs:
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
|
||||
- name: Ensure sytest runs `pip install`
|
||||
# Delete the lockfile so sytest will `pip install` rather than `poetry install`
|
||||
run: rm /src/poetry.lock
|
||||
# Delete the lockfile so sytest will `pip install` rather than `uv sync`
|
||||
run: rm /src/uv.lock
|
||||
working-directory: /src
|
||||
- name: Prepare test blacklist
|
||||
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
|
||||
@@ -229,7 +229,7 @@ jobs:
|
||||
env:
|
||||
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
|
||||
WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
|
||||
TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1
|
||||
TEST_ONLY_IGNORE_LOCKFILE: 1
|
||||
|
||||
- name: Formatted Complement test logs
|
||||
# Always run this step if we attempted to run the Complement tests.
|
||||
@@ -253,7 +253,7 @@ jobs:
|
||||
env:
|
||||
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
|
||||
WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
|
||||
TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1
|
||||
TEST_ONLY_IGNORE_LOCKFILE: 1
|
||||
|
||||
- name: Formatted in-repo Complement test logs
|
||||
# Always run this step if we attempted to run the Complement tests.
|
||||
|
||||
+35
-66
@@ -15,55 +15,11 @@
|
||||
# DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
|
||||
#
|
||||
|
||||
# Irritatingly, there is no blessed guide on how to distribute an application with its
|
||||
# poetry-managed environment in a docker image. We have opted for
|
||||
# `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in
|
||||
# in `poetry export` in the past.
|
||||
|
||||
ARG DEBIAN_VERSION=trixie
|
||||
ARG PYTHON_VERSION=3.13
|
||||
ARG POETRY_VERSION=2.2.1
|
||||
|
||||
###
|
||||
### Stage 0: generate requirements.txt
|
||||
###
|
||||
### This stage is platform-agnostic, so we can use the build platform in case of cross-compilation.
|
||||
###
|
||||
FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS requirements
|
||||
|
||||
WORKDIR /synapse
|
||||
|
||||
# Copy just what we need to run `poetry export`...
|
||||
COPY pyproject.toml poetry.lock /synapse/
|
||||
|
||||
|
||||
# If specified, we won't verify the hashes of dependencies.
|
||||
# This is only needed if the hashes of dependencies cannot be checked for some
|
||||
# reason, such as when a git repository is used directly as a dependency.
|
||||
ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
|
||||
|
||||
# If specified, we won't use the Poetry lockfile.
|
||||
# Instead, we'll just install what a regular `pip install` would from PyPI.
|
||||
ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
|
||||
|
||||
# This silences a warning as uv isn't able to do hardlinks between its cache
|
||||
# (mounted as --mount=type=cache) and the target directory.
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
# Export the dependencies, but only if we're actually going to use the Poetry lockfile.
|
||||
# Otherwise, just create an empty requirements file so that the Dockerfile can
|
||||
# proceed.
|
||||
ARG POETRY_VERSION
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
|
||||
uvx --with poetry-plugin-export==1.9.0 \
|
||||
poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
|
||||
else \
|
||||
touch /synapse/requirements.txt; \
|
||||
fi
|
||||
|
||||
###
|
||||
### Stage 1: builder
|
||||
### Stage 0: builder
|
||||
###
|
||||
FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-${DEBIAN_VERSION} AS builder
|
||||
|
||||
@@ -84,39 +40,51 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
|
||||
ARG CARGO_NET_GIT_FETCH_WITH_CLI=false
|
||||
ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI
|
||||
|
||||
WORKDIR /synapse
|
||||
|
||||
# If specified, we won't verify the hashes of dependencies.
|
||||
# This is only needed if the hashes of dependencies cannot be checked for some
|
||||
# reason, such as when a git repository is used directly as a dependency.
|
||||
ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
|
||||
|
||||
# If specified, we won't use the uv lockfile.
|
||||
# Instead, we'll just install what a regular `pip install` would from PyPI.
|
||||
ARG TEST_ONLY_IGNORE_LOCKFILE
|
||||
|
||||
# To speed up rebuilds, install all of the dependencies before we copy over
|
||||
# the whole synapse project, so that this layer in the Docker cache can be
|
||||
# used while you develop on the source
|
||||
# used while you develop on the source.
|
||||
#
|
||||
# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
|
||||
COPY --from=requirements /synapse/requirements.txt /synapse/
|
||||
# We use `uv sync --no-install-project` to install only the dependencies
|
||||
# from the lockfile, without installing the project itself.
|
||||
COPY pyproject.toml uv.lock /synapse/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --prefix="/install" --no-deps -r /synapse/requirements.txt
|
||||
if [ -z "$TEST_ONLY_IGNORE_LOCKFILE" ]; then \
|
||||
uv sync --extra all --frozen --no-install-project --no-dev ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--no-verify-hashes}; \
|
||||
fi
|
||||
|
||||
# Copy over the rest of the synapse source code.
|
||||
COPY synapse /synapse/synapse/
|
||||
COPY rust /synapse/rust/
|
||||
# ... and what we need to `pip install`.
|
||||
COPY pyproject.toml README.rst build_rust.py Cargo.toml Cargo.lock /synapse/
|
||||
|
||||
# Repeat of earlier build argument declaration, as this is a new build stage.
|
||||
ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
|
||||
# ... and what we need to build and install the project.
|
||||
COPY README.rst build_rust.py Cargo.toml Cargo.lock /synapse/
|
||||
|
||||
# Install the synapse package itself.
|
||||
# If we have populated requirements.txt, we don't install any dependencies
|
||||
# as we should already have those from the previous `pip install` step.
|
||||
# If we are using the lockfile, we use `uv sync` which will reuse the
|
||||
# already-installed dependencies from the previous step.
|
||||
# Otherwise, we fall back to `uv pip install` which resolves freely from PyPI.
|
||||
RUN \
|
||||
--mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=cache,target=/synapse/target,sharing=locked \
|
||||
--mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \
|
||||
if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
|
||||
uv pip install --prefix="/install" --no-deps /synapse[all]; \
|
||||
if [ -z "$TEST_ONLY_IGNORE_LOCKFILE" ]; then \
|
||||
uv sync --extra all --frozen --no-dev --no-editable; \
|
||||
else \
|
||||
uv pip install --prefix="/install" /synapse[all]; \
|
||||
uv pip install --python=.venv /synapse[all]; \
|
||||
fi
|
||||
|
||||
###
|
||||
### Stage 2: runtime dependencies download for ARM64 and AMD64
|
||||
### Stage 1: runtime dependencies download for ARM64 and AMD64
|
||||
###
|
||||
FROM --platform=$BUILDPLATFORM docker.io/library/debian:${DEBIAN_VERSION} AS runtime-deps
|
||||
|
||||
@@ -164,7 +132,7 @@ RUN \
|
||||
|
||||
|
||||
###
|
||||
### Stage 3: runtime
|
||||
### Stage 2: runtime
|
||||
###
|
||||
|
||||
FROM docker.io/library/python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION}
|
||||
@@ -188,11 +156,12 @@ COPY --from=runtime-deps /install-${TARGETARCH}/etc /etc
|
||||
COPY --from=runtime-deps /install-${TARGETARCH}/usr /usr
|
||||
COPY --from=runtime-deps /install-${TARGETARCH}/var /var
|
||||
|
||||
# Copy the installed python packages from the builder stage.
|
||||
#
|
||||
# uv will generate a `.lock` file when installing packages, which we don't want
|
||||
# to copy to the final image.
|
||||
COPY --from=builder --exclude=.lock /install /usr/local
|
||||
# Copy the virtual environment from the builder stage.
|
||||
COPY --from=builder /synapse/.venv /synapse/.venv
|
||||
|
||||
# Put the virtualenv on PATH
|
||||
ENV PATH="/synapse/.venv/bin:$PATH"
|
||||
|
||||
COPY ./docker/start.py /start.py
|
||||
COPY ./docker/conf /conf
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/local/bin/python
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
||||
#
|
||||
|
||||
@@ -6,11 +6,15 @@
|
||||
ARG PYTHON_VERSION=3.10
|
||||
|
||||
###
|
||||
### Stage 0: generate requirements.txt
|
||||
### Stage 0: build editable install
|
||||
###
|
||||
# We hardcode the use of Debian trixie here because this could change upstream
|
||||
# and other Dockerfiles used for testing are expecting trixie.
|
||||
FROM docker.io/library/python:${PYTHON_VERSION}-slim-trixie
|
||||
FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-trixie
|
||||
|
||||
# This silences a warning as uv isn't able to do hardlinks between its cache
|
||||
# (mounted as --mount=type=cache) and the target directory.
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
# Install Rust and other dependencies (stolen from normal Dockerfile)
|
||||
# install the OS build deps
|
||||
@@ -49,18 +53,19 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
|
||||
# at runtime.
|
||||
COPY synapse /editable-src/synapse/
|
||||
COPY rust /editable-src/rust/
|
||||
# ... and what we need to `pip install`.
|
||||
COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/
|
||||
# ... and what we need to install.
|
||||
COPY pyproject.toml uv.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/
|
||||
|
||||
RUN pip install poetry
|
||||
RUN poetry config virtualenvs.create false
|
||||
RUN cd /editable-src && poetry install --extras all
|
||||
# Install into the system Python (no virtualenv) using the lockfile
|
||||
ENV UV_PROJECT_ENVIRONMENT=/usr/local
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
cd /editable-src && uv sync --extra all --frozen
|
||||
|
||||
# Make copies of useful things for inspection:
|
||||
# - the Rust module (must be copied to the editable source tree before startup)
|
||||
# - poetry.lock is useful for checking if dependencies have changed.
|
||||
# - uv.lock is useful for checking if dependencies have changed.
|
||||
RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak
|
||||
RUN cp /editable-src/poetry.lock /poetry.lock.bak
|
||||
RUN cp /editable-src/uv.lock /uv.lock.bak
|
||||
|
||||
|
||||
### Extra setup from original Dockerfile
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
#!/usr/local/bin/python
|
||||
#!/usr/bin/env python
|
||||
|
||||
import codecs
|
||||
import glob
|
||||
|
||||
@@ -28,8 +28,8 @@
|
||||
#
|
||||
# ./complement.sh -run "TestOutboundFederation(Profile|Send)"
|
||||
#
|
||||
# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `poetry export`
|
||||
# to not emit any hashes when building the Docker image. This then means that
|
||||
# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `uv sync`
|
||||
# to not verify hashes when building the Docker image. This then means that
|
||||
# you can use 'unverifiable' sources such as git repositories as dependencies.
|
||||
|
||||
# Exit if a line returns a non-zero exit code
|
||||
@@ -192,13 +192,13 @@ main() {
|
||||
elif $CONTAINER_RUNTIME inspect "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" &>/dev/null; then
|
||||
# complement-synapse-editable already exists: see if we can still use it:
|
||||
# - The Rust module must still be importable; it will fail to import if the Rust source has changed.
|
||||
# - The Poetry lock file must be the same (otherwise we assume dependencies have changed)
|
||||
# - The uv lock file must be the same (otherwise we assume dependencies have changed)
|
||||
|
||||
# First set up the module in the right place for an editable installation.
|
||||
$CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
|
||||
|
||||
if ($CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'python' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" -c 'import synapse.synapse_rust' \
|
||||
&& $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" --brief /editable-src/poetry.lock /poetry.lock.bak); then
|
||||
&& $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' "$COMPLEMENT_SYNAPSE_EDITABLE_IMAGE_PATH" --brief /editable-src/uv.lock /uv.lock.bak); then
|
||||
skip_docker_build=1
|
||||
else
|
||||
echo "Editable Synapse image is stale. Will rebuild."
|
||||
@@ -234,7 +234,7 @@ main() {
|
||||
# up with our current reality.
|
||||
rm -rf matrix_synapse.egg-info/
|
||||
# Figure out the Synapse version string in our current checkout
|
||||
synapse_version_string="$(poetry run python -c 'from synapse.util import SYNAPSE_VERSION; print(SYNAPSE_VERSION)')"
|
||||
synapse_version_string="$(python -c 'from synapse.util import SYNAPSE_VERSION; print(SYNAPSE_VERSION)')"
|
||||
|
||||
# Build the base Synapse image from the local checkout
|
||||
echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
|
||||
@@ -242,7 +242,7 @@ main() {
|
||||
-t "$SYNAPSE_IMAGE_PATH" \
|
||||
--build-arg SYNAPSE_VERSION_STRING="$synapse_version_string" \
|
||||
--build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
|
||||
--build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
|
||||
--build-arg TEST_ONLY_IGNORE_LOCKFILE \
|
||||
-f "docker/Dockerfile" .
|
||||
echo_if_github "::endgroup::"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user