From ea269d2761ccbd7552a6f0c1b044ba51ec9018ed Mon Sep 17 00:00:00 2001 From: Morgan Date: Thu, 20 Aug 2026 13:39:22 +0100 Subject: [PATCH] ci: Compile arm64 natively on the ubuntu-arm64 runner The arm64 build cross-compiled with the Rust target triple inside an amd64 container and copied the binary out, deliberately avoiding Docker emulation because arm64 emulation is unreliable on some BuildKit instances. A dedicated Ubuntu VM on an M4 Mac Mini is now available under the ubuntu-arm64 label. Each matrix entry carries its own runner label and a native flag, so builds can be moved between runners or between build modes without changing the workflow structure. Where native is set, the build-native action compiles on the host and assembles the artefact tree, leaving Docker to package it. It builds into a target directory outside the per-run workspace so cargo can reuse artefacts between runs. The Dockerfile's ARTIFACTS argument selects whether artefacts come from the builder stage or from out/ in the build context; prepper and the final image are unchanged. Libraries are resolved where the binary was linked, so arm64 images carry Ubuntu's rather than Debian's. Architecture flags are split into TARGET_MARCH, which feeds -march for the C and C++ compilers, and TARGET_RUSTFLAGS, which is appended to RUSTFLAGS. amd64 passes haswell to both as before; arm64 passes +crc, which rustc does not enable by default. TARGET_CPU still drives the tag and artefact suffix. --- .forgejo/actions/build-native/action.yml | 149 +++++++++++++++++++++++ .forgejo/workflows/release-image.yml | 40 +++++- docker/Dockerfile | 31 +++-- 3 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 .forgejo/actions/build-native/action.yml diff --git a/.forgejo/actions/build-native/action.yml b/.forgejo/actions/build-native/action.yml new file mode 100644 index 000000000..392c8a59f --- /dev/null +++ b/.forgejo/actions/build-native/action.yml @@ -0,0 +1,149 @@ +name: build-native +description: | + Compile the binary on the runner host rather than inside Docker, then assemble the + artefact tree the Docker packaging stages expect. + + Intended for runners whose architecture matches the target, where a persistent local + sccache is available. Produces out/{sbin,sbom,libs,libs-root} for consumption as a + named build context. + +inputs: + package: + description: Cargo package to build + required: false + default: conduwuit + profile: + description: Cargo build profile (release or release-max-perf) + required: true + features: + description: Comma-separated cargo features + required: false + default: default + march: + description: Value for -march passed to the C and C++ compilers + required: false + default: '' + rust-cpu-flags: + description: Architecture flags appended to RUSTFLAGS + required: false + default: '' + llvm-version: + description: LLVM major version the host toolchain must provide + required: false + default: '22' + target-dir: + description: Cargo target directory, defaults to one outside the per-run workspace + required: false + default: '' + out-dir: + description: Directory to assemble the artefact tree in + required: false + default: out + +outputs: + out-dir: + description: Directory containing sbin/, sbom/, libs/ and libs-root/ + value: ${{ steps.assemble.outputs.dir }} + +runs: + using: composite + steps: + - name: Add cargo binaries to PATH + shell: bash + run: echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "$GITHUB_PATH" + + - name: Verify host toolchain + shell: bash + run: | + set -euo pipefail + fail=0 + for tool in clang clang++ ld.lld cargo sccache cargo-sbom lddtree; do + if ! command -v "$tool" >/dev/null; then + echo "missing: $tool" + fail=1 + fi + done + [ "$fail" -eq 0 ] || { echo "host is not provisioned for native builds"; exit 1; } + + # Must match the container's ARG LLVM_VERSION or the two paths diverge. + got=$(clang --version | sed -n 's/.*clang version \([0-9]*\).*/\1/p' | head -1) + if [ "$got" != "${{ inputs.llvm-version }}" ]; then + echo "clang major version is $got, expected ${{ inputs.llvm-version }}" + exit 1 + fi + echo "clang $got, $(cargo --version), $(sccache --version)" + + - name: Configure build environment + shell: bash + run: | + set -euo pipefail + # Mirrors the values docker/Dockerfile writes into /etc/environment. + cflags="-flto" + if [ -n "${{ inputs.march }}" ]; then + cflags="$cflags -march=${{ inputs.march }}" + fi + { + echo "CARGO_INCREMENTAL=0" + echo "CC=clang" + echo "CXX=clang++" + echo "CFLAGS=$cflags" + echo "CXXFLAGS=$cflags" + echo "RUSTC_WRAPPER=sccache" + } >> "$GITHUB_ENV" + + # Outside the per-run workspace, so cargo can reuse artefacts between runs. + target_dir="${{ inputs.target-dir }}" + [ -n "$target_dir" ] || target_dir="$HOME/.cache/continuwuity-target" + echo "CARGO_TARGET_DIR=$target_dir" >> "$GITHUB_ENV" + + # A fat LTO link runs for minutes without compiling, which would otherwise + # idle the sccache server out and lose the statistics with it. + echo "SCCACHE_IDLE_TIMEOUT=1800" >> "$GITHUB_ENV" + + rustflags='-Clinker-plugin-lto -Clink-arg=-fuse-ld=lld' + if [ -n "${{ inputs.rust-cpu-flags }}" ]; then + rustflags="$rustflags ${{ inputs.rust-cpu-flags }}" + fi + echo "RUSTFLAGS=$rustflags" >> "$GITHUB_ENV" + + - name: Build + shell: bash + run: | + set -euo pipefail + sccache --zero-stats >/dev/null 2>&1 || true + cargo build --locked \ + --profile ${{ inputs.profile }} \ + --no-default-features --features ${{ inputs.features }} \ + -p ${{ inputs.package }} + + - name: Assemble artefact tree + id: assemble + shell: bash + run: | + set -euo pipefail + out="${{ inputs.out-dir }}" + rm -rf "$out" + mkdir -p "$out/sbin" "$out/sbom" "$out/libs" "$out/libs-root" + + profile_dir="${{ inputs.profile }}" + [ "$profile_dir" = "dev" ] && profile_dir=debug + target_dir=$(cargo metadata --no-deps --format-version 1 | jq -r '.target_directory') + + cp "$target_dir/$profile_dir/${{ inputs.package }}" "$out/sbin/${{ inputs.package }}" + cargo sbom --cargo-package ${{ inputs.package }} > "$out/sbom/${{ inputs.package }}.spdx.json" + + # Resolved here, not in the container: these are the libraries linked against. + for binary in "$out"/sbin/*; do + resolved=$(lddtree "$binary" 2>/dev/null) || continue + [ -n "$resolved" ] || continue + echo "$resolved" | awk '{print $(NF-0) " " $1}' | sort -u -k 1,1 | \ + awk -v out="$out" '{dest = ($2 ~ /^\//) ? out "/libs-root" $2 : out "/libs/" $2; print $1 " " dest}' | \ + while read -r src dest; do install -D "$src" "$dest"; done + done + + echo "dir=$out" >> "$GITHUB_OUTPUT" + find "$out" -type f -printf '%10s %P\n' | sort -k2 + + - name: Report cache statistics + shell: bash + run: sccache --show-stats || true diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 6673755d2..df34fffa2 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -30,7 +30,7 @@ env: jobs: build-release: name: "Build ${{ matrix.slug }} (release)" - runs-on: dind + runs-on: ${{ matrix.runner }} permissions: contents: read packages: write @@ -41,14 +41,29 @@ jobs: include: - platform: "linux/amd64" slug: "linux-amd64" + runner: "dind" + march: "" + rust_cpu_flags: "" - platform: "linux/arm64" slug: "linux-arm64" + runner: "ubuntu-arm64" + native: true + march: "armv8-a+crc" + rust_cpu_flags: "-C target-feature=+crc" steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false + - name: Compile on the runner host + if: matrix.native + uses: ./.forgejo/actions/build-native + with: + profile: "release" + features: "default" + march: ${{ matrix.march }} + rust-cpu-flags: ${{ matrix.rust_cpu_flags }} - name: Prepare Docker build environment id: prepare uses: ./.forgejo/actions/prepare-docker-build @@ -73,8 +88,11 @@ jobs: GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} CARGO_INCREMENTAL=${{ env.BUILDKIT_ENDPOINT != '' && '1' || '0' }} TARGET_CPU= + TARGET_MARCH=${{ matrix.march }} + TARGET_RUSTFLAGS=${{ matrix.rust_cpu_flags }} RUST_PROFILE=release CARGO_FEATURES=default + ARTIFACTS=${{ matrix.native && 'artifacts-prebuilt' || 'artifacts-builder' }} platforms: ${{ matrix.platform }} labels: ${{ steps.prepare.outputs.metadata_labels }} annotations: ${{ steps.prepare.outputs.metadata_annotations }} @@ -115,7 +133,7 @@ jobs: build-maxperf: name: "Build ${{ matrix.slug }} (max-perf)" - runs-on: dind + runs-on: ${{ matrix.runner }} needs: build-release permissions: contents: read @@ -128,15 +146,30 @@ jobs: - platform: "linux/amd64" slug: "linux-amd64" target_cpu: "haswell" + runner: "dind" + march: "haswell" + rust_cpu_flags: "-C target-cpu=haswell" - platform: "linux/arm64" slug: "linux-arm64" target_cpu: "" + runner: "ubuntu-arm64" + native: true + march: "armv8-a+crc" + rust_cpu_flags: "-C target-feature=+crc" steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false + - name: Compile on the runner host + if: matrix.native + uses: ./.forgejo/actions/build-native + with: + profile: "release-max-perf" + features: "default,release_max_log_level" + march: ${{ matrix.march }} + rust-cpu-flags: ${{ matrix.rust_cpu_flags }} - name: Prepare max-perf Docker build environment id: prepare uses: ./.forgejo/actions/prepare-docker-build @@ -161,8 +194,11 @@ jobs: GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} CARGO_INCREMENTAL=${{ env.BUILDKIT_ENDPOINT != '' && '1' || '0' }} TARGET_CPU=${{ matrix.target_cpu }} + TARGET_MARCH=${{ matrix.march }} + TARGET_RUSTFLAGS=${{ matrix.rust_cpu_flags }} RUST_PROFILE=release-max-perf CARGO_FEATURES=default,release_max_log_level + ARTIFACTS=${{ matrix.native && 'artifacts-prebuilt' || 'artifacts-builder' }} platforms: ${{ matrix.platform }} labels: ${{ steps.prepare.outputs.metadata_labels }} annotations: ${{ steps.prepare.outputs.metadata_annotations }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 31cc70212..eab4c115f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,10 @@ ARG RUST_VERSION=1 ARG DEBIAN_VERSION=trixie +# Source of compiled artefacts: the builder stage, or out/ from the build context +# when the runner compiled natively. +ARG ARTIFACTS=artifacts-builder + FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS base FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS toolchain @@ -118,15 +122,19 @@ EOF # Apply CPU-specific optimizations if TARGET_CPU is provided ARG TARGET_CPU +ARG TARGET_MARCH +ARG TARGET_RUSTFLAGS RUN <> /etc/environment - echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_CPU}'" >> /etc/environment - echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment + if [ -n "${TARGET_MARCH}" ]; then + echo "CFLAGS='${CFLAGS} -march=${TARGET_MARCH}'" >> /etc/environment + echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_MARCH}'" >> /etc/environment + fi + if [ -n "${TARGET_RUSTFLAGS}" ]; then + echo "RUSTFLAGS='${RUSTFLAGS} ${TARGET_RUSTFLAGS}'" >> /etc/environment fi EOF @@ -245,6 +253,13 @@ RUN <<'DEPS_EOF' find /out/libs* -type f 2>/dev/null | sort || echo "No libraries found" DEPS_EOF +FROM scratch AS artifacts-prebuilt +COPY out/ /out/ + +FROM builder AS artifacts-builder + +FROM ${ARTIFACTS} AS artifacts + FROM ubuntu:latest AS prepper # Create layer structure @@ -254,14 +269,14 @@ RUN mkdir -p /layer1/etc/ssl/certs \ # Copy SSL certs and root-path libraries to layer1 (ultra-stable) COPY --from=base /etc/ssl/certs /layer1/etc/ssl/certs -COPY --from=builder /out/libs-root/ /layer1/ +COPY --from=artifacts /out/libs-root/ /layer1/ # Copy application libraries to layer2 (semi-stable) -COPY --from=builder /out/libs/ /layer2/usr/lib/ +COPY --from=artifacts /out/libs/ /layer2/usr/lib/ # Copy binaries and SBOM to layer3 (volatile) -COPY --from=builder /out/sbin/ /layer3/sbin/ -COPY --from=builder /out/sbom/ /layer3/sbom/ +COPY --from=artifacts /out/sbin/ /layer3/sbin/ +COPY --from=artifacts /out/sbom/ /layer3/sbom/ # Fix permissions after copying RUN chmod -R 755 /layer1 /layer2 /layer3