From 10aa6aa4f554d51a976ebaa4146e9c00af829439 Mon Sep 17 00:00:00 2001 From: Dan Theisen Date: Sun, 30 Aug 2026 15:10:33 +0000 Subject: [PATCH] Add dev container for reproducible builds Preconfigured VS Code dev container with the Zephyr SDK, toolchains, west and vendor blobs matched to CI, so a build can be reproduced without installing prerequisites on the host. Co-Authored-By: Claude Opus 5 (1M context) --- .devcontainer/Dockerfile | 108 +++++++++++++++++++++++++++ .devcontainer/README.md | 104 ++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 77 +++++++++++++++++++ .devcontainer/post-create.sh | 128 ++++++++++++++++++++++++++++++++ README.md | 2 + 5 files changed, 419 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/post-create.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..984a80e --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,108 @@ +# ZephCore development container. +# +# Mirrors what .github/workflows/build.yml installs on the CI runner, so a build +# in here matches a build in CI: Zephyr SDK 1.0.1, the same four toolchains, the +# same Python version, and the arm/aarch64 Linux cross compilers that the +# native_sim SBC presets need. +# +# Ubuntu 24.04 is used because it ships Python 3.12 (CI's python_version) and +# CMake 3.28 (Zephyr 4.4 needs >= 3.20.5). +ARG BASE_VARIANT=ubuntu-24.04 +FROM mcr.microsoft.com/devcontainers/base:${BASE_VARIANT} + +# Keep these in sync with `zephyr_sdk` and `toolchains` in +# .github/workflows/build.yml. +ARG ZSDK_VERSION=1.0.1 +ARG ZSDK_TOOLCHAINS="arm-zephyr-eabi xtensa-espressif_esp32s3_zephyr-elf xtensa-espressif_esp32_zephyr-elf riscv64-zephyr-elf" + +ENV DEBIAN_FRONTEND=noninteractive + +# Zephyr's documented host dependencies, plus: +# - gcc/g++-arm-linux-gnueabihf + aarch64-linux-gnu: the `build.sh linux` +# native_sim presets are cross-compiled for the target SBC. See +# docs/LINUX_NATIVE.md. +# - ccache: Zephyr picks it up automatically as a compiler launcher. +# - dfu-util / usbutils / picocom: flashing and serial console on Linux hosts. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ccache \ + cmake \ + device-tree-compiler \ + dfu-util \ + file \ + gdb-multiarch \ + gperf \ + libmagic1 \ + make \ + ninja-build \ + picocom \ + python3 \ + python3-dev \ + python3-venv \ + udev \ + unzip \ + usbutils \ + wget \ + xz-utils \ + zip \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf \ + gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \ + && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# Zephyr SDK +# +# The "minimal" bundle is the SDK scaffolding + host tools only; setup.sh then +# pulls just the toolchains named in ZSDK_TOOLCHAINS. Installing all ~35 GNU +# toolchains would add tens of GB for architectures ZephCore never targets. +# +# The `cmake -P zephyr_sdk_export.cmake` step registers the SDK in the *vscode* +# user's CMake package registry rather than root's -- that script writes to +# $HOME/.cmake unconditionally. ZEPHYR_SDK_INSTALL_DIR below already covers +# find_package(Zephyr-sdk); the registry entry is belt and braces for tooling +# that shells out with a different environment. +# --------------------------------------------------------------------------- +ENV ZEPHYR_SDK_INSTALL_DIR=/opt/zephyr-sdk-${ZSDK_VERSION} +ENV ZEPHYR_TOOLCHAIN_VARIANT=zephyr + +RUN set -eux; \ + arch="$(uname -m)"; \ + base="https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZSDK_VERSION}"; \ + bundle="zephyr-sdk-${ZSDK_VERSION}_linux-${arch}_minimal.tar.xz"; \ + cd /opt; \ + wget -q --progress=dot:giga -O "${bundle}" "${base}/${bundle}"; \ + wget -q -O sha256.sum "${base}/sha256.sum"; \ + grep " ${bundle}\$" sha256.sum | sha256sum -c -; \ + rm -f sha256.sum; \ + tar xf "${bundle}"; \ + rm -f "${bundle}"; \ + cd "zephyr-sdk-${ZSDK_VERSION}"; \ + set --; \ + for t in ${ZSDK_TOOLCHAINS}; do set -- "$@" -t "$t"; done; \ + ./setup.sh "$@" -h; \ + HOME=/home/vscode cmake -P cmake/zephyr_sdk_export.cmake; \ + chown -R vscode:vscode /home/vscode/.cmake + +# --------------------------------------------------------------------------- +# Python environment +# +# Lives in /opt (image layer, not the bind-mounted workspace) so it survives a +# host-side `git clean` and stays fast on Windows/macOS bind mounts. Ubuntu 24.04 +# marks the system interpreter PEP 668 externally-managed, so a venv is required +# rather than merely tidy. +# +# Only west and the flash/DFU tools ZephCore's docs call for go in here. The bulk +# of Zephyr's Python requirements are installed by post-create.sh via +# `west packages pip --install`, which needs a populated workspace to enumerate +# the per-module requirement files (that is where esptool comes from). +# --------------------------------------------------------------------------- +ENV VIRTUAL_ENV=/opt/zephyr-venv +ENV PATH=${VIRTUAL_ENV}/bin:${PATH} + +RUN set -eux; \ + python3 -m venv "${VIRTUAL_ENV}"; \ + pip install --no-cache-dir --upgrade pip setuptools wheel; \ + pip install --no-cache-dir west pyocd adafruit-nrfutil; \ + chown -R vscode:vscode "${VIRTUAL_ENV}" + +USER vscode diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000..58cb9cd --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,104 @@ +# ZephCore dev container + +A ready-to-build Zephyr workspace: **Zephyr SDK 1.0.1**, **west**, and the four +toolchains ZephCore's boards need. It reproduces the environment from +[.github/workflows/build.yml](../.github/workflows/build.yml), so a local build +matches CI. + +## Getting started + +Open the repo in VS Code and run **Dev Containers: Reopen in Container**, or from +a terminal: + +```bash +devcontainer up --workspace-folder . +``` + +First creation takes a while — it downloads the SDK toolchains (~1 GB), clones +Zephyr and its modules, and fetches the vendor BLE blobs. Subsequent starts are +instant. + +When it finishes you can build immediately, from the workspace root: + +```bash +west build -b rak4631 zephcore --pristine +west build -b xiao_esp32s3/esp32s3/procpu zephcore --pristine --sysbuild +bash build.sh nrf +``` + +Artifacts land in `build/zephyr/` and, for `build.sh`, `firmware/`. + +## What's inside + +| | | +|---|---| +| Base | `mcr.microsoft.com/devcontainers/base:ubuntu-24.04` (Python 3.12, CMake 3.28) | +| Zephyr SDK | 1.0.1 at `/opt/zephyr-sdk-1.0.1` | +| Toolchains | `arm-zephyr-eabi`, `xtensa-espressif_esp32_zephyr-elf`, `xtensa-espressif_esp32s3_zephyr-elf`, `riscv64-zephyr-elf` | +| Cross compilers | `arm-linux-gnueabihf`, `aarch64-linux-gnu` (for the `build.sh linux` native_sim presets) | +| Python | venv at `/opt/zephyr-venv`, first on `PATH` — holds `west`, `esptool`, `pyocd`, `adafruit-nrfutil`, and Zephyr's own requirements | +| Vendor blobs | `hal_espressif` and `hal_silabs` BLE controllers, fetched on create | + +Only the toolchains ZephCore actually targets are installed. Adding a board on a +new architecture means adding its toolchain to `ZSDK_TOOLCHAINS` in +[devcontainer.json](devcontainer.json) (and to `toolchains` in the CI workflow) +and rebuilding the container. + +## Workspace layout + +The west topdir is the workspace root — the folder that *contains* `zephcore/` — +matching what `west init -l zephcore` and `build.sh` assume: + +``` +/workspaces/ZephCore/ +├── zephcore/ the application (this git repo) +├── zephyr/ \ +├── modules/ | west-managed, in named volumes +├── bootloader/ | +├── tools/ / +├── build/ on the host bind mount +└── firmware/ on the host bind mount +``` + +`zephyr/`, `modules/`, `bootloader/` and `tools/` live in Docker named volumes +rather than on the host bind mount. They are tens of thousands of files nobody +edits from the host, and container-native storage is several times faster for +them on Windows and macOS. `build/` and `firmware/` stay on the bind mount so you +can reach the binaries from the host to flash them. + +To reclaim that space after deleting the container: `docker volume prune`, or +`docker volume ls | grep zephcore-` to remove them individually. + +## Setup knobs + +[post-create.sh](post-create.sh) is idempotent — re-run it any time with +`bash .devcontainer/post-create.sh`. Two environment variables change what it +does: + +- `ZEPHCORE_WEST_FULL=1` — clone Zephyr and modules with full history. The + default uses `west update --narrow -o=--depth=1`, which cuts the checkout from + ~5 GB to well under 1 GB. The trade-off is that `zephyr/` and the modules carry + no history and no tags, so `git log`/`git describe`/`git bisect` in them don't + work. Builds are unaffected — ZephCore's version string comes from + `ZEPHCORE_FIRMWARE_VERSION` in `zephcore/CMakeLists.txt`. Set this if you need + to dig through upstream history. +- `ZEPHCORE_SKIP_BLOBS=1` — skip the vendor BLE blobs. Fine for nRF-only work; + ESP32 and MG24 *companion* builds will fail to link without them. + +## Flashing + +`west flash` from inside the container only works on a **Linux host** — Docker +Desktop on Windows and macOS cannot pass USB devices through. On Linux, uncomment +the `runArgs` line in [devcontainer.json](devcontainer.json). + +Everywhere else, build in the container and flash from the host: the `.uf2`, +`.zip`, `.hex` and `.bin` files in `build/zephyr/` and `firmware/` are on the +bind mount and visible to host tools. Drag-and-drop UF2 flashing needs nothing +installed on the host at all. + +## Note on `.vscode/settings.json` + +If you have a local (gitignored) `.vscode/settings.json` with a Windows path in +`cmake.sourceDirectory`, it overrides the container setting and points CMake +Tools at a path that does not exist inside the container. Either delete that file +or change the value to `${workspaceFolder}/zephcore`. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..9c8f6f4 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,77 @@ +{ + "name": "ZephCore (Zephyr SDK 1.0.1)", + + "build": { + "dockerfile": "Dockerfile", + "args": { + // Keep in sync with `zephyr_sdk` / `toolchains` in .github/workflows/build.yml. + // arm-zephyr-eabi -> nRF52/nRF54L, MG24, STM32WL + // xtensa-espressif_esp32* -> classic ESP32 and ESP32-S3 + // riscv64-zephyr-elf -> ESP32-C3 / C6 + "ZSDK_VERSION": "1.0.1", + "ZSDK_TOOLCHAINS": "arm-zephyr-eabi xtensa-espressif_esp32s3_zephyr-elf xtensa-espressif_esp32_zephyr-elf riscv64-zephyr-elf" + } + }, + + "remoteUser": "vscode", + + // The west topdir is the folder containing zephcore/, i.e. the workspace root + // itself -- that is what `west init -l zephcore` and build.sh both assume. + // west then clones into zephyr/, modules/, bootloader/ and tools/ here, all of + // which .gitignore already excludes. + // + // Those four live in named volumes rather than on the host bind mount: they are + // ~50k files that no one edits from the host, and container-native storage is + // several times faster for them on Windows and macOS. build/ and firmware/ are + // deliberately left on the bind mount so the host can flash what you build. + "mounts": [ + "source=zephcore-zephyr-${devcontainerId},target=${containerWorkspaceFolder}/zephyr,type=volume", + "source=zephcore-modules-${devcontainerId},target=${containerWorkspaceFolder}/modules,type=volume", + "source=zephcore-bootloader-${devcontainerId},target=${containerWorkspaceFolder}/bootloader,type=volume", + "source=zephcore-tools-${devcontainerId},target=${containerWorkspaceFolder}/tools,type=volume", + "source=zephcore-ccache-${devcontainerId},target=/home/vscode/.cache/ccache,type=volume" + ], + + // Flashing over USB from inside the container only works on a Linux host -- + // Docker Desktop on Windows/macOS cannot pass through USB devices. On Linux, + // uncomment to expose probes and USB serial: + // "runArgs": ["--device=/dev/bus/usb", "--volume=/dev/serial:/dev/serial"], + + "postCreateCommand": "bash .devcontainer/post-create.sh", + "waitFor": "postCreateCommand", + + "hostRequirements": { + "cpus": 4, + "storage": "64gb" + }, + + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "twxs.cmake", + "ms-python.python", + "marus25.cortex-debug", + "trond-snekvik.devicetree", + "trond-snekvik.kconfig-lang" + ], + "settings": { + // Zephyr apps are configured by west, not by CMake Tools. Letting the + // extension auto-configure zephcore/ on open just produces a failed + // configure and a misleading error. + "cmake.configureOnOpen": false, + "cmake.sourceDirectory": "${workspaceFolder}/zephcore", + "C_Cpp.default.compileCommands": "${workspaceFolder}/build/compile_commands.json", + "devicetree.zephyr": "${workspaceFolder}/zephyr", + "python.defaultInterpreterPath": "/opt/zephyr-venv/bin/python", + "files.associations": { + "*.overlay": "dts", + "*.dtsi": "dts", + "*.conf": "kconfig", + "Kconfig*": "kconfig" + } + } + } + } +} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 0000000..0efadcb --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# One-time setup for the ZephCore dev container. +# +# Initialises the west workspace at the repo root (topdir = the folder that +# *contains* zephcore/, exactly as README.md and build.sh assume), pulls Zephyr +# and its modules, installs the Python requirements those modules declare, and +# fetches the vendor BLE controller blobs that ESP32 and MG24 builds link +# against. +# +# Safe to re-run: every step is idempotent. Useful knobs: +# ZEPHCORE_WEST_FULL=1 full (non-shallow) clones -- needed if you plan to +# bisect or `git log` inside zephyr/ or modules/ +# ZEPHCORE_SKIP_BLOBS=1 skip the ~100 MB of vendor blobs (nRF-only work) +set -euo pipefail + +# postCreateCommand runs in the workspace folder, which is the west topdir. +TOPDIR="$(pwd)" + +step() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; } +note() { printf ' %s\n' "$*"; } + +# --------------------------------------------------------------------------- +# Named-volume mount points arrive root-owned and empty. Hand them to the +# container user before west tries to clone into them. +# --------------------------------------------------------------------------- +step "Preparing mount points" +for d in zephyr modules bootloader tools; do + if [ -d "$TOPDIR/$d" ] && [ ! -w "$TOPDIR/$d" ]; then + sudo chown -R "$(id -u):$(id -g)" "$TOPDIR/$d" + note "took ownership of $d/" + fi +done +if [ -d "$HOME/.cache/ccache" ] && [ ! -w "$HOME/.cache/ccache" ]; then + sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache/ccache" + note "took ownership of ~/.cache/ccache" +fi + +# The workspace is a bind mount from the host, so its files can carry a uid that +# does not match the container user. Without this, git refuses to operate and +# Zephyr's build-time `git describe` calls fail. +git config --global --get-all safe.directory | grep -qx '\*' \ + || git config --global --add safe.directory '*' + +# --------------------------------------------------------------------------- +# West workspace +# --------------------------------------------------------------------------- +if [ -d "$TOPDIR/.west" ]; then + step "West workspace already initialised" +else + step "Initialising west workspace (topdir: $TOPDIR)" + west init -l zephcore +fi + +step "Fetching Zephyr and modules (this is the slow one on a cold start)" +if [ "${ZEPHCORE_WEST_FULL:-0}" = "1" ]; then + west update +else + # --narrow fetches only the manifest-pinned revision instead of every branch + # and tag; --depth=1 makes each of those a shallow clone. Together they turn + # a ~5 GB checkout into well under 1 GB. The trade-off is that zephyr/ and the + # modules carry no history and no tags, so anything reading them with + # `git describe` or `git log` falls back to the VERSION file. ZephCore's own + # version string comes from ZEPHCORE_FIRMWARE_VERSION in + # zephcore/CMakeLists.txt and is unaffected. Set ZEPHCORE_WEST_FULL=1 for + # full history. + west update --narrow -o=--depth=1 || { + note "shallow update failed -- retrying with full clones" + west update + } +fi + +step "Registering Zephyr CMake package" +west zephyr-export + +# --------------------------------------------------------------------------- +# Python requirements +# +# `west packages pip --install` walks zephyr/scripts/requirements.txt plus every +# module that declares requirement files in its module.yml. hal_espressif is one +# of them, and that is where build.sh's `python -m esptool merge-bin` comes from. +# --------------------------------------------------------------------------- +step "Installing Zephyr Python requirements" +if ! west packages pip --install; then + note "west packages unavailable -- falling back to zephyr/scripts/requirements.txt" + pip install -r "$TOPDIR/zephyr/scripts/requirements.txt" +fi + +step "Installing ZephCore Python requirements" +pip install -r "$TOPDIR/requirements.txt" + +# --------------------------------------------------------------------------- +# Vendor blobs +# +# hal_espressif: BLE controller libraries -- ESP32 companion builds will not link +# without them. +# hal_silabs: same story for the EFR32MG24 (xiao_mg24) companion. +# --------------------------------------------------------------------------- +if [ "${ZEPHCORE_SKIP_BLOBS:-0}" = "1" ]; then + step "Skipping vendor blobs (ZEPHCORE_SKIP_BLOBS=1)" + note "ESP32 and MG24 companion builds will fail to link until you run:" + note " west blobs fetch hal_espressif && west blobs fetch hal_silabs" +else + step "Fetching vendor BLE controller blobs" + west blobs fetch hal_espressif + west blobs fetch hal_silabs +fi + +# --------------------------------------------------------------------------- +step "Environment" +printf ' %-24s %s\n' \ + "Zephyr SDK" "$(cat "${ZEPHYR_SDK_INSTALL_DIR}/sdk_version" 2>/dev/null || echo '??') (${ZEPHYR_SDK_INSTALL_DIR})" \ + "west" "$(west --version 2>/dev/null | head -1)" \ + "Zephyr" "$(sed -n 's/^VERSION_MAJOR = //p' "$TOPDIR/zephyr/VERSION" 2>/dev/null).$(sed -n 's/^VERSION_MINOR = //p' "$TOPDIR/zephyr/VERSION" 2>/dev/null).$(sed -n 's/^PATCHLEVEL = //p' "$TOPDIR/zephyr/VERSION" 2>/dev/null)" \ + "Python" "$(python --version 2>&1) ($(command -v python))" \ + "Toolchains" "$(ls "${ZEPHYR_SDK_INSTALL_DIR}/gnu" 2>/dev/null | tr '\n' ' ')" + +cat <<'EOF' + + Ready. Build from this directory (the west topdir), e.g.: + + west build -b rak4631 zephcore --pristine + west build -b rak4631 zephcore --pristine -- -DEXTRA_CONF_FILE="boards/common/repeater.conf" + bash build.sh nrf + + Artifacts land in build/zephyr/ and (for build.sh) firmware/, both of which + are on the host bind mount -- flash them with your host's tools. +EOF diff --git a/README.md b/README.md index fd6c429..c6591c1 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Prerequisites: [Zephyr SDK >=1.0.1 (!)](https://docs.zephyrproject.org/latest/de Optional: [adafruit-nrfutil](https://github.com/adafruit/Adafruit_nRF52_nrfutil) to allow DFU zip generation for OTA updates on nRF52. +Or skip the prerequisites entirely and use the [dev container](.devcontainer/README.md) -- SDK, toolchains, west and the vendor blobs, preconfigured to match CI. Open the repo in VS Code and run **Dev Containers: Reopen in Container**. + ```bash # Initialize workspace (first time only) cd