diff --git a/docs/esp32_wifi_partition_migration.md b/docs/esp32_wifi_partition_migration.md index 15503187..1b1c821d 100644 --- a/docs/esp32_wifi_partition_migration.md +++ b/docs/esp32_wifi_partition_migration.md @@ -19,10 +19,36 @@ and its generated table-prefix test; the bridge core is otherwise shared. ## Repeatable ESP32 build recipe -From a clean commit, run the recipe in WSL/Linux. It builds the Full repeater -images first, then both small bridges for each selected board, then verifies -and publishes one ZIP per board. Only one PlatformIO process runs at a time; -`build.sh` can clean the shared `.pio/build` tree between Full targets. +From a clean commit, run the shell menu in WSL/Linux: + +```bash +sh scripts/build_esp32_partition_migration.sh +``` + +Choose one board, or **all qualified boards**. The script asks for the +firmware version, radio preset, and profile, then builds the Full repeater +images first, both small bridges for each selected board, and verifies each +ZIP. Only one PlatformIO process runs at a time; `build.sh` can clean the +shared `.pio/build` tree between Full targets. The same menu can be run +non-interactively: + +```bash +sh scripts/build_esp32_partition_migration.sh --board heltec-v4 \ + --version v1.17.1.7-halo-keymind-cascade-dev \ + --radio-preset usa-cascadia --profile cascade +sh scripts/build_esp32_partition_migration.sh --all \ + --version v1.17.1.7-halo-keymind-cascade-dev \ + --radio-preset usa-cascadia --profile cascade +``` + +`--all` creates one verified ZIP per currently qualified board and a release +ZIP containing all of them, their hashes, and a manifest. This is a **local +release bundle**, not an upload to GitHub Releases. It is not an archive of +old firmware binaries. `--list-boards` shows the presently qualified choices. +The release fails instead of publishing a partial bundle if one board package +is absent or fails verification. + +The underlying Python recipe remains available: ```bash python3 -B scripts/build_esp32_partition_migration.py \ @@ -30,13 +56,21 @@ python3 -B scripts/build_esp32_partition_migration.py \ --radio-preset usa-cascadia --profile cascade ``` -The default builds both currently validated boards. Add `--board heltec-v4` +The default builds all currently qualified boards. Add `--board heltec-v4` or `--board xiao-s3-wio` to build one; repeat `--board` for a selected set. Use `--dry-run` to inspect the command order without building. The resulting ZIPs appear under `.releases/esp32-expanded-/packages/`. The recipe requires a clean checkout so each ZIP identifies the firmware commit it was built from. It creates files only; it does not flash a device or erase flash. +The historical 1.25 MiB release population is wider than these two boards. +In particular, the portable LilyGo T3S3 SX1262/SX1276, Station G2, and +ThinkNode M2 repeater/room-server targets have had 1.25 MiB slots. They are +**not** silently treated as supported by `--all`: the 4 MiB boards require a +different target table and a power-loss-safe LoRa receiver handoff, and every +additional role needs an audited exact target identity. A 4 MiB Full build +offers 1,984 KiB slots, not the 4 MiB-or-more slots possible on larger flash. + Other ESP32 boards need a reviewed flash-size/partition plan, exact LoRa OTA target ID, and board entry before they can be added to this recipe. Matching flash capacity alone is not enough to claim a package is safe for a board. diff --git a/scripts/build_esp32_partition_migration.py b/scripts/build_esp32_partition_migration.py index 2fef6600..83710daa 100644 --- a/scripts/build_esp32_partition_migration.py +++ b/scripts/build_esp32_partition_migration.py @@ -12,6 +12,7 @@ import hashlib import json import os from pathlib import Path +import re import shlex import shutil import subprocess @@ -67,10 +68,59 @@ def verify_archive(path: Path, board: str, version: str, source: str) -> None: raise ValueError(f"{path}: invalid {filename}") +def bundle_release(output_root: Path, package_dir: Path, boards: list[str], + version: str, source: str, radio_preset: str, + profile: str) -> Path: + """Atomically publish the complete, exact-source migration set.""" + packages = [] + for board in boards: + matches = list(package_dir.glob(f"{board}-{version}-{source}-migration.zip")) + if len(matches) != 1: + raise ValueError(f"release is missing {board}") + verify_archive(matches[0], board, version, source) + data = matches[0].read_bytes() + packages.append((matches[0].name, data)) + manifest = { + "format": "meshcore-esp32-partition-migration-release-v1", + "scope": "currently-qualified-board-recipes-only", + "historical_firmware_binaries_included": False, + "source_commit": source, + "firmware_version": version, + "radio_preset": radio_preset, + "profile": profile, + "boards": boards, + "packages": {name: {"bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest()} + for name, data in packages}, + } + archive = output_root / f"esp32-partition-migration-{version}-{source}-release.zip" + with tempfile.NamedTemporaryFile(prefix="migration-release-", suffix=".zip", + dir=output_root, delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + with zipfile.ZipFile(temporary_path, "w", compression=zipfile.ZIP_STORED) as result: + result.writestr("release-manifest.json", json.dumps(manifest, indent=2) + "\n") + for name, data in packages: + result.writestr(name, data) + with zipfile.ZipFile(temporary_path) as result: + if result.testzip() or set(result.namelist()) != {"release-manifest.json"} | { + name for name, _ in packages}: + raise ValueError("release ZIP failed integrity check") + if archive.exists(): + if hashlib.sha256(archive.read_bytes()).digest() != \ + hashlib.sha256(temporary_path.read_bytes()).digest(): + raise ValueError(f"existing release differs: {archive}") + else: + temporary_path.replace(archive) + finally: + temporary_path.unlink(missing_ok=True) + return archive + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--version", required=True, help="firmware release version") - parser.add_argument("--radio-preset", required=True, + parser.add_argument("--version", help="firmware release version") + parser.add_argument("--radio-preset", help="radio preset passed to build.sh") parser.add_argument("--profile", choices=("default", "cascade"), default="default", help="embedded runtime profile") @@ -82,7 +132,17 @@ def main() -> None: help="release folder (default: .releases/esp32-expanded-)") parser.add_argument("--dry-run", action="store_true", help="show the build order without writing files") + parser.add_argument("--list-boards", action="store_true", + help="print qualified board keys and exit") args = parser.parse_args() + if args.list_boards: + for name in BOARDS: + print(name) + return + if not args.version or not args.radio_preset: + parser.error("--version and --radio-preset are required for a build") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+\-]*", args.version): + parser.error("--version must be a filename-safe version token") if args.jobs < 1: parser.error("--jobs must be positive") if os.name == "nt" and not args.dry_run: @@ -100,6 +160,8 @@ def main() -> None: prefix = f"OUTPUT_DIR={shlex.quote(str(build_dir))} " if full else "" print(prefix + shlex.join(command), flush=True) print(f"Package {', '.join(boards)} into {package_dir}", flush=True) + if set(boards) == set(BOARDS): + print(f"Bundle complete qualified set (not all historical ESP32 targets) in {output_root}", flush=True) if args.dry_run: return @@ -146,6 +208,10 @@ def main() -> None: else: staged.replace(destination) print(f"Ready: {destination}", flush=True) + if set(boards) == set(BOARDS): + print("Release: " + str(bundle_release( + output_root, package_dir, boards, args.version, source, + args.radio_preset, args.profile)), flush=True) if __name__ == "__main__": diff --git a/scripts/build_esp32_partition_migration.sh b/scripts/build_esp32_partition_migration.sh new file mode 100755 index 00000000..bba584fa --- /dev/null +++ b/scripts/build_esp32_partition_migration.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Interactive front end for the verified ESP32 in-place migration recipe. +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +recipe=$script_dir/build_esp32_partition_migration.py + +usage() { + cat <<'EOF' +Usage: scripts/build_esp32_partition_migration.sh [options] + +With no arguments, choose one qualified ESP32 board or the complete qualified +set from a menu. The complete set creates one ZIP per board plus a release ZIP. + + --board KEY Build one board (repeatable) + --all Build the complete qualified set + --version VERSION Firmware version (required without the menu) + --radio-preset NAME Radio preset (required without the menu) + --profile NAME default or cascade (default: cascade) + --jobs N PlatformIO workers within each serial build + --output-root PATH Release output folder + --dry-run Print commands without building + --list-boards Show board keys that have verified migration recipes + --help Show this help + +Boards without an audited flash/partition/identity plan are not offered. A +name appearing in an old firmware release is not proof that OTA migration is +safe on that hardware. No release ZIP is published unless every selected +package passes the recipe's checks. +EOF +} + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required" >&2 + exit 1 +fi + +version= +radio_preset= +profile=cascade +jobs= +output_root= +boards= +all=0 +dry_run=0 +interactive=1 + +while [ "$#" -gt 0 ]; do + interactive=0 + case "$1" in + --board|--version|--radio-preset|--profile|--jobs|--output-root) + if [ "$#" -lt 2 ]; then + echo "Missing value for $1" >&2 + exit 2 + fi + case "$1" in + --board) boards="${boards}${boards:+ }$2" ;; + --version) version=$2 ;; + --radio-preset) radio_preset=$2 ;; + --profile) profile=$2 ;; + --jobs) jobs=$2 ;; + --output-root) output_root=$2 ;; + esac + shift 2 ;; + --all) all=1; shift ;; + --dry-run) dry_run=1; shift ;; + --list-boards) exec python3 -B "$recipe" --list-boards ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +board_keys=$(python3 -B "$recipe" --list-boards) +if [ -z "$board_keys" ]; then + echo "No qualified boards are configured" >&2 + exit 1 +fi + +if [ "$interactive" -eq 1 ]; then + echo "ESP32 in-place partition migration" + echo "Only audited board/layout recipes are listed." + index=1 + for board in $board_keys; do + echo " $index) $board" + index=$((index + 1)) + done + echo " a) Build all qualified boards and create a release ZIP" + echo " q) Quit" + printf 'Choose: ' + IFS= read -r choice + case "$choice" in + a|A|all) all=1 ;; + q|Q|quit) exit 0 ;; + *) + index=1 + for board in $board_keys; do + if [ "$choice" = "$index" ]; then + boards=$board + break + fi + index=$((index + 1)) + done + if [ -z "$boards" ]; then + echo "Invalid selection" >&2 + exit 2 + fi ;; + esac + printf 'Firmware version: ' + IFS= read -r version + printf 'Radio preset [usa-cascadia]: ' + IFS= read -r radio_preset + radio_preset=${radio_preset:-usa-cascadia} + printf 'Profile [cascade]: ' + IFS= read -r selected_profile + profile=${selected_profile:-cascade} +fi + +if [ "$all" -eq 1 ] && [ -n "$boards" ]; then + echo "Choose either --all or --board, not both" >&2 + exit 2 +fi +if [ -z "$version" ] || [ -z "$radio_preset" ]; then + echo "Firmware version and radio preset are required" >&2 + exit 2 +fi +if [ "$profile" != default ] && [ "$profile" != cascade ]; then + echo "Profile must be default or cascade" >&2 + exit 2 +fi + +set -- "$recipe" --version "$version" --radio-preset "$radio_preset" --profile "$profile" +if [ -n "$boards" ]; then + for board in $boards; do + found=0 + for known in $board_keys; do + if [ "$board" = "$known" ]; then found=1; break; fi + done + if [ "$found" -ne 1 ]; then + echo "Board is not qualified for this recipe: $board" >&2 + exit 2 + fi + set -- "$@" --board "$board" + done +fi +if [ -n "$jobs" ]; then set -- "$@" --jobs "$jobs"; fi +if [ -n "$output_root" ]; then set -- "$@" --output-root "$output_root"; fi +if [ "$dry_run" -eq 1 ]; then set -- "$@" --dry-run; fi +exec python3 -B "$@" diff --git a/test/test_esp32_partition_migration_recipe.py b/test/test_esp32_partition_migration_recipe.py index 16acf2cd..483cc381 100644 --- a/test/test_esp32_partition_migration_recipe.py +++ b/test/test_esp32_partition_migration_recipe.py @@ -2,6 +2,7 @@ import hashlib import json +import os from pathlib import Path import subprocess import sys @@ -12,10 +13,35 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) -from build_esp32_partition_migration import build_steps, verify_archive # noqa: E402 +from build_esp32_partition_migration import ( # noqa: E402 + build_steps, bundle_release, verify_archive, +) class Esp32MigrationRecipeTest(unittest.TestCase): + @unittest.skipIf(os.name == "nt", "POSIX shell menu runs in WSL/Linux") + def test_shell_menu_and_noninteractive_selection(self): + script = str(ROOT / "scripts/build_esp32_partition_migration.sh") + common = ["--version", "v1.17.1.7-test", + "--radio-preset", "usa-cascadia", "--dry-run"] + one = subprocess.run(["sh", script, "--board", "heltec-v4", *common], + cwd=ROOT, text=True, capture_output=True, check=True) + self.assertIn("build-firmware heltec_v4_repeater", one.stdout) + self.assertNotIn("build-firmware Xiao_S3_WIO_repeater", one.stdout) + self.assertNotIn("Bundle complete qualified set", one.stdout) + all_boards = subprocess.run(["sh", script, "--all", *common], + cwd=ROOT, text=True, capture_output=True, check=True) + self.assertIn("build-firmware Xiao_S3_WIO_repeater", all_boards.stdout) + self.assertIn("Bundle complete qualified set", all_boards.stdout) + interactive = subprocess.run(["sh", script], input="q\n", + cwd=ROOT, text=True, capture_output=True, + check=True) + self.assertIn("ESP32 in-place partition migration", interactive.stdout) + invalid = subprocess.run(["sh", script, "--board", "unknown", *common], + cwd=ROOT, text=True, capture_output=True) + self.assertNotEqual(0, invalid.returncode) + self.assertIn("not qualified", invalid.stderr) + def test_full_images_precede_every_bridge_and_builds_are_serial(self): steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test", "usa-cascadia", "cascade", 4) @@ -69,6 +95,49 @@ class Esp32MigrationRecipeTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "invalid full-application.bin"): verify_archive(archive_path, "heltec-v4", "v1.17.1.7-test", "01234567") + def test_all_bundle_contains_only_verified_board_packages(self): + with tempfile.TemporaryDirectory(prefix="esp32-migration-release-") as temporary: + output_root = Path(temporary) + package_dir = output_root / "packages" + package_dir.mkdir() + version = "v1.17.1.7-test" + source = "01234567" + for board in ("heltec-v4", "xiao-s3-wio"): + payload = (board + " image").encode() + manifest = { + "board": board, "firmware_version": version, + "source_commit": source, "files": { + "full-application.bin": { + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + archive_path = package_dir / f"{board}-{version}-{source}-migration.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + archive.writestr("full-application.bin", payload) + release = bundle_release(output_root, package_dir, + ["heltec-v4", "xiao-s3-wio"], + version, source, "usa-cascadia", "cascade") + with zipfile.ZipFile(release) as archive: + self.assertIsNone(archive.testzip()) + release_manifest = json.loads(archive.read("release-manifest.json")) + self.assertEqual(["heltec-v4", "xiao-s3-wio"], + release_manifest["boards"]) + for name, checks in release_manifest["packages"].items(): + data = archive.read(name) + self.assertEqual(checks["bytes"], len(data)) + self.assertEqual(checks["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(release, bundle_release( + output_root, package_dir, ["heltec-v4", "xiao-s3-wio"], + version, source, "usa-cascadia", "cascade")) + (package_dir / f"xiao-s3-wio-{version}-{source}-migration.zip").unlink() + with self.assertRaisesRegex(ValueError, "missing xiao-s3-wio"): + bundle_release(output_root, package_dir, + ["heltec-v4", "xiao-s3-wio"], + version, source, "usa-cascadia", "cascade") + if __name__ == "__main__": unittest.main()