diff --git a/docs/esp32_wifi_partition_migration.md b/docs/esp32_wifi_partition_migration.md index 741e9be4d..155031878 100644 --- a/docs/esp32_wifi_partition_migration.md +++ b/docs/esp32_wifi_partition_migration.md @@ -17,6 +17,30 @@ and its generated table-prefix test; the bridge core is otherwise shared. | 8 MiB | `default_8MB.csv` | 0x330000 / 0x330000 | Seeed XIAO ESP32-S3 | | 16 MiB | `default_16MB.csv` | 0x640000 / 0x640000 | Heltec V4 / V4.3 OLED | +## Repeatable ESP32 build recipe + +From a clean commit, run the recipe in WSL/Linux. It builds the Full repeater +images first, then both small bridges for each selected board, then verifies +and publishes one ZIP per board. Only one PlatformIO process runs at a time; +`build.sh` can clean the shared `.pio/build` tree between Full targets. + +```bash +python3 -B scripts/build_esp32_partition_migration.py \ + --version v1.17.1.7-halo-keymind-cascade-dev \ + --radio-preset usa-cascadia --profile cascade +``` + +The default builds both currently validated boards. Add `--board heltec-v4` +or `--board xiao-s3-wio` to build one; repeat `--board` for a selected set. +Use `--dry-run` to inspect the command order without building. The resulting +ZIPs appear under `.releases/esp32-expanded-/packages/`. The recipe +requires a clean checkout so each ZIP identifies the firmware commit it was +built from. It creates files only; it does not flash a device or erase flash. + +Other ESP32 boards need a reviewed flash-size/partition plan, exact LoRa OTA +target ID, and board entry before they can be added to this recipe. Matching +flash capacity alone is not enough to claim a package is safe for a board. + ## What the bridge preserves Before it touches the partition table, the bridge reads the historical @@ -56,25 +80,15 @@ Keep power stable while the bridge is replacing the 4 KiB table sector. A loss of power before that point leaves the old table and source intact. After that small critical write, the selected A copy of the bridge is the recovery path. -## Build and migrate a Seeed XIAO ESP32-S3 +## Migrate a Seeed XIAO ESP32-S3 -The production bridge is a legacy-slot application image: - -```powershell -& "$env:USERPROFILE\.platformio\penv\Scripts\pio.exe" run -e xiao_s3_partition_migrator -``` - -Build the Full XIAO image with its 8 MiB table: - -```powershell -$env:MESHCORE_ESP32_FULL_BUILD = '1' -& "$env:USERPROFILE\.platformio\penv\Scripts\pio.exe" run -e Xiao_S3_WIO_repeater_observer_mqtt -``` - -Upload the bridge through the legacy node's existing Wi-Fi browser updater. +Use `--board xiao-s3-wio` with the recipe above. Its Wi-Fi bridge is a +legacy-slot application image; its Full image keeps the old repeater's exact +LoRa OTA target ID. Upload `wifi-bridge.bin` from the ZIP through the legacy +node's existing Wi-Fi browser updater. Then join `MeshCore-Migrate` (password `meshcore-migrate`) and wait for -**Expanded layout ready**. Upload the Full app-only `.bin` from the second -build at `/update`. Do not upload an `-merged.bin`: merged images include +**Expanded layout ready**. Upload `full-application.bin` at `/update`. +Do not upload an `-merged.bin`: merged images include bootloader and table offsets for cable flashing, not browser OTA. `xiao_s3_partition_legacy_seed` is a disposable-hardware test image only. It diff --git a/docs/heltec_v4_wifi_partition_migration.md b/docs/heltec_v4_wifi_partition_migration.md index 3ce690570..b199080bc 100644 --- a/docs/heltec_v4_wifi_partition_migration.md +++ b/docs/heltec_v4_wifi_partition_migration.md @@ -5,16 +5,19 @@ Heltec V4/V4.3 OLED repeaters with their normal 16 MiB flash use the shared It changes Arduino's old `default.csv` layout, with two 1.25 MiB OTA slots, to `default_16MB.csv`, with two 6.25 MiB OTA slots. -Build the bridge application image: +Build the complete V4 package with the repeatable ESP32 recipe in WSL/Linux: -```powershell -& "$env:USERPROFILE\.platformio\penv\Scripts\pio.exe" run -e heltec_v4_partition_migrator +```bash +python3 -B scripts/build_esp32_partition_migration.py \ + --board heltec-v4 --version v1.17.1.7-halo-keymind-cascade-dev \ + --radio-preset usa-cascadia --profile cascade ``` -Upload `.pio/build/heltec_v4_partition_migrator/firmware.bin` through the +The ZIP contains the Wi-Fi bridge, LoRa bridge, and exact-target Full image. +Upload its `wifi-bridge.bin` through the legacy browser Wi-Fi updater. Rejoin `MeshCore-Migrate` with password `meshcore-migrate`; when it reports **Expanded layout ready**, upload the -app-only Full V4 repeater image at `/update`. +ZIP's `full-application.bin` at `/update`. Do not use an `-merged.bin` in either browser uploader. Merged images contain bootloader and table offsets for cable flashing. The bridge preserves the diff --git a/scripts/build_esp32_partition_migration.py b/scripts/build_esp32_partition_migration.py new file mode 100644 index 000000000..57f2459a3 --- /dev/null +++ b/scripts/build_esp32_partition_migration.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Build verified in-place Wi-Fi and LoRa ESP32 partition-migration ZIPs. + +Run in Linux/WSL with PlatformIO installed. PlatformIO steps are deliberately +serial: build.sh cleans the shared .pio/build tree between Full targets. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import tempfile +import zipfile + +from package_esp32_partition_migration import BOARDS + + +ROOT = Path(__file__).resolve().parents[1] + + +def git_output(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip() + + +def build_steps(boards: list[str], version: str, radio_preset: str, + profile: str, jobs: int) -> list[tuple[list[str], bool]]: + steps: list[tuple[list[str], bool]] = [] + for name in boards: + steps.append(([ + "bash", "build.sh", "build-firmware", BOARDS[name]["target"], + "--full-exact", "--firmware-version", version, + "--radio-preset", radio_preset, "--profile", profile, + "--require-ota", "--resume", + ], True)) + # Full builds can clear .pio/build; every bridge must be built afterwards. + for name in boards: + for key in ("wifi_bridge", "lora_bridge"): + steps.append((["pio", "run", "-e", BOARDS[name][key], + "-j", str(jobs)], False)) + return steps + + +def verify_archive(path: Path, board: str, version: str, source: str) -> None: + with zipfile.ZipFile(path) as archive: + damaged = archive.testzip() + if damaged: + raise ValueError(f"{path}: damaged ZIP entry {damaged}") + manifest = json.loads(archive.read("manifest.json")) + if (manifest["board"] != board or manifest["firmware_version"] != version + or manifest["source_commit"] != source): + raise ValueError(f"{path}: package identity does not match recipe") + for filename, expected in manifest["files"].items(): + data = archive.read(filename) + if (len(data) != expected["bytes"] + or hashlib.sha256(data).hexdigest() != expected["sha256"]): + raise ValueError(f"{path}: invalid {filename}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="firmware release version") + parser.add_argument("--radio-preset", required=True, + help="radio preset passed to build.sh") + parser.add_argument("--profile", choices=("default", "cascade"), + default="default", help="embedded runtime profile") + parser.add_argument("--board", action="append", choices=tuple(BOARDS), + help="repeat to select boards (default: all supported boards)") + parser.add_argument("--jobs", type=int, default=4, + help="workers within each serial PlatformIO build") + parser.add_argument("--output-root", type=Path, + help="release folder (default: .releases/esp32-expanded-)") + parser.add_argument("--dry-run", action="store_true", + help="show the build order without writing files") + args = parser.parse_args() + if args.jobs < 1: + parser.error("--jobs must be positive") + if os.name == "nt" and not args.dry_run: + parser.error("run this recipe inside Linux/WSL, where bash and pio are available") + + source = git_output("rev-parse", "--short=8", "HEAD") + boards = list(dict.fromkeys(args.board or BOARDS)) + requested_root = args.output_root or Path(".releases") / f"esp32-expanded-{source}" + output_root = (requested_root if requested_root.is_absolute() + else ROOT / requested_root).resolve() + build_dir = output_root / "builds" + package_dir = output_root / "packages" + steps = build_steps(boards, args.version, args.radio_preset, args.profile, args.jobs) + for command, full in steps: + prefix = f"OUTPUT_DIR={shlex.quote(str(build_dir))} " if full else "" + print(prefix + shlex.join(command), flush=True) + print(f"Package {', '.join(boards)} into {package_dir}", flush=True) + if args.dry_run: + return + + if git_output("status", "--porcelain", "--untracked-files=normal"): + parser.error("commit or remove working-tree changes before building a versioned package") + for executable in ("bash", "pio"): + if not shutil.which(executable): + parser.error(f"{executable} is unavailable in this environment") + + output_root.mkdir(parents=True, exist_ok=True) + for command, full in steps: + environment = os.environ.copy() + if full: + environment["OUTPUT_DIR"] = str(build_dir) + subprocess.run(command, cwd=ROOT, env=environment, check=True) + + # Package in a fresh directory so an interruption never leaves a partial + # release ZIP at the final path. Existing identical ZIPs are reusable. + with tempfile.TemporaryDirectory(prefix="migration-package-", dir=output_root) as temporary: + staging = Path(temporary) / "packages" + command = [ + sys.executable, "-B", "scripts/package_esp32_partition_migration.py", + "--build-dir", str(build_dir), "--output-dir", str(staging), + "--version", args.version, "--source", source, + ] + for name in boards: + command.extend(("--board", name)) + subprocess.run(command, cwd=ROOT, check=True) + archives = sorted(staging.glob("*.zip")) + if len(archives) != len(boards): + raise ValueError("packager did not produce one ZIP per selected board") + package_dir.mkdir(parents=True, exist_ok=True) + for name in boards: + matches = [path for path in archives if path.name.startswith(name + "-")] + if len(matches) != 1: + raise ValueError(f"expected exactly one {name} ZIP") + staged = matches[0] + verify_archive(staged, name, args.version, source) + destination = package_dir / staged.name + if destination.exists(): + if hashlib.sha256(destination.read_bytes()).digest() != \ + hashlib.sha256(staged.read_bytes()).digest(): + raise ValueError(f"existing package differs: {destination}") + else: + staged.replace(destination) + print(f"Ready: {destination}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/package_esp32_partition_migration.py b/scripts/package_esp32_partition_migration.py index e8075606c..b6d570814 100644 --- a/scripts/package_esp32_partition_migration.py +++ b/scripts/package_esp32_partition_migration.py @@ -225,13 +225,16 @@ def main() -> None: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--version", required=True) parser.add_argument("--source", required=True, help="eight-character source commit") + parser.add_argument("--board", action="append", choices=tuple(BOARDS), + help="board to package; repeat for multiple boards (default: all)") args = parser.parse_args() if args.output_dir.exists() and any(args.output_dir.iterdir()): parser.error("output directory must be empty") args.output_dir.mkdir(parents=True, exist_ok=True) - archives = [package_board(name, spec, args.build_dir, args.output_dir, + selected = dict.fromkeys(args.board or BOARDS) + archives = [package_board(name, BOARDS[name], args.build_dir, args.output_dir, args.version, args.source) - for name, spec in BOARDS.items()] + for name in selected] for archive in archives: print(f"{archive}: {archive.stat().st_size} bytes") diff --git a/test/test_esp32_partition_migration_recipe.py b/test/test_esp32_partition_migration_recipe.py new file mode 100644 index 000000000..16acf2cd3 --- /dev/null +++ b/test/test_esp32_partition_migration_recipe.py @@ -0,0 +1,74 @@ +"""No-hardware contract tests for the serial ESP32 migration build recipe.""" + +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from build_esp32_partition_migration import build_steps, verify_archive # noqa: E402 + + +class Esp32MigrationRecipeTest(unittest.TestCase): + def test_full_images_precede_every_bridge_and_builds_are_serial(self): + steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test", + "usa-cascadia", "cascade", 4) + self.assertEqual(6, len(steps)) + self.assertEqual([True, True, False, False, False, False], + [full for _, full in steps]) + self.assertEqual("heltec_v4_repeater", steps[0][0][3]) + self.assertEqual("Xiao_S3_WIO_repeater", steps[1][0][3]) + self.assertTrue(all("--full-exact" in command for command, _ in steps[:2])) + self.assertEqual([ + "heltec_v4_partition_migrator", + "heltec_v4_partition_migrator_lora_repeater", + "xiao_s3_partition_migrator", + "xiao_s3_partition_migrator_lora_repeater", + ], [command[3] for command, _ in steps[2:]]) + + def test_dry_run_has_no_build_side_effects(self): + with tempfile.TemporaryDirectory(prefix="esp32-migration-plan-") as temporary: + output_root = Path(temporary) / "release" + result = subprocess.run([ + sys.executable, "-B", + str(ROOT / "scripts/build_esp32_partition_migration.py"), + "--version", "v1.17.1.7-test", "--radio-preset", "usa-cascadia", + "--board", "heltec-v4", "--output-root", str(output_root), + "--dry-run", + ], cwd=ROOT, text=True, capture_output=True, check=True) + self.assertIn("build-firmware heltec_v4_repeater", result.stdout) + self.assertIn("pio run -e heltec_v4_partition_migrator", result.stdout) + self.assertFalse(output_root.exists()) + + def test_archive_verification_rejects_changed_payload(self): + with tempfile.TemporaryDirectory(prefix="esp32-migration-zip-") as temporary: + archive_path = Path(temporary) / "test.zip" + payload = b"correct image" + manifest = { + "board": "heltec-v4", "firmware_version": "v1.17.1.7-test", + "source_commit": "01234567", "files": { + "full-application.bin": { + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + archive.writestr("full-application.bin", payload) + verify_archive(archive_path, "heltec-v4", "v1.17.1.7-test", "01234567") + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + archive.writestr("full-application.bin", b"damaged image") + with self.assertRaisesRegex(ValueError, "invalid full-application.bin"): + verify_archive(archive_path, "heltec-v4", "v1.17.1.7-test", "01234567") + + +if __name__ == "__main__": + unittest.main()