dandri/HaloKeymind
1
0
Fork 0
mirror of https://github.com/mikecarper/MeshCore.git synced 2026-09-26 14:07:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ESP32 migration build menu and verified release bundle

Browse Source
This commit is contained in:
mikecarper
2026-09-24 08:02:31 -07:00
parent 9f3977bcfb
commit af73d05f8b
4 changed files with 325 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/esp32_wifi_partition_migration.md
+39 -5
View File
@@ -19,10 +19,36 @@ and its generated table-prefix test; the bridge core is otherwise shared.
## Repeatable ESP32 build recipe
From a clean commit, run the recipe in WSL/Linux. It builds the Full repeater
images first, then both small bridges for each selected board, then verifies
and publishes one ZIP per board. Only one PlatformIO process runs at a time;
`build.sh` can clean the shared `.pio/build` tree between Full targets.
From a clean commit, run the shell menu in WSL/Linux:
```bash
sh scripts/build_esp32_partition_migration.sh
```
Choose one board, or **all qualified boards**. The script asks for the
firmware version, radio preset, and profile, then builds the Full repeater
images first, both small bridges for each selected board, and verifies each
ZIP. Only one PlatformIO process runs at a time; `build.sh` can clean the
shared `.pio/build` tree between Full targets. The same menu can be run
non-interactively:
```bash
sh scripts/build_esp32_partition_migration.sh --board heltec-v4 \
--version v1.17.1.7-halo-keymind-cascade-dev \
--radio-preset usa-cascadia --profile cascade
sh scripts/build_esp32_partition_migration.sh --all \
--version v1.17.1.7-halo-keymind-cascade-dev \
--radio-preset usa-cascadia --profile cascade
```
`--all` creates one verified ZIP per currently qualified board and a release
ZIP containing all of them, their hashes, and a manifest. This is a **local
release bundle**, not an upload to GitHub Releases. It is not an archive of
old firmware binaries. `--list-boards` shows the presently qualified choices.
The release fails instead of publishing a partial bundle if one board package
is absent or fails verification.
The underlying Python recipe remains available:
```bash
python3 -B scripts/build_esp32_partition_migration.py \
@@ -30,13 +56,21 @@ python3 -B scripts/build_esp32_partition_migration.py \
--radio-preset usa-cascadia --profile cascade
```
The default builds both currently validated boards. Add `--board heltec-v4`
The default builds all currently qualified boards. Add `--board heltec-v4`
or `--board xiao-s3-wio` to build one; repeat `--board` for a selected set.
Use `--dry-run` to inspect the command order without building. The resulting
ZIPs appear under `.releases/esp32-expanded-<commit>/packages/`. The recipe
requires a clean checkout so each ZIP identifies the firmware commit it was
built from. It creates files only; it does not flash a device or erase flash.
The historical 1.25 MiB release population is wider than these two boards.
In particular, the portable LilyGo T3S3 SX1262/SX1276, Station G2, and
ThinkNode M2 repeater/room-server targets have had 1.25 MiB slots. They are
**not** silently treated as supported by `--all`: the 4 MiB boards require a
different target table and a power-loss-safe LoRa receiver handoff, and every
additional role needs an audited exact target identity. A 4 MiB Full build
offers 1,984 KiB slots, not the 4 MiB-or-more slots possible on larger flash.
Other ESP32 boards need a reviewed flash-size/partition plan, exact LoRa OTA
target ID, and board entry before they can be added to this recipe. Matching
flash capacity alone is not enough to claim a package is safe for a board.
scripts/build_esp32_partition_migration.py
+68 -2
View File
@@ -12,6 +12,7 @@ import hashlib
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
@@ -67,10 +68,59 @@ def verify_archive(path: Path, board: str, version: str, source: str) -> None:
raise ValueError(f"{path}: invalid {filename}")
def bundle_release(output_root: Path, package_dir: Path, boards: list[str],
version: str, source: str, radio_preset: str,
profile: str) -> Path:
"""Atomically publish the complete, exact-source migration set."""
packages = []
for board in boards:
matches = list(package_dir.glob(f"{board}-{version}-{source}-migration.zip"))
if len(matches) != 1:
raise ValueError(f"release is missing {board}")
verify_archive(matches[0], board, version, source)
data = matches[0].read_bytes()
packages.append((matches[0].name, data))
manifest = {
"format": "meshcore-esp32-partition-migration-release-v1",
"scope": "currently-qualified-board-recipes-only",
"historical_firmware_binaries_included": False,
"source_commit": source,
"firmware_version": version,
"radio_preset": radio_preset,
"profile": profile,
"boards": boards,
"packages": {name: {"bytes": len(data),
"sha256": hashlib.sha256(data).hexdigest()}
for name, data in packages},
}
archive = output_root / f"esp32-partition-migration-{version}-{source}-release.zip"
with tempfile.NamedTemporaryFile(prefix="migration-release-", suffix=".zip",
dir=output_root, delete=False) as temporary:
temporary_path = Path(temporary.name)
try:
with zipfile.ZipFile(temporary_path, "w", compression=zipfile.ZIP_STORED) as result:
result.writestr("release-manifest.json", json.dumps(manifest, indent=2) + "\n")
for name, data in packages:
result.writestr(name, data)
with zipfile.ZipFile(temporary_path) as result:
if result.testzip() or set(result.namelist()) != {"release-manifest.json"} | {
name for name, _ in packages}:
raise ValueError("release ZIP failed integrity check")
if archive.exists():
if hashlib.sha256(archive.read_bytes()).digest() != \
hashlib.sha256(temporary_path.read_bytes()).digest():
raise ValueError(f"existing release differs: {archive}")
else:
temporary_path.replace(archive)
finally:
temporary_path.unlink(missing_ok=True)
return archive
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="firmware release version")
parser.add_argument("--radio-preset", required=True,
parser.add_argument("--version", help="firmware release version")
parser.add_argument("--radio-preset",
help="radio preset passed to build.sh")
parser.add_argument("--profile", choices=("default", "cascade"),
default="default", help="embedded runtime profile")
@@ -82,7 +132,17 @@ def main() -> None:
help="release folder (default: .releases/esp32-expanded-<commit>)")
parser.add_argument("--dry-run", action="store_true",
help="show the build order without writing files")
parser.add_argument("--list-boards", action="store_true",
help="print qualified board keys and exit")
args = parser.parse_args()
if args.list_boards:
for name in BOARDS:
print(name)
return
if not args.version or not args.radio_preset:
parser.error("--version and --radio-preset are required for a build")
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+\-]*", args.version):
parser.error("--version must be a filename-safe version token")
if args.jobs < 1:
parser.error("--jobs must be positive")
if os.name == "nt" and not args.dry_run:
@@ -100,6 +160,8 @@ def main() -> None:
prefix = f"OUTPUT_DIR={shlex.quote(str(build_dir))} " if full else ""
print(prefix + shlex.join(command), flush=True)
print(f"Package {', '.join(boards)} into {package_dir}", flush=True)
if set(boards) == set(BOARDS):
print(f"Bundle complete qualified set (not all historical ESP32 targets) in {output_root}", flush=True)
if args.dry_run:
return
@@ -146,6 +208,10 @@ def main() -> None:
else:
staged.replace(destination)
print(f"Ready: {destination}", flush=True)
if set(boards) == set(BOARDS):
print("Release: " + str(bundle_release(
output_root, package_dir, boards, args.version, source,
args.radio_preset, args.profile)), flush=True)
if __name__ == "__main__":
scripts/build_esp32_partition_migration.sh
Executable
+148
View File
@@ -0,0 +1,148 @@
#!/bin/sh
# Interactive front end for the verified ESP32 in-place migration recipe.
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
recipe=$script_dir/build_esp32_partition_migration.py
usage() {
cat <<'EOF'
Usage: scripts/build_esp32_partition_migration.sh [options]
With no arguments, choose one qualified ESP32 board or the complete qualified
set from a menu. The complete set creates one ZIP per board plus a release ZIP.
--board KEY Build one board (repeatable)
--all Build the complete qualified set
--version VERSION Firmware version (required without the menu)
--radio-preset NAME Radio preset (required without the menu)
--profile NAME default or cascade (default: cascade)
--jobs N PlatformIO workers within each serial build
--output-root PATH Release output folder
--dry-run Print commands without building
--list-boards Show board keys that have verified migration recipes
--help Show this help
Boards without an audited flash/partition/identity plan are not offered. A
name appearing in an old firmware release is not proof that OTA migration is
safe on that hardware. No release ZIP is published unless every selected
package passes the recipe's checks.
EOF
}
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required" >&2
exit 1
fi
version=
radio_preset=
profile=cascade
jobs=
output_root=
boards=
all=0
dry_run=0
interactive=1
while [ "$#" -gt 0 ]; do
interactive=0
case "$1" in
--board|--version|--radio-preset|--profile|--jobs|--output-root)
if [ "$#" -lt 2 ]; then
echo "Missing value for $1" >&2
exit 2
fi
case "$1" in
--board) boards="${boards}${boards:+ }$2" ;;
--version) version=$2 ;;
--radio-preset) radio_preset=$2 ;;
--profile) profile=$2 ;;
--jobs) jobs=$2 ;;
--output-root) output_root=$2 ;;
esac
shift 2 ;;
--all) all=1; shift ;;
--dry-run) dry_run=1; shift ;;
--list-boards) exec python3 -B "$recipe" --list-boards ;;
--help|-h) usage; exit 0 ;;
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
esac
done
board_keys=$(python3 -B "$recipe" --list-boards)
if [ -z "$board_keys" ]; then
echo "No qualified boards are configured" >&2
exit 1
fi
if [ "$interactive" -eq 1 ]; then
echo "ESP32 in-place partition migration"
echo "Only audited board/layout recipes are listed."
index=1
for board in $board_keys; do
echo " $index) $board"
index=$((index + 1))
done
echo " a) Build all qualified boards and create a release ZIP"
echo " q) Quit"
printf 'Choose: '
IFS= read -r choice
case "$choice" in
a|A|all) all=1 ;;
q|Q|quit) exit 0 ;;
*)
index=1
for board in $board_keys; do
if [ "$choice" = "$index" ]; then
boards=$board
break
fi
index=$((index + 1))
done
if [ -z "$boards" ]; then
echo "Invalid selection" >&2
exit 2
fi ;;
esac
printf 'Firmware version: '
IFS= read -r version
printf 'Radio preset [usa-cascadia]: '
IFS= read -r radio_preset
radio_preset=${radio_preset:-usa-cascadia}
printf 'Profile [cascade]: '
IFS= read -r selected_profile
profile=${selected_profile:-cascade}
fi
if [ "$all" -eq 1 ] && [ -n "$boards" ]; then
echo "Choose either --all or --board, not both" >&2
exit 2
fi
if [ -z "$version" ] || [ -z "$radio_preset" ]; then
echo "Firmware version and radio preset are required" >&2
exit 2
fi
if [ "$profile" != default ] && [ "$profile" != cascade ]; then
echo "Profile must be default or cascade" >&2
exit 2
fi
set -- "$recipe" --version "$version" --radio-preset "$radio_preset" --profile "$profile"
if [ -n "$boards" ]; then
for board in $boards; do
found=0
for known in $board_keys; do
if [ "$board" = "$known" ]; then found=1; break; fi
done
if [ "$found" -ne 1 ]; then
echo "Board is not qualified for this recipe: $board" >&2
exit 2
fi
set -- "$@" --board "$board"
done
fi
if [ -n "$jobs" ]; then set -- "$@" --jobs "$jobs"; fi
if [ -n "$output_root" ]; then set -- "$@" --output-root "$output_root"; fi
if [ "$dry_run" -eq 1 ]; then set -- "$@" --dry-run; fi
exec python3 -B "$@"
test/test_esp32_partition_migration_recipe.py
+70 -1
View File
@@ -2,6 +2,7 @@
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
@@ -12,10 +13,35 @@ import zipfile
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from build_esp32_partition_migration import build_steps, verify_archive # noqa: E402
from build_esp32_partition_migration import ( # noqa: E402
build_steps, bundle_release, verify_archive,
)
class Esp32MigrationRecipeTest(unittest.TestCase):
@unittest.skipIf(os.name == "nt", "POSIX shell menu runs in WSL/Linux")
def test_shell_menu_and_noninteractive_selection(self):
script = str(ROOT / "scripts/build_esp32_partition_migration.sh")
common = ["--version", "v1.17.1.7-test",
"--radio-preset", "usa-cascadia", "--dry-run"]
one = subprocess.run(["sh", script, "--board", "heltec-v4", *common],
cwd=ROOT, text=True, capture_output=True, check=True)
self.assertIn("build-firmware heltec_v4_repeater", one.stdout)
self.assertNotIn("build-firmware Xiao_S3_WIO_repeater", one.stdout)
self.assertNotIn("Bundle complete qualified set", one.stdout)
all_boards = subprocess.run(["sh", script, "--all", *common],
cwd=ROOT, text=True, capture_output=True, check=True)
self.assertIn("build-firmware Xiao_S3_WIO_repeater", all_boards.stdout)
self.assertIn("Bundle complete qualified set", all_boards.stdout)
interactive = subprocess.run(["sh", script], input="q\n",
cwd=ROOT, text=True, capture_output=True,
check=True)
self.assertIn("ESP32 in-place partition migration", interactive.stdout)
invalid = subprocess.run(["sh", script, "--board", "unknown", *common],
cwd=ROOT, text=True, capture_output=True)
self.assertNotEqual(0, invalid.returncode)
self.assertIn("not qualified", invalid.stderr)
def test_full_images_precede_every_bridge_and_builds_are_serial(self):
steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test",
"usa-cascadia", "cascade", 4)
@@ -69,6 +95,49 @@ class Esp32MigrationRecipeTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "invalid full-application.bin"):
verify_archive(archive_path, "heltec-v4", "v1.17.1.7-test", "01234567")
def test_all_bundle_contains_only_verified_board_packages(self):
with tempfile.TemporaryDirectory(prefix="esp32-migration-release-") as temporary:
output_root = Path(temporary)
package_dir = output_root / "packages"
package_dir.mkdir()
version = "v1.17.1.7-test"
source = "01234567"
for board in ("heltec-v4", "xiao-s3-wio"):
payload = (board + " image").encode()
manifest = {
"board": board, "firmware_version": version,
"source_commit": source, "files": {
"full-application.bin": {
"bytes": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
}
},
}
archive_path = package_dir / f"{board}-{version}-{source}-migration.zip"
with zipfile.ZipFile(archive_path, "w") as archive:
archive.writestr("manifest.json", json.dumps(manifest))
archive.writestr("full-application.bin", payload)
release = bundle_release(output_root, package_dir,
["heltec-v4", "xiao-s3-wio"],
version, source, "usa-cascadia", "cascade")
with zipfile.ZipFile(release) as archive:
self.assertIsNone(archive.testzip())
release_manifest = json.loads(archive.read("release-manifest.json"))
self.assertEqual(["heltec-v4", "xiao-s3-wio"],
release_manifest["boards"])
for name, checks in release_manifest["packages"].items():
data = archive.read(name)
self.assertEqual(checks["bytes"], len(data))
self.assertEqual(checks["sha256"], hashlib.sha256(data).hexdigest())
self.assertEqual(release, bundle_release(
output_root, package_dir, ["heltec-v4", "xiao-s3-wio"],
version, source, "usa-cascadia", "cascade"))
(package_dir / f"xiao-s3-wio-{version}-{source}-migration.zip").unlink()
with self.assertRaisesRegex(ValueError, "missing xiao-s3-wio"):
bundle_release(output_root, package_dir,
["heltec-v4", "xiao-s3-wio"],
version, source, "usa-cascadia", "cascade")
if __name__ == "__main__":
unittest.main()