From 949acb2b7d793851e5c8ccdb2c5062912584e2fc Mon Sep 17 00:00:00 2001 From: Valentin Kivachuk Burda Date: Fri, 3 Jul 2026 08:24:54 +0200 Subject: [PATCH] dev-only: fork rolling-CI workflow + venv .gitignore + EndF/apply-sim helper scripts --- .github/workflows/dev-firmware-rolling.yml | 160 +++++++++++++++++++++ .gitignore | 4 +- .vscode/extensions.json | 2 + tools/mota/README.md | 2 + tools/mota/endf.py | 44 ++++++ tools/mota/extract_apply.py | 18 +++ 6 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dev-firmware-rolling.yml create mode 100644 tools/mota/endf.py create mode 100644 tools/mota/extract_apply.py diff --git a/.github/workflows/dev-firmware-rolling.yml b/.github/workflows/dev-firmware-rolling.yml new file mode 100644 index 00000000..f0cfec69 --- /dev/null +++ b/.github/workflows/dev-firmware-rolling.yml @@ -0,0 +1,160 @@ +# Rolling DEV firmware release (fork convenience). +# +# On every push, rebuild a representative set of OTA-capable firmwares (ESP32 + nRF52, one per board family +# plus the RAK4631 + Heltec T096 (nRF52) and Heltec V3 + V4 (ESP32) OTA test boards in every role) and +# replace the `dev-latest` prerelease assets +# with the flashable binaries (ESP32: .bin/-merged.bin; nRF52: .uf2/.zip). UNSIGNED dev builds for testing. +# OTA `.mota` are not packaged here — build them on demand from a firmware with tools/motatool (which reads +# a .bin or nRF52 .hex directly). +# +# Design (atomic, never-empty): each board builds into a workflow ARTIFACT; a final `release` job collects +# them and ATOMICALLY recreates the release at the end. So if the run is cancelled by the next push, or some +# boards fail to compile, the PREVIOUS release stays intact until a full new set is ready (the old "empty +# the release first, then fill it" design left `dev-latest` empty whenever a build was cancelled/failed). +# +# To change which boards build, edit the `matrix.include` list below (env + its platform). Only OTA-capable +# (ENABLE_OTA) envs make sense here. The full OTA set is ~308 envs; this is a curated ~one-per-board subset. + +name: Dev Firmware (rolling) + +on: + workflow_dispatch: + push: + +permissions: + contents: write + +concurrency: + group: dev-firmware-rolling + cancel-in-progress: true + +env: + RELEASE_TAG: dev-latest + FIRMWARE_VERSION: dev + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # --- nRF52 (OTA in-place; RAK4631 is the validated target, all roles) --- + - { env: RAK_4631_repeater, platform: NRF52 } + - { env: RAK_4631_room_server, platform: NRF52 } + - { env: RAK_4631_companion_radio_ble, platform: NRF52 } + - { env: RAK_4631_companion_radio_usb, platform: NRF52 } + # --- nRF52 OTA test board (Heltec T096, all roles) --- + - { env: Heltec_t096_repeater, platform: NRF52 } + - { env: Heltec_t096_room_server, platform: NRF52 } + - { env: Heltec_t096_companion_radio_ble, platform: NRF52 } + - { env: Heltec_t096_companion_radio_usb, platform: NRF52 } + # --- ESP32 test board (Heltec V3, all roles) --- + - { env: Heltec_v3_repeater, platform: ESP32 } + - { env: Heltec_v3_room_server, platform: ESP32 } + - { env: Heltec_v3_companion_radio_ble, platform: ESP32 } + - { env: Heltec_v3_companion_radio_usb, platform: ESP32 } + # --- ESP32 OTA test board (Heltec V4, all roles) --- + - { env: heltec_v4_repeater, platform: ESP32 } + - { env: heltec_v4_room_server, platform: ESP32 } + - { env: heltec_v4_companion_radio_ble, platform: ESP32 } + - { env: heltec_v4_companion_radio_usb, platform: ESP32 } + # --- ESP32, one representative per board family --- + - { env: Ebyte_EoRa-S3_room_server, platform: ESP32 } + - { env: Generic_E22_sx1262_repeater, platform: ESP32 } + - { env: Heltec_E213_repeater, platform: ESP32 } + - { env: Heltec_E290_repeater, platform: ESP32 } + - { env: Heltec_T190_repeater_, platform: ESP32 } + - { env: Heltec_ct62_repeater, platform: ESP32 } + - { env: Heltec_v2_repeater, platform: ESP32 } + - { env: LilyGo_T3S3_sx1262_repeater, platform: ESP32 } + - { env: LilyGo_TBeam_1W_repeater, platform: ESP32 } + - { env: LilyGo_TDeck_repeater, platform: ESP32 } + - { env: LilyGo_TETH_Elite_sx1262_repeater, platform: ESP32 } + - { env: LilyGo_TLora_V2_1_1_6_repeater, platform: ESP32 } + - { env: M5Stack_Unit_C6L_repeater, platform: ESP32 } + - { env: Station_G2_logging_repeater, platform: ESP32 } + - { env: Station_G3_ESP32_logging_repeater, platform: ESP32 } + - { env: T_Beam_S3_Supreme_SX1262_repeater, platform: ESP32 } + - { env: Tbeam_SX1262_repeater, platform: ESP32 } + - { env: ThinkNode_M2_room_server, platform: ESP32 } + - { env: Xiao_C3_repeater, platform: ESP32 } + - { env: Xiao_S3_WIO_repeater, platform: ESP32 } + - { env: heltec_tracker_v2_repeater, platform: ESP32 } + - { env: heltec_v4_expansionkit_repeater, platform: ESP32 } + - { env: nibble_screen_connect_repeater, platform: ESP32 } + steps: + - name: Clone Repo + uses: actions/checkout@v6 + + - name: Setup Build Environment + uses: ./.github/actions/setup-build-environment + + - name: Build ${{ matrix.env }} + run: /usr/bin/env bash build.sh build-firmware ${{ matrix.env }} + # build.sh writes the flashable firmware into out/ (ESP32: .bin + -merged.bin; nRF52: .uf2 + .zip). + # OTA .mota are NOT packaged in CI — build them locally with tools/motatool when needed. + + - name: Upload this board's artifact + uses: actions/upload-artifact@v7 + with: + name: fw-${{ matrix.env }} + path: out + if-no-files-found: ignore + retention-days: 5 + + # Collect every board's artifact and ATOMICALLY (re)create the rolling release. Runs even if some boards + # failed (if: always()), and only touches the release once assets are in hand — so a cancelled/partial run + # never empties dev-latest. + release: + needs: build + if: always() + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v6 + + - name: Collect built artifacts (via gh — no download-artifact version coupling) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p dist + gh run download "${{ github.run_id }}" --dir dist 2>/dev/null || true + # flatten dist/fw-/ -> dist/ + find dist -mindepth 2 -type f -exec mv -f -t dist {} + 2>/dev/null || true + find dist -mindepth 1 -type d -empty -delete 2>/dev/null || true + echo "Collected:"; ls -la dist || true + + - name: Recreate the rolling release (only if we actually have assets) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + shopt -s nullglob + files=(dist/*) + if [ ${#files[@]} -eq 0 ]; then + echo "::error::no firmware built — leaving the existing dev-latest release untouched" + exit 1 + fi + NOTES=$(cat <\`. + + A representative set of OTA-capable boards (ESP32 + nRF52): flashable firmware binaries + (ESP32 \`.bin\`/\`-merged.bin\`, nRF52 \`.uf2\`/\`.zip\`). To make an OTA \`.mota\` from one of + these, use \`tools/motatool\` (it reads a \`.bin\` or nRF52 \`.hex\` directly). + EOF + ) + # delete + recreate so removed boards don't leave stale assets; assets are ready, so it's atomic + gh release delete "$RELEASE_TAG" --yes --cleanup-tag 2>/dev/null || true + gh release create "$RELEASE_TAG" "${files[@]}" \ + --title "Dev firmware (latest commit)" \ + --notes "$NOTES" \ + --prerelease \ + --target "$GITHUB_SHA" diff --git a/.gitignore b/.gitignore index 4b135664..20cb0368 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,5 @@ compile_commands.json .venv/ venv/ platformio.local.ini - -__pycache__/ +meshcore +__pycache__/ \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 8057bc70..5c5735c6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,4 +1,6 @@ { + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format "recommendations": [ "pioarduino.pioarduino-ide", "platformio.platformio-ide" diff --git a/tools/mota/README.md b/tools/mota/README.md index a3239353..90323a82 100644 --- a/tools/mota/README.md +++ b/tools/mota/README.md @@ -27,6 +27,8 @@ Uses the repo's Python venv (`meshcore/`). Dependencies: `detools` (delta), `cry | `gen_vectors.py` | Generates `test/test_ota/mota_vectors.h` — the cross-check vectors the native C++ tests run against. | | `gen_targets.py` | Generates `src/helpers/ota/OtaTargets.h` — the `target_id → env-name` table (every `ENABLE_OTA` env, resolved from `pio project config`). Shared by the firmware and `motatool` so a node can name a target seen over the air without sending the string. Regenerate when the OTA env set changes. | | `test_mota.py` | Unit tests for `motalib` (run directly or via pytest). | +| `endf.py` | Standalone `EndF` trailer injector (idempotent) — handy for one-off `.bin` patching. | +| `extract_apply.py` | Dev helper: split a firmware into payload + fixed-manifest bytes for the apply test. | ## Tests diff --git a/tools/mota/endf.py b/tools/mota/endf.py new file mode 100644 index 00000000..f5277fa8 --- /dev/null +++ b/tools/mota/endf.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +""" +endf — append the MeshCore ``EndF`` trailer to a firmware image. + +The trailer lets a running node discover its own firmware size/identity on any MCU: + + EndF (16 bytes): "EndF"(4) | body_len(4 LE) | sha2-256:8(body)(8) + +Standalone usage (idempotent — a no-op if a valid EndF is already present): + + ./meshcore/bin/python tools/mota/endf.py firmware.bin # in place + ./meshcore/bin/python tools/mota/endf.py firmware.bin out.bin # to a new file + +As a PlatformIO post-build step (see tools/mota/README.md), wire it so that for OTA-enabled builds +the flashed artifact carries the trailer. EndF must be in the FLASHED image (not just the .mota), +because a node serves its own firmware and matches a delta's base against its own EndF. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import motalib as ml + + +def inject(in_path: str, out_path: str | None = None) -> int: + image = Path(in_path).read_bytes() + if ml.has_endf(image): + body, h8 = ml.parse_endf(image) + print(f"EndF already present: body_len={len(body)} body_hash={h8.hex()} (no change)") + out = image + else: + out, h8 = ml.ensure_endf(image) + print(f"EndF appended: body_len={len(image)} body_hash={h8.hex()} " + f"({len(image)} -> {len(out)} bytes)") + Path(out_path or in_path).write_bytes(out) + return 0 + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3): + sys.exit(__doc__) + raise SystemExit(inject(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else None)) diff --git a/tools/mota/extract_apply.py b/tools/mota/extract_apply.py new file mode 100644 index 00000000..35e1e4b7 --- /dev/null +++ b/tools/mota/extract_apply.py @@ -0,0 +1,18 @@ +"""Extract the payload (bootable image) + manifest-fixed bytes from a firmware .bin for the apply test. +Usage: extract_apply.py """ +import sys +import motalib as ml +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +fw = open(sys.argv[1], "rb").read() +priv = Ed25519PrivateKey.from_private_bytes(bytes.fromhex(open(sys.argv[2]).read().strip())) +image, _ = ml.ensure_endf(fw) # payload = bootable image (+ EndF), what goes to the slot +m = ml.build_manifest(target_id=0, fw_version=ml.pack_version("1.16.0"), + image_size=len(image), payload=image, block_size=1024, + image_hash=ml.mh32(image), codec_id=ml.CODEC_FULL, is_full=True, sign_priv=priv) +manifest_fixed = m.signed_region() + m.signature + m.approval # manifest WITHOUT leaves[] +open(sys.argv[3], "wb").write(image) +open(sys.argv[4], "wb").write(manifest_fixed) +print(f"payload(image)={len(image)} manifest_fixed={len(manifest_fixed)}") +print(f"image_hash={m.image_hash.hex()}") +print(f"signer={priv.public_key().public_bytes_raw().hex()}")