From ee413c30ff544a7d4cbb2cf7bfe03de63b6bdd39 Mon Sep 17 00:00:00 2001 From: Valentin Kivachuk Burda Date: Sun, 28 Jun 2026 00:28:29 +0200 Subject: [PATCH] small fixes --- .github/workflows/dev-firmware-rolling.yml | 107 ++++++++++++--------- docs/ota_protocol.md | 51 +++++++--- docs/ota_user_guide.md | 11 ++- src/helpers/ota/FirmwareInfo.cpp | 10 ++ src/helpers/ota/FirmwareInfo.h | 9 +- src/helpers/ota/OtaCli.cpp | 18 ++-- src/helpers/ota/OtaContext.h | 9 ++ src/helpers/ota/OtaFormat.h | 4 + src/helpers/ota/OtaSelf.cpp | 49 +++++++++- test/test_ota/test_ota_core.cpp | 37 +++++++ tools/mota/README.md | 7 ++ tools/mota/mota.py | 39 +++++++- tools/mota/motalib.py | 79 +++++++++++---- tools/mota/pio_endf.py | 37 +++++-- tools/mota/test_mota.py | 18 ++++ 15 files changed, 383 insertions(+), 102 deletions(-) diff --git a/.github/workflows/dev-firmware-rolling.yml b/.github/workflows/dev-firmware-rolling.yml index 421e8431..d33db75b 100644 --- a/.github/workflows/dev-firmware-rolling.yml +++ b/.github/workflows/dev-firmware-rolling.yml @@ -1,12 +1,14 @@ # Rolling DEV firmware release (fork convenience). # # On every push, rebuild a representative set of OTA-capable firmwares (ESP32 + nRF52, one per board family -# plus the RAK4631 / Heltec V3 test boards in every role) and replace the assets of ONE rolling prerelease -# (tag `dev-latest`). Each firmware ships with a full + same-image-delta `.mota` so the OTA format/transport -# can be exercised per board. UNSIGNED dev builds for testing only. +# plus the RAK4631 / Heltec V3 test boards in every role) and replace the `dev-latest` prerelease assets. +# Each firmware ships a full + same-image-delta `.mota` so the OTA format/transport can be exercised per +# board. UNSIGNED dev builds for testing only. # -# `prepare` (re)creates the empty release, then each matrix build uploads its own assets to it — so one -# board that fails to compile only loses its own cell, and there is no cross-job artifact plumbing. +# Design (atomic, never-empty): each board builds into a workflow ARTIFACT; a final `release` job collects +# them and ATOMICALLY recreates the release at the end. So if the run is cancelled by the next push, or some +# boards fail to compile, the PREVIOUS release stays intact until a full new set is ready (the old "empty +# the release first, then fill it" design left `dev-latest` empty whenever a build was cancelled/failed). # # To change which boards build, edit the `matrix.include` list below (env + its platform). Only OTA-capable # (ENABLE_OTA) envs make sense here. The full OTA set is ~308 envs; this is a curated ~one-per-board subset. @@ -29,39 +31,7 @@ env: FIRMWARE_VERSION: dev jobs: - prepare: - runs-on: ubuntu-latest - steps: - - name: Clone Repo - uses: actions/checkout@v6 - - name: (Re)create the empty rolling release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - gh release delete "$RELEASE_TAG" --yes --cleanup-tag 2>/dev/null || true - NOTES=$(cat <\`. - - A representative set of OTA-capable boards (ESP32 + nRF52). Each firmware ships a \`.full.mota\` - (the flashable image) and a \`.delta.mota\` (a same-image patch — intentionally tiny — to exercise - the delta path: sequential on ESP32, in-place on nRF52/RAK4631). - EOF - ) - gh release create "$RELEASE_TAG" \ - --title "Dev firmware (latest commit)" \ - --notes "$NOTES" \ - --prerelease \ - --target "$GITHUB_SHA" - build: - needs: prepare runs-on: ubuntu-latest strategy: fail-fast: false @@ -122,17 +92,66 @@ jobs: --out-prefix "out/${{ matrix.env }}-${FIRMWARE_VERSION}-${SHA}" --work /tmp \ || echo "::warning::.mota packaging failed for ${{ matrix.env }}" - - name: Upload assets to the rolling release + - name: Upload this board's artifact + uses: actions/upload-artifact@v7 + with: + name: fw-${{ matrix.env }} + path: out + if-no-files-found: ignore + retention-days: 5 + + # Collect every board's artifact and ATOMICALLY (re)create the rolling release. Runs even if some boards + # failed (if: always()), and only touches the release once assets are in hand — so a cancelled/partial run + # never empties dev-latest. + release: + needs: build + if: always() + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v6 + + - name: Collect built artifacts (via gh — no download-artifact version coupling) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p dist + gh run download "${{ github.run_id }}" --dir dist 2>/dev/null || true + # flatten dist/fw-/ -> dist/ + find dist -mindepth 2 -type f -exec mv -f -t dist {} + 2>/dev/null || true + find dist -mindepth 1 -type d -empty -delete 2>/dev/null || true + echo "Collected:"; ls -la dist || true + + - name: Recreate the rolling release (only if we actually have assets) env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail shopt -s nullglob - files=(out/*) + files=(dist/*) if [ ${#files[@]} -eq 0 ]; then - echo "::warning::no assets for ${{ matrix.env }}"; exit 0 + echo "::error::no firmware built — leaving the existing dev-latest release untouched" + exit 1 fi - echo "Uploading: ${files[*]}" - # retry once for transient API hiccups when many matrix jobs upload concurrently - gh release upload "$RELEASE_TAG" "${files[@]}" --clobber \ - || (sleep 5 && gh release upload "$RELEASE_TAG" "${files[@]}" --clobber) + NOTES=$(cat <\`. + + A representative set of OTA-capable boards (ESP32 + nRF52). Each firmware ships a \`.full.mota\` + (the flashable image) and a \`.delta.mota\` (a same-image patch — intentionally tiny — to exercise + the delta path: sequential on ESP32, in-place on nRF52/RAK4631). + EOF + ) + # delete + recreate so removed boards don't leave stale assets; assets are ready, so it's atomic + gh release delete "$RELEASE_TAG" --yes --cleanup-tag 2>/dev/null || true + gh release create "$RELEASE_TAG" "${files[@]}" \ + --title "Dev firmware (latest commit)" \ + --notes "$NOTES" \ + --prerelease \ + --target "$GITHUB_SHA" diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index bf58a8b0..d5a5fb4f 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -73,22 +73,36 @@ where a section names a source file, that file is the authoritative reference fo ## 2. Firmware image & the `EndF` trailer -Every OTA-capable build appends a 16-byte `EndF` trailer to its flashed image so a running node can -discover its own size/identity on any MCU (no linker symbols needed). Implemented by +Every OTA-capable build appends an `EndF` trailer to its flashed image so a running node can discover its +own size **and self-describing identity** on any MCU (no linker symbols needed). Implemented by `FirmwareInfo.cpp`; appended at build time by `tools/mota/pio_endf.py` (post-build hook). ``` flashed image = BODY (image bytes) || EndF trailer -EndF trailer (16 bytes): - off 0 4 "EndF" 45 6E 64 46 - off 4 4 body_len uint32 LE — length of BODY (excludes this 16-byte trailer) - off 8 8 body_hash sha2-256:8 of BODY +EndF trailer: + off 0 4 "EndF" 45 6E 64 46 + off 4 4 body_len uint32 LE — length of BODY (excludes the whole trailer) + off 8 8 body_hash sha2-256:8 of BODY + --- the 16 bytes above are the whole (legacy) trailer; the identity block below is optional: --- + off 16 4 "EnFx" 45 6E 46 78 — present iff this is an extended (identity) trailer + off 20 4 fw_version uint32 LE, packed MAJOR<<24|MINOR<<16|PATCH<<8|pre + off 24 4 target_id uint32 LE — sha2-256:4(pio_env): hardware + role + partition (fetch routing) + off 28 32 hw_id NUL-padded ASCII hardware tag (brick-safety), e.g. "RAK4631" + --- extended trailer = 60 bytes --- ``` -- **Size discovery:** scan flash from the partition top downward for the `EndF` marker; the byte before - it is the last BODY byte. (See `ota_self_firmware()`.) -- **Delta base matching:** a node's `body_hash` is read directly from its own `EndF`; a delta's - `base_hash` (§5) must equal it. No self-hashing pass at match time. +- **Self-describing identity (extended trailer).** `pio_endf.py` computes `target_id` from the PlatformIO + env name itself (so it's correct even without `build.sh`'s `-D MOTA_TARGET_ID`), `hw_id` from `MOTA_HW_ID`, + and `fw_version` from `FIRMWARE_VERSION`. The device reads them back (`ota_self_firmware()`), so a node's + advertised identity is correct regardless of how it was built — and the packaging tool reads them straight + from a raw `.bin` (no `--target-env`/`--fw-version` flags, no reliance on filenames; §9, §13). +- **Backward-compatible:** the first 16 bytes are unchanged, so the bootloader and any legacy reader (which + read only `[marker, marker+16)`) are unaffected by the extension. A reader detects the extension by the + `EnFx` magic at `+16`; absence ⇒ a 16-byte legacy trailer (identity unknown). +- **Size discovery:** scan flash from the partition top downward for the `EndF` marker; the byte before it + is the last BODY byte. The trailer is 60 bytes when `EnFx` follows, else 16. (See `ota_self_firmware()`.) +- **Delta base matching:** a node's `body_hash` is read directly from its own `EndF`; a delta's `base_hash` + (§5) must equal it. `body_hash` is over BODY only, so it is identical whether the trailer is 16 or 60 bytes. - **No circularity:** `EndF` hashes only the BODY, never itself. The "reconstructed image" referenced by the manifest is the full `BODY || EndF` (what gets flashed). @@ -420,12 +434,19 @@ All serving stays reactive and lowest-priority, so seeding never competes with r ## 9. Identity, trust & versioning -- **`target_id`** (4 B): compile-time `sha2-256:4(pio_env_name)` (little-endian uint32), injected as - `-D MOTA_TARGET_ID` by `build.sh` and read via `MainBoard::getOtaTargetId()`; `tools/mota` computes the - same from `--target-env`. The PlatformIO env name uniquely captures hardware **and** role/partition, so a - node auto-fetches only matching firmware. A manual `ota pull`/`want` can override target (deliberate role +- **`target_id`** (4 B): `sha2-256:4(pio_env_name)` (little-endian uint32). The env name uniquely captures + hardware **and** role/partition, so a node auto-fetches only matching firmware (a companion image is not + fetched onto a repeater even though it shares `hw_id`). It is **self-described in the firmware's EndF** + (§2, written by `pio_endf.py`) and read via `ota_self_firmware()`, so it is correct on any build; the + legacy `-D MOTA_TARGET_ID` / `MainBoard::getOtaTargetId()` path is the fallback. `tools/mota` reads it from + the firmware's EndF (or `--target-env`). A manual `ota pull`/`want` can override target (deliberate role switch); the `hw_id` brick-safety gate (§4) still applies at apply time. -- **`fw_version`:** packed comparable uint32 (`MAJOR<<24 | MINOR<<16 | PATCH<<8 | pre`). +- **`target_id` vs `hw_id`** — complementary, not redundant: `target_id` is the fetch-routing key + (hw + role + partition); `hw_id` is the human-readable brick-safety key (hardware only). Same board, two + roles ⇒ same `hw_id`, different `target_id`. +- **`fw_version`:** packed comparable uint32 (`MAJOR<<24 | MINOR<<16 | PATCH<<8 | pre`); also self-described + in EndF. `ota ls` decodes it for display and flags each update `[yours]` / `[other hw]` / `[?]` by + comparing the advertised `target_id` to the node's own. - **`hw_id`:** 32-byte NUL-padded ASCII hardware tag inside the signed head. The applier refuses a `.mota` whose `hw_id` differs from the device's own tag (empty on either side = permissive). Brick-safety independent of signature. diff --git a/docs/ota_user_guide.md b/docs/ota_user_guide.md index 76d0a2bb..3255b98b 100644 --- a/docs/ota_user_guide.md +++ b/docs/ota_user_guide.md @@ -59,10 +59,17 @@ and how recently it was seen. For example: ``` Updates nearby (2 src) — `ota get <#>` to download: - 1) v1.2.3 delta 3 nodes 5s - 2) v1.2.0 full 1 node 12s [downloading] + 1) v1.2.3 delta [yours] 3n 5s + 2) v1.2.0 full [other hw] 1n 12s [downloading] ``` +Each row shows the version, full-vs-delta, **whether it fits your node**, how many nodes have it, and how +long ago it was seen. The fit marker: + +- **[yours]** — built for your exact hardware **and** role; safe to install. +- **[other hw]** — a different board or role (e.g. a companion image, or another board). Don't install it. +- **[?]** — can't tell (a build with no target id set, e.g. a bare IDE build rather than a release build). + Run it again after a few seconds — discovery happens in the background, so the list fills in. Nothing is downloaded yet; this is just looking around. (`ota neighbors` / `ota updates` also work.) diff --git a/src/helpers/ota/FirmwareInfo.cpp b/src/helpers/ota/FirmwareInfo.cpp index a82f93d2..b2b35c9d 100644 --- a/src/helpers/ota/FirmwareInfo.cpp +++ b/src/helpers/ota/FirmwareInfo.cpp @@ -30,6 +30,16 @@ bool find_self_firmware(const uint8_t* region, uint32_t region_len, out.body_len = body_len; out.image_len = off + ENDF_LEN; memcpy(out.body_hash, region + off + 8, 8); + // Extended EndF? An "EnFx" block right after body_hash carries self-describing identity. The trailer is + // then 60 bytes, so the reconstructed image extends to off + ENDF_EXT_LEN. + if (off + ENDF_EXT_LEN <= region_len && memcmp(region + off + 16, ENDF_EXT_MAGIC, 4) == 0) { + out.has_ident = true; + out.fw_version = rd_u32(region + off + 20); + out.target_id = rd_u32(region + off + 24); + memcpy(out.hw_id, region + off + 28, 32); + out.hw_id[32] = 0; + out.image_len = off + ENDF_EXT_LEN; + } return true; } return false; diff --git a/src/helpers/ota/FirmwareInfo.h b/src/helpers/ota/FirmwareInfo.h index 112a26fb..a6231fa9 100644 --- a/src/helpers/ota/FirmwareInfo.h +++ b/src/helpers/ota/FirmwareInfo.h @@ -13,10 +13,15 @@ namespace ota { struct SelfFwInfo { bool valid = false; - uint32_t body_len = 0; // firmware body length (excludes the 16-byte EndF trailer) - uint32_t image_len = 0; // body_len + ENDF_LEN (what a delta base / full image hashes over) + uint32_t body_len = 0; // firmware body length (excludes the EndF trailer) + uint32_t image_len = 0; // body_len + trailer length (what a delta base / full image hashes over) uint32_t endf_offset = 0; // offset of the "EndF" marker within the region (== body_len) uint8_t body_hash[8] = {0}; // sha2-256:8 of the body (read from EndF; == a delta's base_hash) + // self-describing identity (extended EndF only; has_ident=false on a legacy 16-byte trailer) + bool has_ident = false; + uint32_t fw_version = 0; // packed MAJOR<<24|MINOR<<16|PATCH<<8|pre + uint32_t target_id = 0; // sha2-256:4(env) as uint32 — hw+role+partition (fetch routing) + char hw_id[33] = {0}; // readable hardware tag (NUL-terminated), e.g. "RAK4631" }; // Scan `region[0..region_len)` for the firmware's EndF trailer. The body starts at offset 0, so the diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index 3a73ff3d..b71b7dd0 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -106,10 +106,11 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board unsigned age = c.session_started_ms ? (unsigned)((millis() - c.session_started_ms) / 1000) : 0; snprintf(dl, sizeof dl, "download: %s %u/%u (%u%%) id=%s %us", state_word(fs), have, tot, pct, midhx, age); } - snprintf(reply, 160, "OTA | this fw %s (%uK) | %s | serving:%s (%u) | trusted keys:%u | target %08X", - selfhx, (unsigned)((s ? fi.image_len : 0) / 1024), dl, + const char* hw = (c.hw_id[0]) ? c.hw_id : "?"; + snprintf(reply, 160, "OTA | this fw %s (%uK) hw=%s | %s | serving:%s (%u) | keys:%u | target:%08X", + selfhx, (unsigned)((s ? fi.image_len : 0) / 1024), hw, dl, c.serving ? "on" : "off", (unsigned)c.manager.servedCount(), - (unsigned)c.allow.count(), (unsigned)board.getOtaTargetId()); + (unsigned)c.allow.count(), (unsigned)c.manager.target()); // ---- what's available around me (catalogued from beacons + OTA_HAVE), best/most-recent first ---- } else if (is_cmd(a, "neighbors|nbrs|updates|ls|n", &rest)) { @@ -121,16 +122,19 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board int n = snprintf(reply, CAP, "Updates nearby (%u src) — `ota get <#>` to download:", (unsigned)c.manager.sourceCount()); const uint8_t* cur = (c.manager.fetchState() != OtaManager::IDLE) ? c.manager.fetchManifestId() : nullptr; + uint32_t myt = c.manager.target(); // effective target (EndF identity if present, else build flag) uint32_t now = millis(); int shown = 0, more = 0; for (uint8_t i = 0; i < c.manager.catalogCount(); i++) { const OtaManager::CatRow* h = c.manager.catalogRow(i); - if (CAP - n < 40) { more++; continue; } + if (CAP - n < 48) { more++; continue; } bool on = cur && memcmp(cur, h->mid, 4) == 0; uint32_t age = (now - h->last_ms) / 1000; if (age > 99999) age = 99999; char ver[20]; ver_str(ver, sizeof ver, h->fw_version); - n += snprintf(reply + n, CAP - n, "\n %d) %s %s %u node%s %us%s", shown + 1, ver, - codec_kind(h->codec), (unsigned)h->n_seeders, h->n_seeders == 1 ? "" : "s", - (unsigned)age, on ? " [downloading]" : ""); + // is this update for THIS node (same hw+role)? '?' when either target id is unset (e.g. a manual build) + const char* fit = (myt == 0 || h->target_id == 0) ? "?" : (h->target_id == myt ? "yours" : "other hw"); + n += snprintf(reply + n, CAP - n, "\n %d) %s %s [%s] %un %us%s", shown + 1, ver, + codec_kind(h->codec), fit, (unsigned)h->n_seeders, (unsigned)age, + on ? " [downloading]" : ""); shown++; } if (more && n < CAP) snprintf(reply + n, CAP - n, "\n +%d more", more); diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index 2cf22342..117ce9e4 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -7,6 +7,7 @@ #include "SignerAllowlist.h" #include "OtaApply.h" #include "OtaFormat.h" +#include "OtaSelf.h" // ota_self_firmware() — prefer self-describing EndF identity at begin() #if defined(NRF52_PLATFORM) && defined(OTA_FLASH_STORE) #include "OtaStoreFlashNrf52.h" #elif defined(ESP32_PLATFORM) && defined(OTA_FLASH_STORE) @@ -153,6 +154,14 @@ struct OtaContext { } void begin(uint32_t target_id, OtaSend send, void* ctx, const char* hw = nullptr) { + // Prefer the firmware's SELF-DESCRIBING EndF identity (docs §2) over the build-flag values the caller + // passed — it's correct on any build (build.sh injection, bare IDE build, ...), so `ota ls`/`status` + // and fetch-routing show the right hardware/role instead of 0 / "". + SelfFwInfo _fi; + if (ota_self_firmware(_fi) && _fi.valid && _fi.has_ident) { + if (_fi.target_id) target_id = _fi.target_id; + if (_fi.hw_id[0]) hw = _fi.hw_id; + } manager.begin(target_id, send, ctx); if (hw) { strncpy(hw_id, hw, sizeof(hw_id) - 1); hw_id[sizeof(hw_id) - 1] = 0; } // a node only fetches firmware it can apply: ESP32 A/B -> sequential, nRF52 single-slot -> in-place diff --git a/src/helpers/ota/OtaFormat.h b/src/helpers/ota/OtaFormat.h index e9808ae5..1b7e080d 100644 --- a/src/helpers/ota/OtaFormat.h +++ b/src/helpers/ota/OtaFormat.h @@ -16,6 +16,10 @@ static const uint8_t MOTA_MAGIC[4] = { 'm', 'O', 'T', 'A' }; // 6D 4F 54 4 static const uint8_t MOTA_TRAILER[5] = { 'v', 'k', '4', '9', '6' }; // 76 6B 34 39 36 static const uint8_t ENDF_MAGIC[4] = { 'E', 'n', 'd', 'F' }; // 45 6E 64 46 static const uint32_t ENDF_LEN = 16; // marker(4)+body_len(4)+body_hash8(8) +// Extended EndF (docs/ota_protocol.md §2): the 16-byte trailer above, then a self-describing firmware +// identity block. The 16-byte prefix is unchanged, so the bootloader + legacy readers ignore the rest. +static const uint8_t ENDF_EXT_MAGIC[4] = { 'E', 'n', 'F', 'x' }; // 45 6E 46 78 +static const uint32_t ENDF_EXT_LEN = 60; // +EnFx(4)+fw_version(4)+target_id(4)+hw_id(32) // ---- manifest ------------------------------------------------------------- static const uint8_t MOTA_FORMAT_VER = 2; // v2 adds hw_id[32] (a human-readable hardware tag) diff --git a/src/helpers/ota/OtaSelf.cpp b/src/helpers/ota/OtaSelf.cpp index 30b0f732..863089db 100644 --- a/src/helpers/ota/OtaSelf.cpp +++ b/src/helpers/ota/OtaSelf.cpp @@ -44,10 +44,22 @@ bool ota_self_firmware(SelfFwInfo& out) { | ((uint32_t)buf[i+6] << 16) | ((uint32_t)buf[i+7] << 24); if (body_len != base + i) continue; // must sit immediately after a body of that length out.valid = true; - out.endf_offset = base + i; + out.endf_offset = body_len; out.body_len = body_len; out.image_len = body_len + ENDF_LEN; memcpy(out.body_hash, buf + i + 8, 8); + // Extended identity? Re-read the full trailer at the marker (it may straddle the chunk window, so + // the EnFx fields aren't reliably in `buf`). docs/ota_protocol.md §2. + uint8_t tr[ENDF_EXT_LEN]; + if (body_len + ENDF_EXT_LEN <= p->size && + esp_partition_read(p, body_len, tr, ENDF_EXT_LEN) == ESP_OK && + memcmp(tr + 16, ENDF_EXT_MAGIC, 4) == 0) { + out.has_ident = true; + out.fw_version = (uint32_t)tr[20] | ((uint32_t)tr[21]<<8) | ((uint32_t)tr[22]<<16) | ((uint32_t)tr[23]<<24); + out.target_id = (uint32_t)tr[24] | ((uint32_t)tr[25]<<8) | ((uint32_t)tr[26]<<16) | ((uint32_t)tr[27]<<24); + memcpy(out.hw_id, tr + 28, 32); out.hw_id[32] = 0; + out.image_len = body_len + ENDF_EXT_LEN; + } return true; } } @@ -96,7 +108,32 @@ static void wr_u32le(uint8_t* p, uint32_t v) { // Build (once) the full-image manifest + merkle leaves for the running firmware, cache them in `c`, and // hand the manager a flash-read callback for the payload. The image is read ONCE here to compute the // leaves + image_hash; thereafter a block REQ reads only that block (proof comes from the cached leaves). +// Pack the first "MAJOR.MINOR.PATCH" found in `s` into the comparable uint32 the manifest uses +// (MAJOR<<24 | MINOR<<16 | PATCH<<8). Returns 0 if there's no dotted number (e.g. a "dev-" build). +static uint32_t parse_fw_version(const char* s) { + if (!s) return 0; + for (; *s; s++) { // find the start of a "d.d" run + if (*s < '0' || *s > '9') continue; + const char* p = s; uint32_t a = 0, b = 0, d = 0; int dots = 0; + uint32_t* cur = &a; + for (; *p; p++) { + if (*p >= '0' && *p <= '9') { *cur = *cur * 10 + (uint32_t)(*p - '0'); } + else if (*p == '.' && dots < 2) { dots++; cur = (dots == 1) ? &b : &d; } + else break; + } + if (dots >= 1) return ((a & 0xFF) << 24) | ((b & 0xFF) << 16) | ((d & 0xFF) << 8); + s = p - 1; // a bare number, no dots — keep scanning + } + return 0; +} + bool ota_serve_self(OtaContext& c, uint32_t fw_version) { + // Derive our version from the build string when the caller didn't supply one, so the mOTA we advertise + // carries a real version (was hard-coded 0 -> peers saw "v0.0.0"). A dev build with no dotted number + // still reads 0 — the self-describing EndF identity (docs) is the durable fix for that. +#ifdef FIRMWARE_VERSION + if (fw_version == 0) fw_version = parse_fw_version(FIRMWARE_VERSION); +#endif SelfFwInfo fi; if (!ota_self_firmware(fi) || !fi.valid) return false; // 1 KB logical blocks (delivered as multiple LoRa fragments): 8x fewer merkle leaves than 128 B, so a @@ -128,16 +165,22 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { uint8_t image_hash[32]; sha.finalize(image_hash, 32); uint8_t root[4]; merkle_root(root, c.serve_self_leaves, bc); + // Prefer the SELF-DESCRIBING identity embedded in our own EndF (docs §2) over build flags / the param — + // it's correct regardless of how the firmware was built (build.sh injection, IDE, etc.). + uint32_t out_target = (fi.has_ident && fi.target_id) ? fi.target_id : c.manager.target(); + uint32_t out_ver = (fi.has_ident && fi.fw_version) ? fi.fw_version : fw_version; + const char* out_hw = (fi.has_ident && fi.hw_id[0]) ? fi.hw_id : c.hw_id; + uint8_t* m = c.serve_self_manifest; // assemble v2 manifest-minus-leaves (full, unsigned) = 93 bytes memset(m, 0, 96); m[0] = MOTA_FORMAT_VER; m[1] = MFLAG_FULL; m[2] = HASH_ALGO_SHA256; - wr_u32le(m + 3, c.manager.target()); wr_u32le(m + 7, fw_version); + wr_u32le(m + 3, out_target); wr_u32le(m + 7, out_ver); wr_u32le(m + 11, image_size); wr_u32le(m + 15, image_size); // full: payload == image m[19] = 10; // block_size_log2 = 10 (1024 B logical block) memcpy(m + 20, root, 4); memcpy(m + 24, image_hash, 32); m[56] = CODEC_FULL; - memcpy(m + 57, c.hw_id, strlen(c.hw_id) < 32 ? strlen(c.hw_id) : 32); // hw_id[32] (NUL-padded by memset) + memcpy(m + 57, out_hw, strlen(out_hw) < 32 ? strlen(out_hw) : 32); // hw_id[32] (NUL-padded by memset) memcpy(m + 89, APPROVAL_NOT, 4); // approval marker (fetching device's apply-gate handles it) return c.manager.serve_self(m, 93, c.serve_self_leaves, bc, c.serve_self_proof, (size_t)bc * 4, self_read_cb, nullptr); diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 75f357f8..c0ff2443 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -232,6 +232,43 @@ TEST(OtaFirmwareInfo, FindsEndFInImage) { EXPECT_EQ(0, std::memcmp(fi.body_hash, h, 8)); } +// Build a body || EXTENDED EndF (identity-carrying), the way pio_endf / motalib do. +static std::vector make_image_v2(const std::vector& body, uint32_t fw_version, + uint32_t target_id, const char* hw_id) { + std::vector img = make_image(body); // body + 16-byte base trailer + static const uint8_t EXT[4] = {'E','n','F','x'}; + img.insert(img.end(), EXT, EXT + 4); + for (int i = 0; i < 4; i++) img.push_back((uint8_t)(fw_version >> (8 * i))); + for (int i = 0; i < 4; i++) img.push_back((uint8_t)(target_id >> (8 * i))); + uint8_t hw[32] = {0}; size_t n = strlen(hw_id); if (n > 32) n = 32; memcpy(hw, hw_id, n); + img.insert(img.end(), hw, hw + 32); // -> 60-byte extended trailer + return img; +} + +// The self-describing identity in an extended EndF is parsed; a legacy 16-byte trailer reports no identity. +TEST(OtaFirmwareInfo, ParsesExtendedIdentity) { + std::vector body(2000); + for (size_t i = 0; i < body.size(); i++) body[i] = (uint8_t)(i * 13 + 5); + + auto img = make_image_v2(body, 0x01100000u, 0x04d413fdu, "RAK4631"); + std::vector region = img; region.resize(img.size() + 4096, 0xFF); + SelfFwInfo fi; + ASSERT_TRUE(find_self_firmware(region.data(), (uint32_t)region.size(), fi, /*verify_body=*/true)); + EXPECT_EQ(fi.body_len, body.size()); + EXPECT_EQ(fi.image_len, body.size() + 60); // extended trailer length + EXPECT_TRUE(fi.has_ident); + EXPECT_EQ(fi.fw_version, 0x01100000u); + EXPECT_EQ(fi.target_id, 0x04d413fdu); + EXPECT_STREQ(fi.hw_id, "RAK4631"); + + auto img1 = make_image(body); // legacy 16-byte trailer + std::vector r1 = img1; r1.resize(img1.size() + 64, 0xFF); + SelfFwInfo fi1; + ASSERT_TRUE(find_self_firmware(r1.data(), (uint32_t)r1.size(), fi1, true)); + EXPECT_FALSE(fi1.has_ident); + EXPECT_EQ(fi1.image_len, body.size() + 16); +} + TEST(OtaFirmwareInfo, IgnoresStagedMotaHigherInRegion) { // The firmware's own EndF must win even when a staged .mota (which embeds its own EndF) sits // above it in the same region — the body_len == offset check disambiguates. diff --git a/tools/mota/README.md b/tools/mota/README.md index 59d20bfe..d65a080c 100644 --- a/tools/mota/README.md +++ b/tools/mota/README.md @@ -49,6 +49,13 @@ $PY tools/mota/mota.py verify fw_v1.16.0_delta.mota --pub signer.priv.pub --bas `build` notes: - `--fw` may be a plain `.bin`; the tool appends `EndF` if absent (idempotent). +- **Self-describing identity (no flags / no filenames).** A firmware built by `pio_endf.py` carries its + `target_id`, `fw_version` and `hw_id` in an extended `EndF` trailer (docs/ota_protocol.md §2). `build` + reads them, so `--target-env`/`--fw-version`/`--hw-id` are **optional** — point `--fw` at any such `.bin` + (e.g. from a folder) and it packages with the right identity. Explicit flags still override. +- **Cross-hardware delta guard.** A delta is built only if the base and target firmware have the **same** + `hw_id` + `target_id` (read from their `EndF`, not the filenames). A mismatch is refused with a clear + reason; use `--force` to override deliberately. - For deltas, `base_hash` is taken from the base image's `EndF` and embedded so a device can confirm the delta applies to its current firmware. - `image_hash` (full SHA-256, signed) is the security anchor checked on the reconstructed image before diff --git a/tools/mota/mota.py b/tools/mota/mota.py index 9c87d3d3..9b336563 100644 --- a/tools/mota/mota.py +++ b/tools/mota/mota.py @@ -70,6 +70,17 @@ def cmd_build(args): image_hash = ml.mh32(new_image) image_size = len(new_image) + # Self-describing identity: if the firmware carries an extended EndF (target_id/fw_version/hw_id), use + # it as the default so a raw .bin from a folder packages correctly WITHOUT --target-env/--fw-version/ + # --hw-id (we can't rely on filenames). Explicit flags still override. + ident = ml.parse_endf_ident(new_image) + if args.target_env: target_id = ml.target_id_for_env(args.target_env) + elif args.target_id: target_id = _parse_target_id(args.target_id) + elif ident and ident.target_id: target_id = ident.target_id + else: sys.exit("no target: pass --target-env/--target-id, or build a firmware whose EndF carries one") + fw_version = ml.pack_version(args.fw_version) if args.fw_version else (ident.fw_version if ident else 0) + hw_id = args.hw_id or (ident.hw_id if ident else "") + codec_map = {"full": ml.CODEC_FULL, "sequential": ml.CODEC_DETOOLS_SEQUENTIAL, "inplace": ml.CODEC_DETOOLS_INPLACE} @@ -85,13 +96,30 @@ def cmd_build(args): if not args.base: sys.exit("delta codec requires --base ") old_image, base_hash = ml.ensure_endf(Path(args.base).read_bytes()) + # A delta is only applicable to the SAME hardware+role as the target. Verify via the firmwares' + # self-describing EndF identity (not the filenames), so we never ship a cross-HW delta that would + # brick a node. Skippable with --force for deliberate cross-target experiments. + base_ident = ml.parse_endf_ident(old_image) + if ident and base_ident: + mismatch = [] + if ident.hw_id and base_ident.hw_id and ident.hw_id != base_ident.hw_id: + mismatch.append(f"hw_id {base_ident.hw_id!r} (base) != {ident.hw_id!r} (target)") + if ident.target_id and base_ident.target_id and ident.target_id != base_ident.target_id: + mismatch.append(f"target_id {base_ident.target_id:#010x} != {ident.target_id:#010x}") + if mismatch and not args.force: + sys.exit("refusing cross-hardware delta (use --force to override):\n " + "\n ".join(mismatch)) + if mismatch: + print("WARNING: building a cross-hardware delta (--force): " + "; ".join(mismatch)) + elif not args.force: + print("note: base and/or target firmware has no EndF identity — cannot verify same-hardware " + "(build a firmware whose EndF carries identity, or pass --force to silence)") payload = _make_delta(old_image, new_image, args.codec, args.compression, args) sign_priv = _load_priv(args.sign) if args.sign else None manifest = ml.build_manifest( - target_id=_resolve_target_id(args), - fw_version=ml.pack_version(args.fw_version), + target_id=target_id, + fw_version=fw_version, image_size=image_size, payload=payload, block_size=args.block_size, @@ -100,7 +128,7 @@ def cmd_build(args): is_full=is_full, base_hash=base_hash, sign_priv=sign_priv, - hw_id=args.hw_id, + hw_id=hw_id, ) blob = ml.build_container(manifest, payload) Path(args.out).write_bytes(blob) @@ -195,7 +223,10 @@ def main(argv=None): b.add_argument("--target-id", help="target_id (0x.. or decimal)") b.add_argument("--target-env", help="PlatformIO env name; target_id = sha2-256:4(env) " "(matches build.sh / device getOtaTargetId)") - b.add_argument("--fw-version", required=True, help="e.g. 1.16.0 (or .pre as 1.16.0.2)") + b.add_argument("--fw-version", help="e.g. 1.16.0 (or .pre as 1.16.0.2). Optional: read from the " + "firmware's EndF identity if present.") + b.add_argument("--force", action="store_true", + help="build a delta even if base/target hardware identities differ (normally refused)") b.add_argument("--codec", choices=["full", "sequential", "inplace"], default="full") b.add_argument("--base", help="base firmware (.bin) for delta codecs") b.add_argument("--compression", default="crle", diff --git a/tools/mota/motalib.py b/tools/mota/motalib.py index c62cd771..f0969d27 100644 --- a/tools/mota/motalib.py +++ b/tools/mota/motalib.py @@ -109,37 +109,78 @@ def target_id_for_env(env_name: str) -> int: # EndF trailer # --------------------------------------------------------------------------- -def build_endf(body: bytes) -> bytes: - """The 16-byte EndF trailer for a firmware BODY.""" - return ENDF_MAGIC + struct.pack(" bytes: + """The EndF trailer for a firmware BODY: 16 bytes (legacy) or 60 bytes when `ident` is given. The + 16-byte prefix is identical either way, so the bootloader and legacy readers (which read only the + first 16 bytes) are unaffected by the extension.""" + base = ENDF_MAGIC + struct.pack(" int: + """Length of the trailing EndF (60 if extended, 16 if legacy, 0 if none/invalid).""" + if len(image) >= ENDF_EXT_LEN: + t = image[-ENDF_EXT_LEN:] + if (t[:4] == ENDF_MAGIC and struct.unpack("= ENDF_LEN: + t = image[-ENDF_LEN:] + if (t[:4] == ENDF_MAGIC and struct.unpack(" bool: - """True iff `image` ends with a self-consistent EndF trailer (image == BODY || EndF).""" - if len(image) < ENDF_LEN: - return False - trailer = image[-ENDF_LEN:] - if trailer[:4] != ENDF_MAGIC: - return False - body_len = struct.unpack(" Tuple[bytes, bytes]: """Return (body, body_hash8) for an image that ends with a valid EndF. Raises otherwise.""" - if not has_endf(image): + n = _endf_trailer_len(image) + if not n: raise ValueError("image has no valid EndF trailer") - return image[:-ENDF_LEN], image[-8:] + t = image[-n:] + return image[:-n], t[8:16] -def ensure_endf(image: bytes) -> Tuple[bytes, bytes]: - """Return (image_with_endf, body_hash8). Appends EndF if not already present.""" +def parse_endf_ident(image: bytes) -> Optional["FwIdent"]: + """The self-describing identity from an extended EndF, or None for a legacy/absent trailer.""" + if _endf_trailer_len(image) != ENDF_EXT_LEN: + return None + t = image[-ENDF_EXT_LEN:] + fw, tgt = struct.unpack(" Tuple[bytes, bytes]: + """Return (image_with_endf, body_hash8). Appends EndF (with `ident` if given) if not already present. + If the image already has a trailer it is kept as-is (we never rewrite an existing identity).""" if has_endf(image): - return image, image[-8:] + _, h8 = parse_endf(image) + return image, h8 body_hash8 = mh8(image) - return image + build_endf(image), body_hash8 + return image + build_endf(image, ident), body_hash8 # --------------------------------------------------------------------------- diff --git a/tools/mota/pio_endf.py b/tools/mota/pio_endf.py index 235e338c..61ad0e42 100644 --- a/tools/mota/pio_endf.py +++ b/tools/mota/pio_endf.py @@ -43,16 +43,39 @@ def _is_nrf52() -> bool: return False +def _cppdef(name): # value of a -D= build flag, or None + for d in env.get("CPPDEFINES", []): # noqa: F821 + if isinstance(d, (list, tuple)) and len(d) > 1 and d[0] == name: + return str(d[1]) + if d == name: + return "" + return None + + +def _firmware_ident(): + """Self-describing identity to embed in EndF (docs/ota_protocol.md §2): target_id is computed from the + PlatformIO env name (so it's correct even without build.sh's -D MOTA_TARGET_ID), hw_id from MOTA_HW_ID, + fw_version parsed from FIRMWARE_VERSION.""" + import re + target_id = ml.target_id_for_env(env["PIOENV"]) # noqa: F821 + hw_id = (_cppdef("MOTA_HW_ID") or "").replace("\\", "").strip().strip('"').strip("'") + ver_s = (_cppdef("FIRMWARE_VERSION") or "").replace("\\", "").strip().strip('"').strip("'") + m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?", ver_s) + fw_version = ml.pack_version(f"{m.group(1)}.{m.group(2)}.{m.group(3) or 0}") if m else 0 + return ml.FwIdent(fw_version=fw_version, target_id=target_id, hw_id=hw_id) + + def _append_endf(source, target, env): # raw .bin path (ESP32 / RP2040) path = str(target[0]) with open(path, "rb") as f: data = f.read() - out, h8 = ml.ensure_endf(data) + ident = _firmware_ident() + out, h8 = ml.ensure_endf(data, ident) if len(out) != len(data): with open(path, "wb") as f: f.write(out) - print(f"EndF: appended to {os.path.basename(path)} " - f"(body_len={len(data)} body_hash={h8.hex()})") + print(f"EndF: appended to {os.path.basename(path)} (body_len={len(data)} body_hash={h8.hex()} " + f"target={ident.target_id:#010x} hw='{ident.hw_id}' fw={ident.fw_version:#010x})") else: print(f"EndF: already present in {os.path.basename(path)} (no change)") @@ -66,15 +89,17 @@ def _append_endf_hex(source, target, env): # Intel-HEX path (nRF52: app f print("EndF: empty .hex, skipping"); return app_start, app_end = segs[0] # first (lowest) segment = the application image body = bytes(ih.tobinarray(start=app_start, size=app_end - app_start)) - out, h8 = ml.ensure_endf(body) + ident = _firmware_ident() + out, h8 = ml.ensure_endf(body, ident) if len(out) == len(body): print(f"EndF: already present in {os.path.basename(path)} (no change)"); return - trailer = out[len(body):] # the 16-byte EndF trailer + trailer = out[len(body):] # the EndF trailer (60 bytes with identity) for i, b in enumerate(trailer): ih[app_end + i] = b # write it right after the app's last byte ih.write_hex_file(path) print(f"EndF: appended to {os.path.basename(path)} at 0x{app_end:X} " - f"(app=0x{app_start:X}.. body_len={len(body)} body_hash={h8.hex()})") + f"(app=0x{app_start:X}.. body_len={len(body)} body_hash={h8.hex()} " + f"target={ident.target_id:#010x} hw='{ident.hw_id}' fw={ident.fw_version:#010x})") if _ota_enabled(): diff --git a/tools/mota/test_mota.py b/tools/mota/test_mota.py index f9b68660..ec1e1fda 100644 --- a/tools/mota/test_mota.py +++ b/tools/mota/test_mota.py @@ -61,6 +61,24 @@ def test_endf_rejects_garbage_tail(): assert not ml.has_endf(img) +def test_endf_extended_identity(): + body = _fw(3, 4096) + ident = ml.FwIdent(fw_version=ml.pack_version("1.16.0"), + target_id=ml.target_id_for_env("RAK_4631_repeater"), hw_id="RAK4631") + ext, h8 = ml.ensure_endf(body, ident) + assert len(ext) == len(body) + ml.ENDF_EXT_LEN # 60-byte extended trailer + assert ml.parse_endf(ext) == (body, h8) # body + body_hash still parse + # body_hash is over BODY only -> identical to the legacy trailer (delta base_hash stays valid) + assert h8 == ml.ensure_endf(body)[1] + gi = ml.parse_endf_ident(ext) + assert gi is not None and gi.hw_id == "RAK4631" + assert gi.target_id == ml.target_id_for_env("RAK_4631_repeater") + assert gi.fw_version == ml.pack_version("1.16.0") + # the 16-byte prefix equals the legacy trailer (bootloader-compatible); legacy images carry no identity + assert ext[len(body):len(body) + 16] == ml.ensure_endf(body)[0][len(body):] + assert ml.parse_endf_ident(ml.ensure_endf(body)[0]) is None + + # --- merkle ---------------------------------------------------------------- def test_merkle_single_block():