From 125ddac184a07683c5dcc8a648ca13b5f1630d4e Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 15:38:10 -0700 Subject: [PATCH] feat(ota): add partition-table signature handling for OTA updates Implemented functionality to generate and compare partition-table signatures during OTA updates. This enhancement ensures that the target build's partition layout matches the device's actual layout, improving the reliability of OTA updates and preventing issues related to partition changes. --- .../workflows/build-observer-firmwares.yml | 5 +- build.sh | 7 ++ examples/simple_repeater/MyMesh.cpp | 3 +- examples/simple_repeater/main.cpp | 5 ++ scripts/partition_signature.py | 44 +++++++++++ src/helpers/CommonCLI.cpp | 42 +++++++++-- src/helpers/ESP32Board.cpp | 74 ++++++++++++++++++- 7 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 scripts/partition_signature.py diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 0db674ae..6a0c70cf 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -159,6 +159,8 @@ jobs: run: | mkdir -p out find artifacts -type f -name '*.bin' -exec cp -f {} out/ \; + # Per-env partition-table signatures (for the slim manifest's OTA gate). + find artifacts -type f -name '*.partsig' -exec cp -f {} out/ \; echo "Collected binaries:"; ls -1 out - name: Compute Short SHA @@ -233,7 +235,8 @@ jobs: --config flasher/config.json \ --out-dir flasher/v \ --base-version "$FIRMWARE_VERSION" \ - --build "$BUILD_NUMBER" + --build "$BUILD_NUMBER" \ + --partsig-dir out printf '{\n "baseVersion": "%s",\n "build": %s\n}\n' \ "$FIRMWARE_VERSION" "$BUILD_NUMBER" > flasher/observer-build-counter.json echo "Build $FIRMWARE_VERSION.$BUILD_NUMBER" diff --git a/build.sh b/build.sh index 3f58d2a0..8ea83b40 100755 --- a/build.sh +++ b/build.sh @@ -188,6 +188,13 @@ build_firmware() { cp .pio/build/$1/firmware.uf2 out/${FIRMWARE_FILENAME}.uf2 2>/dev/null || true cp .pio/build/$1/firmware.zip out/${FIRMWARE_FILENAME}.zip 2>/dev/null || true + # Emit the partition-table signature (ESP32) for OTA partition-compatibility + # checks. Keyed by env name so the slim-manifest generator can find it; the + # firmware computes the same signature at runtime from its flashed table. + if [ -f ".pio/build/$1/partitions.bin" ]; then + python3 scripts/partition_signature.py ".pio/build/$1/partitions.bin" > "out/$1.partsig" 2>/dev/null || true + fi + } # firmwares containing $1 will be built diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index d283acbd..833c318f 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1452,10 +1452,11 @@ void MyMesh::loop() { // headroom, then flash: otaFromManifest reboots into the new image on success // (so this never returns); on any abort (already up to date, partition change, // download error) it returns and we resume the bridge. + Serial.println("OTA: starting update"); setBridgeState(false); char ota_reply[160]; if (!_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { - MESH_DEBUG_PRINTLN("ota update aborted: %s", ota_reply); + Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); setBridgeState(true); } } diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index 2ce056f5..c56c0257 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -82,6 +82,11 @@ void setup() { store.save("_main", the_mesh.self_id); } + // Print the running firmware version at boot so it's visible after an OTA + // reboot without having to issue `ver` manually. + Serial.print("Firmware: "); Serial.print(FIRMWARE_VERSION); + Serial.print(" (built "); Serial.print(FIRMWARE_BUILD_DATE); Serial.println(")"); + Serial.print("Repeater ID: "); mesh::Utils::printHex(Serial, the_mesh.self_id.pub_key, PUB_KEY_SIZE); Serial.println(); diff --git a/scripts/partition_signature.py b/scripts/partition_signature.py new file mode 100644 index 00000000..326b9e51 --- /dev/null +++ b/scripts/partition_signature.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Canonical signature of an ESP32 partition table (from a partitions.bin). + +Used to decide OTA partition compatibility: the observer firmware computes the +same signature at runtime from its *flashed* partition table (via esp_partition), +and `ota update` refuses only when the target build's signature differs from the +running device's — i.e. a real partition-table change, not a blanket flag. + +The signature MUST be computed identically here and in firmware +(src/helpers/ESP32Board.cpp). Definition: + + for each partition-table entry: (type, subtype, offset, size) + sort by offset ascending + format each as "%x:%x:%x:%x" (lowercase hex, no 0x, no padding) + join with "," + +partitions.bin layout: 32-byte records, each starting with magic 0xAA 0x50; the +trailing MD5 record (magic 0xEB 0xEB) and 0xFF padding are ignored. + +Usage: partition_signature.py # prints the signature +""" +import struct +import sys + + +def signature(bin_path: str) -> str: + data = open(bin_path, "rb").read() + entries = [] + for i in range(0, len(data) - 31, 32): + rec = data[i:i + 32] + if rec[0:2] != b"\xaa\x50": # not a partition entry (MD5 record / padding) + continue + ptype, subtype = rec[2], rec[3] + offset, size = struct.unpack("", file=sys.stderr) + sys.exit(2) + print(signature(sys.argv[1])) diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 367ee47c..99849b91 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -809,13 +809,41 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re // the live MQTT sessions even on no-PSRAM boards. No bridge bounce needed. _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); } else { - // Update is DEFERRED: the flash blocks the loop and then reboots, so it - // must run only AFTER this reply has gone out over the mesh — otherwise - // the requester never gets a confirmation. The app loop runs it shortly. - if (_callbacks->beginDeferredOtaUpdate()) { - strcpy(reply, "Beginning update... (node will reboot if successful)"); - } else { - strcpy(reply, "ERR: online OTA not available"); + // `ota update`: cheap pre-check first (plain HTTP, bridge stays up). Only + // schedule the real update — which tears the bridge down, flashes, and + // reboots — when an applicable build actually exists. otaFromManifest(dry) + // returns true iff so; otherwise it leaves the explanation (up to date / + // cable flash / error) in reply, which we send without disturbing the + // bridge or misleading the user with a "Beginning update..." that no-ops. + if (_board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply)) { + // reply now holds "update available: -> (N behind|new base)", + // where is "vX.Y.Z.B (hash)". Pull out for a friendlier + // start message. The "-> " ... trailing " (" framing is produced by + // ESP32Board::otaFromManifestImpl; ends at the LAST " (" (the + // "(N behind)"/"(new base)" suffix), since the version's own hash-paren + // comes before it. + char target[48] = {0}; + const char* arrow = strstr(reply, "-> "); + if (arrow) { + arrow += 3; + const char* suffix = nullptr; + for (const char* p = arrow; (p = strstr(p, " (")) != nullptr; p++) suffix = p; + size_t len = suffix ? (size_t)(suffix - arrow) : strlen(arrow); + if (len >= sizeof(target)) len = sizeof(target) - 1; + memcpy(target, arrow, len); + target[len] = 0; + } + // Update is DEFERRED so this ack goes out over the mesh before the flash + // blocks the loop and reboots (the app loop runs it shortly). + if (_callbacks->beginDeferredOtaUpdate()) { + if (target[0]) { + snprintf(reply, 160, "Updating to %s; reboots when done (~30s offline). Check 'ver' after.", target); + } else { + strcpy(reply, "Beginning update... (node will reboot if successful)"); + } + } else { + strcpy(reply, "ERR: online OTA not available"); + } } } #else diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 516c1422..5a457638 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -73,6 +73,7 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { #include #include #include +#include // Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle // builds still link; we check for presence at runtime. @@ -118,6 +119,41 @@ static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, in } } +// Canonical signature of the FLASHED partition table — MUST match +// scripts/partition_signature.py: each entry "type:subtype:offset:size" in +// lowercase hex, sorted by offset, joined by ','. Lets `ota update` compare the +// target build's partition layout (carried in the manifest as partSig) against +// what's actually on this device, instead of a blanket per-variant flag. +static void ota_partitionSignature(char* out, size_t out_sz) { + struct PE { uint8_t type, subtype; uint32_t off, size; } e[24]; + int n = 0; + esp_partition_iterator_t it = + esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, nullptr); + while (it != nullptr && n < (int)(sizeof(e) / sizeof(e[0]))) { + const esp_partition_t* p = esp_partition_get(it); + e[n].type = (uint8_t)p->type; + e[n].subtype = (uint8_t)p->subtype; + e[n].off = p->address; + e[n].size = p->size; + n++; + it = esp_partition_next(it); + } + esp_partition_iterator_release(it); // safe on NULL (loop exhausted) + // insertion sort by offset (matches the script's sort key) + for (int i = 1; i < n; i++) { + PE k = e[i]; + int j = i - 1; + while (j >= 0 && e[j].off > k.off) { e[j + 1] = e[j]; j--; } + e[j + 1] = k; + } + size_t pos = 0; + if (out_sz) out[0] = 0; + for (int i = 0; i < n && pos + 1 < out_sz; i++) { + pos += snprintf(out + pos, out_sz - pos, "%s%x:%x:%x:%x", + i ? "," : "", e[i].type, e[i].subtype, (unsigned)e[i].off, (unsigned)e[i].size); + } +} + // Parameters handed to the worker task; lives on otaFromManifest()'s stack, // which stays valid because that function blocks until the worker signals done. struct OtaTaskArgs { @@ -215,6 +251,8 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } } + if (!dry_run) { Serial.print("OTA: checking manifest "); Serial.println(murl); } + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with chunked encoding // and no Content-Length; the raw chunked stream corrupts the parse. HTTP/1.0 // yields a Connection: close, unframed body. @@ -245,7 +283,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char strncpy(avail_base, doc["baseVersion"] | "", sizeof(avail_base) - 1); strncpy(avail_hash, doc["hash"] | "", sizeof(avail_hash) - 1); int avail_build = doc["build"] | -1; - bool partition_change = doc["partitionChange"] | false; + bool legacy_partition_change = doc["partitionChange"] | false; + char manifest_partsig[256] = {0}; + strncpy(manifest_partsig, doc["partSig"] | "", sizeof(manifest_partsig) - 1); doc.clear(); if (!file_url[0]) { @@ -253,6 +293,19 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char return false; } + // Partition compatibility: prefer the precise per-build signature — compare the + // target build's partition layout (manifest partSig) to what's actually flashed + // on THIS device. Refuse only on a real mismatch (OTA can't rewrite the table). + // Fall back to the legacy bool for manifests that predate partSig. + bool partition_change; + if (manifest_partsig[0]) { + char dev_partsig[256]; + ota_partitionSignature(dev_partsig, sizeof(dev_partsig)); + partition_change = (strcmp(dev_partsig, manifest_partsig) != 0); + } else { + partition_change = legacy_partition_change; + } + // --- Determine current-vs-available -------------------------------------- // Our running version token (e.g. "v1.16.0.5"), i.e. current_ver up to the // first '-' (which precedes "-observer-"). @@ -298,6 +351,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char const char* pc_note = partition_change ? " [partition change: cable flash]" : ""; // --- Report (dry run / `ota check`) -------------------------------------- + // Returns true iff an OTA-applicable update is available (not up-to-date and not + // a partition-change build). `ota check` ignores the return and just shows the + // reply; `ota update` uses it to decide whether to actually schedule the flash. if (dry_run) { if (up_to_date) { snprintf(reply, 160, "up to date: %s", avail_disp); @@ -308,7 +364,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } else { snprintf(reply, 160, "update available: %s -> %s%s", own_disp, avail_disp, pc_note); } - return true; + return (!up_to_date && !partition_change); } // --- Gates (real `ota update`) ------------------------------------------- @@ -322,6 +378,8 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } // --- Stream the .bin (the manifest's full URL) into the inactive OTA slot - + Serial.printf("OTA: update %s -> %s\n", own_disp, avail_disp); + Serial.print("OTA: downloading "); Serial.println(file_url); inhibit_sleep = true; // keep awake through the flash WiFiClientSecure uclient; @@ -332,6 +390,17 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char #endif uclient.setTimeout(20000); + // Console progress to the USB serial (always on; MESH_DEBUG is off on the default + // observer profile). Non-capturing lambdas + a file-static decile, so the global + // httpUpdate object never holds a dangling reference after this function returns. + static int ota_progress_decile; + ota_progress_decile = -1; + httpUpdate.onProgress([](int cur, int total) { + if (total <= 0) return; + int d = (int)((int64_t)cur * 10 / total); + if (d != ota_progress_decile) { ota_progress_decile = d; Serial.printf("OTA: %d%%\n", d * 10); } + }); + httpUpdate.onEnd([]() { Serial.println("OTA: write complete, rebooting..."); }); httpUpdate.rebootOnUpdate(true); // reboots into the new image on success t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); @@ -339,6 +408,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char inhibit_sleep = false; snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret, httpUpdate.getLastErrorString().c_str()); + Serial.print("OTA: FAILED - "); Serial.println(reply); return false; #endif // OTA_MANIFEST_BASE && OTA_VARIANT }