From a1943a8eac75ec7b29b486889bd404861de5356a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:37:49 -0700 Subject: [PATCH 1/3] fix(ota): always offer another channel's image, even from the same commit Prod and beta are often built from one commit (e.g. after a dev->prod merge), so a hash match does not mean the images are the same: the native base differs. After the flash the target channel is native, so this cannot loop. --- src/helpers/ESP32Board.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 206a3a7c..d36661d0 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -334,13 +334,14 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr bool same_base = (own_base[0] && avail_base[0] && strcmp(own_base, avail_base) == 0); bool have_builds = (own_build >= 0 && avail_build >= 0); bool diff_base = (own_base[0] && avail_base[0] && !same_base); - // Build counters are per channel, so build numbers only compare within the native channel. + // Another channel's image always differs (its native base does), even from the same + // commit; build counters are per channel, so build numbers only compare natively. bool cross_channel = (strcmp(manifest_base, OTA_MANIFEST_BASE) != 0); int behind = 0; bool up_to_date; if (cross_channel) { - up_to_date = hash_equal; + up_to_date = false; } else if (same_base && have_builds) { behind = avail_build - own_build; up_to_date = (behind <= 0); From f652e7d185f8752bb1f8b8a73fb97a8031a01a65 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:37:49 -0700 Subject: [PATCH 2/3] fix(ota): flash the channel ota update checked, not a later ota branch The deferred flash re-read the channel preference ~2.5 s after ota update acknowledged its target; capture it when the update is scheduled. --- examples/simple_repeater/MyMesh.cpp | 2 +- examples/simple_repeater/MyMesh.h | 2 ++ examples/simple_room_server/MyMesh.cpp | 2 +- examples/simple_room_server/MyMesh.h | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index b3789657..be201fb4 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1741,7 +1741,7 @@ void MyMesh::loop() { Serial.println("OTA: aborted, MQTT stop did not complete cleanly - resuming bridge"); otaAlert("OTA aborted: MQTT stop unclean, bridge resumed"); setBridgeState(true); - } else if (!_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { + } else if (!_cli.getBoard()->otaFromManifest(ota_resolve_base(_ota_update_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); char ota_alert_msg[160]; snprintf(ota_alert_msg, sizeof(ota_alert_msg), "OTA aborted: %s", ota_reply); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index a232ef52..c1e03a8e 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -123,6 +123,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks CayenneLPP telemetry; unsigned long set_radio_at, revert_radio_at; unsigned long _ota_update_at = 0; // deferred `ota update` fire time (0 = none scheduled) + uint8_t _ota_update_channel = 0; // channel `ota update` checked; a later `ota branch` cannot retarget it float pending_freq; float pending_bw; uint8_t pending_sf; @@ -441,6 +442,7 @@ public: bool beginDeferredOtaUpdate() override { _ota_update_at = millis() + 2500; if (_ota_update_at == 0) _ota_update_at = 1; // 0 means "none" + _ota_update_channel = _prefs.ota_channel; #if defined(WITH_MQTT_BRIDGE) // Broadcast START now, while the loop still runs (the 2.5 s reply window): // the deferred flash blocks the loop and, on success, reboots — so a start diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 2e97d491..079e43fa 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1617,7 +1617,7 @@ void MyMesh::loop() { } char ota_reply[160]; - if (may_flash && !_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { + if (may_flash && !_cli.getBoard()->otaFromManifest(ota_resolve_base(_ota_update_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted - "); Serial.println(ota_reply); may_flash = false; } diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index b0eb4e8f..ef7fb515 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -136,6 +136,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks TransportKey default_scope; unsigned long set_radio_at, revert_radio_at; unsigned long _ota_update_at = 0; // deferred `ota update` fire time (0 = none scheduled) + uint8_t _ota_update_channel = 0; // channel `ota update` checked; a later `ota branch` cannot retarget it float pending_freq; float pending_bw; uint8_t pending_sf; @@ -433,6 +434,7 @@ public: bool beginDeferredOtaUpdate() override { _ota_update_at = millis() + 2500; if (_ota_update_at == 0) _ota_update_at = 1; // 0 means "none" + _ota_update_channel = _prefs.ota_channel; return true; } From ed8af1b7ca317785cecb164b09368b309122081d Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:46:46 -0700 Subject: [PATCH 3/3] fix(ota): refuse a channel switch to a build that cannot carry this node A switch can land on an older build: e.g. prod cannot read dev's /mqtt.json, and prod's M7 image has no Ethernet. Each image carries an ota-compat tag (state generation + transports). A switch is flashed without rebooting, the new image's tag is read (bounded by its length), and boot is pointed back at the running image unless the target keeps this node's state and transports. CI now requires the tag in every published binary. --- scripts/verify_ota_channel.py | 12 ++++- src/helpers/ESP32Board.cpp | 58 +++++++++++++++++++++- src/helpers/OtaChannel.h | 54 ++++++++++++++++++++ test/test_ota_channel/test_ota_channel.cpp | 33 ++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) diff --git a/scripts/verify_ota_channel.py b/scripts/verify_ota_channel.py index 15d91c26..6b74030b 100644 --- a/scripts/verify_ota_channel.py +++ b/scripts/verify_ota_channel.py @@ -2,7 +2,8 @@ """Verify every built observer binary is baked for the expected OTA channel. Observer builds carry three manifest bases (src/helpers/OtaChannel.h), each stored -behind a tag so it can be read back from the binary: +behind a tag so it can be read back from the binary (plus an ota-compat tag, which a +channel switch reads from the downloaded image before booting it): ota-base-native: the channel the build OTAs from by default ota-base-stable: production, the target of `ota branch prod` @@ -26,6 +27,7 @@ import re import sys TAG_RE = re.compile(rb"ota-base-(native|stable|dev):([\x21-\x7e]*)\x00") +COMPAT_RE = re.compile(rb"ota-compat:([0-9]+(?:\+[a-z]+)*)\x00") def read_tags(data): @@ -49,6 +51,10 @@ def check(data, expect, native_url, stable_url, dev_url): found = sorted(tags[name]) if found != [want]: problems.append("ota-base-%s: expected [%s], found %s" % (name, want, found or "nothing")) + # Without its compat tag a build cannot be the target of a channel switch. + compat = sorted({m.group(1).decode() for m in COMPAT_RE.finditer(data)}) + if len(compat) != 1: + problems.append("ota-compat: expected one value, found %s" % (compat or "nothing")) return problems @@ -62,7 +68,7 @@ def self_test(): P, B = "https://h/v", "https://h/beta/v" def blob(native, stable=P, dev=B): - return b"\x00junk\x00ota-base-native:%s\x00ota-base-stable:%s\x00ota-base-dev:%s\x00" % ( + return b"\x00junk\x00ota-base-native:%s\x00ota-base-stable:%s\x00ota-base-dev:%s\x00ota-compat:1\x00" % ( native.encode(), stable.encode(), dev.encode()) cases = [ @@ -74,6 +80,8 @@ def self_test(): ("wrong dev URL fails", blob(P, dev="https://other/v"), "prod", P, False), ("conflicting native tags fail", blob(P) + blob(B), "prod", P, False), ("workflow URL off-channel fails", blob(P), "prod", B, False), + ("missing compat tag fails", blob(P).replace(b"ota-compat:1", b"ota-compat:"), "prod", P, False), + ("conflicting compat tags fail", blob(P) + b"ota-compat:2+eth\x00", "prod", P, False), ] failed = 0 for name, data, expect, native, ok in cases: diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index d36661d0..7c013c23 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -78,6 +78,9 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { #include #include #include +#include +#include +#include "OtaChannel.h" // Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle // builds still link; we check for presence at runtime. @@ -123,6 +126,31 @@ static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, in } } +// Read the compat tag of the image just written to `part`, bounded by the image's own +// length: bytes past it can be a stale, tagged image from an earlier flash. +static bool ota_readImageCompat(const esp_partition_t* part, OtaCompat* out) { + const esp_partition_pos_t pos = { part->address, part->size }; + esp_image_metadata_t meta; + if (esp_image_verify(ESP_IMAGE_VERIFY_SILENT, &pos, &meta) != ESP_OK) return false; + static const size_t kChunk = 2048, kTail = 64; // tail > tag + value, so a split tag is seen whole + static uint8_t buf[kTail + kChunk]; + size_t have = 0; + for (uint32_t off = 0; off < meta.image_len;) { + size_t n = meta.image_len - off; + if (n > kChunk) n = kChunk; + if (esp_partition_read(part, off, buf + have, n) != ESP_OK) return false; + have += n; + off += n; + const char* v = ota_compat_find(buf, have); + if (v) return ota_compat_parse(v, out); + if (have > kTail) { + memmove(buf, buf + have - kTail, kTail); + have = kTail; + } + } + return false; +} + // Canonical signature of the FLASHED partition table — MUST match // scripts/partition_signature.py: each entry "type:subtype:offset:size" in // lowercase hex, sorted by offset, joined by ','. Lets `ota update` compare the @@ -412,9 +440,35 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr int d = (int)((int64_t)cur * 10 / total); if (d != ota_progress_decile) { ota_progress_decile = d; Serial.printf("OTA: %d%%\n", d * 10); } }); - httpUpdate.onEnd([]() { Serial.println("OTA: write complete, rebooting..."); }); - httpUpdate.rebootOnUpdate(true); // reboots into the new image on success + httpUpdate.onEnd([]() { Serial.println("OTA: write complete"); }); + // A channel switch is checked before it may boot (below); native updates reboot here. + httpUpdate.rebootOnUpdate(!cross_channel); t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); + if (ret == HTTP_UPDATE_OK && cross_channel) { + const esp_partition_t* running = esp_ota_get_running_partition(); + const esp_partition_t* target = esp_ota_get_boot_partition(); + OtaCompat own = {0, 0}, img = {0, 0}; + const char* volatile own_tag = ota_compat_tag; // volatile: the tag must stay in the image + ota_compat_parse(own_tag + sizeof(OTA_COMPAT_TAG) - 1, &own); + bool tagged = target && target != running && ota_readImageCompat(target, &img); + if (tagged && ota_compat_ok(own, img)) { + Serial.println("OTA: channel switch compatible, rebooting..."); + delay(100); + ESP.restart(); + } + // Point boot back at the running image so the refused build never starts. + bool reverted = (esp_ota_set_boot_partition(running) == ESP_OK); + inhibit_sleep = false; + if (!tagged) { + snprintf(reply, 160, "ERR: channel switch refused: %s has no compat tag; cable flash%s", + avail_disp, reverted ? "" : " [boot revert FAILED]"); + } else { + snprintf(reply, 160, "ERR: channel switch refused: target compat %d/%x < this node %d/%x; cable flash%s", + img.gen, img.caps, own.gen, own.caps, reverted ? "" : " [boot revert FAILED]"); + } + Serial.print("OTA: "); Serial.println(reply); + return false; + } // Only reached on failure (success reboots inside update()). inhibit_sleep = false; diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h index 2918d83d..974cc4ba 100644 --- a/src/helpers/OtaChannel.h +++ b/src/helpers/OtaChannel.h @@ -76,3 +76,57 @@ static inline bool ota_parse_channel(const char* arg, uint8_t* out) { if (strcmp(arg, "default") == 0) { *out = OTA_CH_NATIVE; return true; } return false; } + +// Compatibility tag, read from a downloaded image before a channel switch boots it. +// A switch can land on an older build, which would boot without state it cannot read +// or without a transport this node depends on. Bump OTA_STATE_GEN when a build starts +// storing state that older builds cannot read. +// 1: /prefs.json + /mqtt_prefs +#ifndef OTA_STATE_GEN +#define OTA_STATE_GEN 1 +#endif +#define OTA_CAP_ETH 0x01 // carries MQTT over Ethernet +#if defined(NETWORK_PREFER_ETHERNET) +#define OTA_CAPS_STR "+eth" +#else +#define OTA_CAPS_STR "" +#endif +#define OTA_STR_(x) #x +#define OTA_STR(x) OTA_STR_(x) +#define OTA_COMPAT_TAG "ota-compat:" +static const char ota_compat_tag[] = OTA_COMPAT_TAG OTA_STR(OTA_STATE_GEN) OTA_CAPS_STR; + +struct OtaCompat { + int gen; + uint8_t caps; +}; + +// Parse the tag value "[+cap...]"; unknown caps are ignored. +static inline bool ota_compat_parse(const char* s, OtaCompat* out) { + if (*s < '0' || *s > '9') return false; + out->gen = 0; + out->caps = 0; + while (*s >= '0' && *s <= '9') out->gen = out->gen * 10 + (*s++ - '0'); + while (*s == '+') { + const char* cap = ++s; + while (*s && *s != '+') s++; + if ((size_t)(s - cap) == 3 && memcmp(cap, "eth", 3) == 0) out->caps |= OTA_CAP_ETH; + } + return *s == 0; +} + +// Find the tag in an image chunk; returns the NUL-terminated value text, or nullptr when +// the chunk holds no complete tag. +static inline const char* ota_compat_find(const uint8_t* buf, size_t len) { + const size_t tag_len = sizeof(OTA_COMPAT_TAG) - 1; + for (size_t i = 0; i + tag_len < len; i++) { + if (memcmp(buf + i, OTA_COMPAT_TAG, tag_len) != 0) continue; + if (memchr(buf + i + tag_len, 0, len - i - tag_len)) return (const char*)buf + i + tag_len; + } + return nullptr; +} + +// A target must read this node's state and keep every transport this node has. +static inline bool ota_compat_ok(const OtaCompat& own, const OtaCompat& target) { + return target.gen >= own.gen && (target.caps & own.caps) == own.caps; +} diff --git a/test/test_ota_channel/test_ota_channel.cpp b/test/test_ota_channel/test_ota_channel.cpp index d2a4e482..75ce673d 100644 --- a/test/test_ota_channel/test_ota_channel.cpp +++ b/test/test_ota_channel/test_ota_channel.cpp @@ -40,6 +40,39 @@ TEST(OtaChannel, NativeChannelNameMatchesBase) { EXPECT_STREQ(ota_native_channel_name(), "prod"); } +TEST(OtaCompat, OwnTagParses) { + OtaCompat own; + ASSERT_TRUE(ota_compat_parse(ota_compat_tag + sizeof(OTA_COMPAT_TAG) - 1, &own)); + EXPECT_EQ(own.gen, OTA_STATE_GEN); + EXPECT_EQ(own.caps, 0); +} +TEST(OtaCompat, ParsesCapsAndRejectsJunk) { + OtaCompat c; + ASSERT_TRUE(ota_compat_parse("12+eth+future", &c)); + EXPECT_EQ(c.gen, 12); + EXPECT_EQ(c.caps, OTA_CAP_ETH); + EXPECT_FALSE(ota_compat_parse("", &c)); + EXPECT_FALSE(ota_compat_parse("x1", &c)); + EXPECT_FALSE(ota_compat_parse("2eth", &c)); +} +TEST(OtaCompat, FindsCompleteTagOnly) { + const char img[] = "\xe9junk\0ota-compat:2+eth\0tail"; + const char* v = ota_compat_find((const uint8_t*)img, sizeof(img)); + ASSERT_NE(v, nullptr); + EXPECT_STREQ(v, "2+eth"); + const char cut[] = "junk ota-compat:2+e"; // value runs past the chunk end + EXPECT_EQ(ota_compat_find((const uint8_t*)cut, sizeof(cut) - 1), nullptr); + const char none[] = "ota-compat"; + EXPECT_EQ(ota_compat_find((const uint8_t*)none, sizeof(none)), nullptr); +} +TEST(OtaCompat, TargetMustKeepStateAndTransports) { + EXPECT_TRUE(ota_compat_ok({2, 0}, {2, 0})); + EXPECT_TRUE(ota_compat_ok({1, 0}, {2, OTA_CAP_ETH})); + EXPECT_FALSE(ota_compat_ok({2, 0}, {1, 0})); // cannot read /mqtt.json + EXPECT_FALSE(ota_compat_ok({2, OTA_CAP_ETH}, {2, 0})); // drops Ethernet + EXPECT_TRUE(ota_compat_ok({2, OTA_CAP_ETH}, {3, OTA_CAP_ETH})); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();