From da204b8fee41e9fbc2460742fc310a1277547017 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 9 Aug 2026 12:04:55 -0700 Subject: [PATCH 01/93] feat(mqtt): persist observer preferences as JSON --- MQTT_IMPLEMENTATION.md | 2 +- MQTT_INTERNALS.md | 139 +++--- examples/simple_repeater/MyMesh.cpp | 4 +- examples/simple_repeater/MyMesh.h | 5 + examples/simple_room_server/MyMesh.cpp | 4 +- examples/simple_room_server/MyMesh.h | 5 + src/helpers/CommonCLI.cpp | 420 +++++++++++++++--- src/helpers/CommonCLI.h | 22 +- src/helpers/CommonCLI_Observer.cpp | 190 +++++--- src/helpers/ConfigSerializer.cpp | 136 +++++- src/helpers/ConfigSerializer.h | 81 +++- src/helpers/MQTTDefaults.h | 2 +- src/helpers/MQTTObserverValidation.h | 17 + src/helpers/MQTTPacketFilter.h | 8 +- src/helpers/MQTTPrefsAtomicStore.h | 49 +- src/helpers/MQTTPrefsCodec.h | 166 ++++++- src/helpers/MQTTPrefsRecovery.h | 40 +- src/helpers/MQTTPrefsSerializer.h | 412 +++++++++++++++++ src/helpers/MQTTPrefsStorage.h | 71 ++- src/helpers/bridges/MQTTBridge.h | 2 +- test/README.md | 3 +- .../test_config_serializer.cpp | 17 + .../test_mqtt_prefs_atomic_store.cpp | 139 +++++- .../test_mqtt_prefs_codec.cpp | 80 +++- .../test_mqtt_prefs_serializer.cpp | 358 +++++++++++++++ .../test_observer_validation.cpp | 12 + .../test_webconfig_batch.cpp | 1 + 27 files changed, 2131 insertions(+), 254 deletions(-) create mode 100644 src/helpers/MQTTPrefsSerializer.h create mode 100644 test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 0df96e67..f18fed75 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -285,7 +285,7 @@ flash regardless of target. #### Compile-time fresh-install defaults (`src/helpers/MQTTDefaults.h`) -Optional PlatformIO `build_flags` override defaults written when `/mqtt_prefs` is first created. They do **not** change existing saved prefs on upgrade or reflash (unless `/mqtt_prefs` is erased). +Optional PlatformIO `build_flags` override defaults used when `/mqtt.json` is first created. They do **not** change existing saved prefs on upgrade or reflash (unless `/mqtt.json` is erased). | Macro | Default | Notes | |-------|---------|-------| diff --git a/MQTT_INTERNALS.md b/MQTT_INTERNALS.md index d0de0398..005de530 100644 --- a/MQTT_INTERNALS.md +++ b/MQTT_INTERNALS.md @@ -8,7 +8,8 @@ Developer-facing notes on how the MQTT observer feature is structured in the cod - `src/helpers/bridges/MQTTBridge.h` - MQTT bridge class definition - `src/helpers/bridges/MQTTBridge.cpp` - MQTT bridge implementation - `src/helpers/MQTTPresets.h` - Preset definitions, CA certificates, and lookup functions -- `src/helpers/MQTTDefaults.h` - Compile-time defaults for fresh `/mqtt_prefs` +- `src/helpers/MQTTDefaults.h` - Compile-time defaults for fresh `/mqtt.json` +- `src/helpers/MQTTPrefsSerializer.h` - Versioned, semantic observer JSON schema - `src/helpers/MQTTMessageBuilder.h` - JSON message formatting utilities - `src/helpers/MQTTMessageBuilder.cpp` - JSON message formatting implementation - `src/helpers/JWTHelper.h` - JWT token generation for Ed25519-based authentication @@ -30,8 +31,9 @@ The observer feature is kept out of upstream-tracked files through three mechani `resolveAlertScope`, `beginDeferredOtaUpdate`). The example apps override them behind `#ifdef WITH_MQTT_BRIDGE`. - **Separate settings file** — observer settings (MQTT slots, WiFi, timezone, SNMP, - radio watchdog, fault alerts) live in the `MQTTPrefs` struct persisted to - `/mqtt_prefs`, keeping `NodePrefs` / `/com_prefs` aligned with the upstream layout. + radio watchdog, fault alerts) live in the runtime `MQTTPrefs` object and are + field-serialized to `/mqtt.json`, keeping `NodePrefs` / `/prefs.json` aligned with + upstream. The two files are independent transactions, not one atomic snapshot. Remaining integration points in upstream files: - `examples/simple_repeater/MyMesh.{h,cpp}`, `examples/simple_room_server/MyMesh.{h,cpp}` - @@ -164,75 +166,104 @@ which packet events non-MQTT bridges capture. The MQTT bridge ignores `bridge.so favour of independent `mqtt.rx` / `mqtt.tx` controls. Everything MQTT-specific lives under `mqtt.*` (shared settings), `mqttN.*` (per-slot broker config), `wifi.*`, and `timezone.*`. -### `/mqtt_prefs` file format +### `/mqtt.json` file format -`/mqtt_prefs` is written with an 8-byte `MQTTPrefsHeader` (`magic`, `version`, -`payload_len`) followed by the raw `MQTTPrefs` payload. The magic is -`{0xF5, 'M', 'Q', 'P'}` — its leading non-ASCII byte can never collide with the first -bytes of a legacy (headerless) file, whose payload begins with the `mqtt_origin` -string. Bump `MQTT_PREFS_VERSION` when the payload layout changes incompatibly; a file -whose version this firmware doesn't recognize is left untouched and the in-memory prefs -fall back to defaults (no downgrade, no misread). `saveMQTTPrefs()` also refuses to -write while such a file is present (`_mqtt_prefs_hold`), so a `set` command after a -firmware downgrade can't clobber the newer config — observer settings changed in that -state simply don't persist. The frozen legacy layouts are pinned with `static_assert`s -in `MQTTPrefsStorage.h`, so every target build re-verifies the fleet's file offsets. +Observer preferences use the same `ConfigSerializer` object notation as upstream +`/prefs.json`: semantic unquoted keys, quoted strings, decimal numbers, and nested +objects. Schema version 1 requires the root `version:1` field. The main shape is: -Adding a field to the current version stays backward compatible: append it to the end -of `MQTTPrefs`, give the older exact payload length an explicit decoder boundary, and -leave the missing tail at its default. The packet-filter addition follows that rule: -the prior 2864-byte v1 payload loads with all six filters set to `all`, while the -full payload is 2876 bytes. +```text +{version:1, + wifi:{ssid:"...",password:"...",power_save:1}, + time:{timezone:"...",utc_offset:0,ntp_server:"..."}, + mqtt:{origin:"...",iata:"SEA",packets_enabled:1,raw_enabled:0, + tx_enabled:2,rx_enabled:1, + status:{enabled:1,interval_ms:300000}, + neighbors:{enabled:0,interval_ms:86400000}, + owner:{public_key:"...",email:"..."}, + slot1:{preset:"analyzer-us",host:"",port:0,username:"",password:"", + token:"",topic:"",audience:"",packet_filter:65535}, + ... slot2 through slot6 ...}, + snmp:{enabled:0,community:"public"}, + radio:{watchdog_min:5}, + alert:{enabled:0,psk_hex:"",wifi_minutes:30,mqtt_minutes:240, + rate_limit_min:60,hashtag:"",region:""}} +``` -#### The downgrade contract +Keys may contain digits after their first character, which permits the readable +`slot1` ... `slot6` names. Every schema key fits `ConfigSerializer`'s 15-character +visible-key limit. Known strings and numbers use strict parsing: duplicates, overlong +strings, malformed decimals, and overflow reject the complete file. A supported file +with a semantically out-of-range value is repaired to that field's safe default and +rewritten atomically. Unknown fields are ignored only within the serializer's general +limits (15-character keys, 127-byte decoded values, and six nested object levels below the root). -**Within a version tag the layout is append-only, and a longer payload is always -readable.** A file written by a later build starts with this binary's exact baseline, -so `classify()` reads that prefix and ignores the tail. A downgraded node keeps its -WiFi credentials, broker slots, and every other setting it understands; the only thing -it loses is the settings the newer build added. +Loading always starts with defaults and parses into a separate heap scratch object. +The live preferences change only after the complete file parses, has an explicit +supported version, and passes validation. A missing/future version, syntax error, +overlength value, or allocation/read failure leaves `/mqtt.json` untouched, runs this +boot on defaults, and holds observer saves so a later CLI/WebConfig write cannot erase +the opaque source. If an observer setter cannot commit its JSON transaction, it restores +the pre-command in-memory preferences and does not restart or apply a bridge change as +though the setting were durable. -That asymmetry is the whole point. Refusing the file costs the operator the network -itself — `/mqtt_prefs` holds `wifi_ssid`/`wifi_password` as well as the broker config, -so a node that falls back to defaults has no WiFi, no portal, and no OTA, recoverable -only over serial. Reading it costs a feature's settings. Losing later settings is the -acceptable half of that trade; losing the node is not. +Saves stream to `/mqtt.json.tmp` through a sticky short-write detector while computing +size and checksum. The firmware closes and rereads the temp, verifies size/checksum, +parses it into another scratch object, then publishes with +`/mqtt.json` -> `/mqtt.json.bak` and temp -> primary. Boot recovery selects the usable +primary/temp/backup without overwriting an opaque future or corrupt primary. A valid +future-version temp that reached the rename phase wins over the stale backup and is held +for newer firmware. If a temp claims a future version but uses grammar this firmware +cannot parse, or cannot be classified because scratch allocation fails, recovery may run +the last usable backup but retains the uncertain temp and holds all observer writes. A +definitively corrupt/incomplete current-version temp may be discarded for that backup. +If power fails during the very first migration, there is no JSON primary or backup yet; +recovery discards a definitively invalid temp so the intact `/mqtt_prefs` source can be +loaded and the migration retried. An uncertain/future temp is still preserved and held. -The tail survives until something actually writes. `saveMQTTPrefs()` rewrites at this -binary's own length, so a rollback that changes no observer setting and is later rolled -forward keeps the newer fields intact — only an explicit `set` while downgraded drops -them. The boot log says so when it happens. +Unknown slot preset names are repaired to `none`, never to a build-specific default +broker. Duplicate known presets from historical firmware are preserved on load; current +CLI and WebConfig setters prevent creating new duplicates without silently changing an +existing deployment during migration. -**A change that is not a pure append MUST bump `MQTT_PREFS_VERSION`.** The version -check is what makes the rule above safe: a different tag is still refused outright and -the file preserved, because the bytes may no longer mean what this binary thinks. Never -reorder, resize, or repurpose an existing field within a version. +Schema changes that reinterpret existing names or values must increment `version`. +Additive fields may remain in version 1 only when old readers can safely ignore them +within the limits above. A future version is treated as opaque rather than partially +loading its known-looking fields. Literal schema keys are compile-time checked against +the 15-character visible-key limit so a new version-1 field cannot accidentally violate +that downgrade contract. -Note the rule is only as old as the build that implements it. Firmware already deployed -carries the *previous* decoder, which rejects any longer v1 payload — so rolling back -from this build to one shipped before it still falls back to defaults. -`MQTTPrefsCodec::payloadLenFor()` covers that gap from the writing side: it returns the -shortest length that still round-trips the configuration, so a node keeps writing 2864 -bytes until a packet filter actually holds something, and clearing the last non-default -filter puts it back. That mitigation can be retired once no supported downgrade target -predates the contract; the contract itself is the durable half. +#### Downgrade and rollback -Shorter payloads keep their existing, stricter treatment: a short length must match a -boundary that really shipped (`MQTT_PREFS_V1_*_PAYLOAD_SIZE`), because raw prefs have -no checksum and an arbitrary short size cannot be trusted to mean anything. +The old `/mqtt_prefs` binary is read only for one-time migration and is deliberately +not updated or deleted. When it exists and is usable, it is the exact pre-migration +rollback snapshot. Firmware older than this JSON change will therefore see stale +observer settings after a downgrade. A fresh JSON-only install has no binary observer +configuration to recover on downgrade. + +The two files are intentionally not reconciled. If an operator changes observer settings +while running old firmware, those changes update only `/mqtt_prefs`. Rolling forward to +this firmware makes the existing `/mqtt.json` authoritative again, so rollback-era edits +are ignored unless the operator exports and reapplies them. Conversely, JSON-only settings +cannot be recovered by old firmware. This is a rollback snapshot, not bidirectional sync. + +`LegacyV1MQTTPrefs` and the older pre-slot/3-slot/6-slot structs are frozen migration +ABIs with size/offset assertions. The runtime `MQTTPrefs` layout is not an on-flash ABI. ### Settings upgrade / migration `loadPrefs()` handles every historical on-device format one-time at boot: -- **`/mqtt_prefs`** — if the file has the version header it is read directly. Otherwise +- **`/mqtt_prefs` -> `/mqtt.json`** — if the legacy file has the version header its + frozen v1 layout is field-copied. Otherwise it is a legacy headerless file and its layout is detected by size: pre-slot (`OldMQTTPrefs`), 3-slot (`ThreeSlotMQTTPrefs`), or the 6-slot layout shipped on `observer-firmware` back when it was named `mqtt-bridge-implementation-flex` (`Legacy6SlotMQTTPrefs`). Each is field-copied into - the current compact `MQTTPrefs` and re-saved with the version header — which also + the runtime `MQTTPrefs` and saved as schema-versioned JSON — which also drops the vestigial `_legacy_*` fields the flex layout carried mid-struct. This is a one-time rewrite; every deployed device performs it on its first boot of versioned - firmware, after which all reads take the header path. + firmware, after which `/mqtt.json` is authoritative. The binary source remains as + a rollback snapshot and is never dual-written. The pre-slot (`OldMQTTPrefs`) copy maps the old single-broker keys onto slots: `mqtt.analyzer.us = on` → slot 1 `analyzer-us`, `mqtt.analyzer.eu = on` → slot 2 `analyzer-eu`, and a configured `mqtt.server` / `mqtt.port` / `mqtt.username` / @@ -241,7 +272,7 @@ no checksum and an arbitrary short size cannot be trusted to mean anything. - **`/com_prefs`** — a file written by fork firmware that predates the `MQTTPrefs` split (a zero-filled MQTT gap plus a trailing observer block) is detected by size; the trailing SNMP / radio-watchdog / fault-alert settings and the `rx_boosted_gain` / - `flood_max_*` fields are recovered, carried into `/mqtt_prefs`, and both files are + `flood_max_*` fields are recovered, carried into `/mqtt.json`, and the active files are rewritten in the current formats. - Settings the pre-split firmware stored *inside* the `/com_prefs` MQTT gap (the MQTT slot/WiFi config itself) are **not** recovered — users upgrading from firmware that diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 41b70da1..c9b1245a 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1029,7 +1029,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.agc_reset_interval = 7; // 28 seconds (secs/4) — prevents AGC drift on long-running observers #endif // Observer defaults (radio_watchdog, alert.*, snmp.*) moved to applyMQTTDefaults() - // in MQTTDefaults.h — they live in /mqtt_prefs now, not NodePrefs. + // in MQTTDefaults.h — they live in /mqtt.json now, not NodePrefs. // bridge defaults _prefs.bridge_enabled = 1; // enabled @@ -1045,7 +1045,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.gps_interval = 0; _prefs.advert_loc_policy = ADVERT_LOC_PREFS; - // MQTT/WiFi/timezone/radio_watchdog defaults live in /mqtt_prefs now (see applyMQTTDefaults). + // MQTT/WiFi/timezone/radio_watchdog defaults live in /mqtt.json now (see applyMQTTDefaults). _prefs.adc_multiplier = 0.0f; // 0.0f means use default board multiplier diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 4a7d5252..1d7c8b7d 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -284,6 +284,11 @@ public: void savePrefs() override { _cli.savePrefs(_fs); } +#ifdef WITH_MQTT_BRIDGE + bool saveObserverPrefs() override { + return _cli.saveObserverPrefs(_fs); + } +#endif void sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 546fed45..b9459d8d 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -891,7 +891,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.radio_fem_rxgain = 1; // Observer defaults (alert.*, etc.) moved to applyMQTTDefaults() — they live - // in /mqtt_prefs now, not NodePrefs. + // in /mqtt.json now, not NodePrefs. // bridge defaults (same as repeater) _prefs.bridge_enabled = 1; // enabled @@ -900,7 +900,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.bridge_baud = 115200; // baud rate _prefs.bridge_channel = 1; // channel 1 - // MQTT/WiFi/timezone defaults live in /mqtt_prefs now (see applyMQTTDefaults). + // MQTT/WiFi/timezone defaults live in /mqtt.json now (see applyMQTTDefaults). next_post_idx = 0; next_client_idx = 0; diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index fb28605f..acd68d9f 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -288,6 +288,11 @@ public: void savePrefs() override { _cli.savePrefs(_fs); } +#ifdef WITH_MQTT_BRIDGE + bool saveObserverPrefs() override { + return _cli.saveObserverPrefs(_fs); + } +#endif void sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size); diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index e579ba24..e0c750e5 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -6,6 +6,7 @@ #include "MQTTPrefsAtomicStore.h" #include #include +#include #ifndef BRIDGE_MAX_BAUD #define BRIDGE_MAX_BAUD 115200 @@ -23,6 +24,7 @@ #include "MQTTDefaults.h" #include "MQTTPrefsCodec.h" #include "MQTTPrefsRecovery.h" +#include "MQTTPrefsSerializer.h" #endif // Believe it or not, this std C function is busted on some platforms! @@ -94,7 +96,8 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { _prefs->bridge_pkt_src = 1; // Default to RX (logRx) for new installs } #ifdef WITH_MQTT_BRIDGE - // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt_prefs. + // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt.json, + // migrating the old /mqtt_prefs binary when JSON does not exist yet. // Readers (MQTTBridge, AlertReporter, observer CLI) use _mqtt_prefs directly — // these fields no longer exist in NodePrefs, so there is nothing to sync. MQTTPrefsAtomicStore::LegacyUpgradeGate legacy_upgrade( @@ -119,13 +122,13 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { } } // mqtt_rx_enabled: new field appended to end of MQTTPrefs. On upgrade from older firmware, - // the shorter /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) + // a shorter legacy /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) // set by setMQTTPrefsDefaults(). No explicit migration needed. #endif // Republish legacy binary prefs as /prefs.json. Old-format files also carried a // trailing observer block, which loadPrefsInt() recovered into _legacy_tail; wait - // for loadMQTTPrefs() to commit that to /mqtt_prefs first. The legacy file is left + // for loadMQTTPrefs() to commit that to /mqtt.json first. The legacy file is left // on flash either way, so a deferred or failed save just retries on the next boot. #ifdef WITH_MQTT_BRIDGE if (loaded_from_legacy || _com_prefs_needs_upgrade) { @@ -134,7 +137,7 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { legacy_upgrade.recordComPrefsRewrite(); _com_prefs_needs_upgrade = false; } else { - MESH_DEBUG_PRINTLN("Prefs: deferring /prefs.json migration until /mqtt_prefs commits"); + MESH_DEBUG_PRINTLN("Prefs: deferring /prefs.json migration until /mqtt.json commits"); } } #else @@ -375,13 +378,17 @@ bool CommonCLI::savePrefs(FILESYSTEM* fs, bool save_mqtt) { #ifdef WITH_MQTT_BRIDGE // Observer config (MQTT/WiFi/timezone/SNMP/alert) is persisted separately. The // observer CLI writes _mqtt_prefs directly, so no NodePrefs->MQTTPrefs sync runs. - // Runs regardless of the NodePrefs result so a failed JSON write cannot strand it. + // Ordinary NodePrefs callers leave save_mqtt false; migration is the only + // workflow that may explicitly combine these independent transactions. if (save_mqtt) saveMQTTPrefs(fs); #endif return success; } #ifdef WITH_MQTT_BRIDGE +static const uint32_t MQTT_JSON_FNV1A_OFFSET_BASIS = 2166136261u; +static const uint32_t MQTT_JSON_FNV1A_PRIME = 16777619u; + // Set default values for MQTT preferences (used when file doesn't exist or is corrupted) static void setMQTTPrefsDefaults(MQTTPrefs* prefs) { applyMQTTDefaults(prefs); @@ -395,6 +402,124 @@ static File openMqttPrefsRead(FILESYSTEM* fs, const char* path = "/mqtt_prefs") #endif } +enum class JsonPrefsLoadResult : uint8_t { + Loaded, + LoadedWithRepairs, + UnsupportedVersion, + FutureClaimed, + Invalid, +}; + +static JsonPrefsLoadResult loadMqttJsonFile(FILESYSTEM* fs, const char* path, + MQTTPrefs* output) { + if (output == nullptr) return JsonPrefsLoadResult::Invalid; + applyMQTTDefaults(output); + + // Probe the root version without applying the v1 schema. A future version + // may legitimately change an existing field's type, and must still be held + // opaquely rather than misclassified as corrupt and rolled back. + File version_file = openMqttPrefsRead(fs, path); + if (!version_file) return JsonPrefsLoadResult::Invalid; + MQTTPrefsVersionProbe version_probe; + const bool version_parsed = version_probe.loadSerial(version_file); + version_file.close(); + if (version_probe.hasFutureVersion()) { + return version_parsed ? JsonPrefsLoadResult::UnsupportedVersion + : JsonPrefsLoadResult::FutureClaimed; + } + + File file = openMqttPrefsRead(fs, path); + if (!file) return JsonPrefsLoadResult::Invalid; + MQTTPrefsSerializer serializer(output); + const bool parsed = serializer.loadSerial(file); + file.close(); + if (!parsed) return JsonPrefsLoadResult::Invalid; + if (serializer.hasFutureVersion()) return JsonPrefsLoadResult::UnsupportedVersion; + bool repaired = false; + if (!serializer.apply(&repaired)) return JsonPrefsLoadResult::Invalid; + return repaired ? JsonPrefsLoadResult::LoadedWithRepairs : JsonPrefsLoadResult::Loaded; +} + +static MQTTPrefsRecovery::FileState mqttJsonFileState(FILESYSTEM* fs, const char* path) { + if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; + MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; + if (scratch == nullptr) return MQTTPrefsRecovery::FileState::Indeterminate; + const JsonPrefsLoadResult result = loadMqttJsonFile(fs, path, scratch); + delete scratch; + if (result == JsonPrefsLoadResult::UnsupportedVersion) { + // Syntax and the mandatory version field were valid, so this can be a + // fully verified transaction written by newer firmware. Keep it distinct + // from a torn/corrupt temp during recovery. + return MQTTPrefsRecovery::FileState::FutureUsable; + } + if (result == JsonPrefsLoadResult::FutureClaimed) { + return MQTTPrefsRecovery::FileState::FutureClaimed; + } + return result == JsonPrefsLoadResult::Loaded || + result == JsonPrefsLoadResult::LoadedWithRepairs + ? MQTTPrefsRecovery::FileState::Usable + : MQTTPrefsRecovery::FileState::Preserve; +} + +static bool recoverMqttJsonFiles(FILESYSTEM* fs) { + const MQTTPrefsRecovery::FileState primary = mqttJsonFileState(fs, "/mqtt.json"); + const MQTTPrefsRecovery::FileState temp = mqttJsonFileState(fs, "/mqtt.json.tmp"); + const MQTTPrefsRecovery::FileState backup = mqttJsonFileState(fs, "/mqtt.json.bak"); + const MQTTPrefsRecovery::Action action = MQTTPrefsRecovery::select(primary, temp, backup); + + if (action == MQTTPrefsRecovery::Action::KeepPrimary) { + if (primary == MQTTPrefsRecovery::FileState::Usable) { + if (!MQTTPrefsRecovery::uncertain(temp) && + temp != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.tmp"); + } + if (!MQTTPrefsRecovery::uncertain(backup) && + backup != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.bak"); + } + } + return MQTTPrefsRecovery::uncertain(primary) || + MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + if (action == MQTTPrefsRecovery::Action::DiscardTemp) { + if (fs->remove("/mqtt.json.tmp")) { + MESH_DEBUG_PRINTLN("MQTT: discarded incomplete first-migration JSON temp"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not discard incomplete /mqtt.json temp; source held"); + return true; + } + if (action == MQTTPrefsRecovery::Action::PromoteTemp) { + if (fs->rename("/mqtt.json.tmp", "/mqtt.json")) { + if (temp == MQTTPrefsRecovery::FileState::Usable && + backup != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.bak"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction temp"); + return MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json temp; files preserved"); + return true; + } + if (action == MQTTPrefsRecovery::Action::PromoteBackup) { + if (fs->rename("/mqtt.json.bak", "/mqtt.json")) { + if (backup == MQTTPrefsRecovery::FileState::Usable && + !MQTTPrefsRecovery::uncertain(temp) && + temp != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.tmp"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction backup"); + return MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json backup; files preserved"); + return true; + } + return false; +} + static MQTTPrefsRecovery::FileState mqttPrefsFileState(FILESYSTEM* fs, const char* path) { if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; File file = openMqttPrefsRead(fs, path); @@ -431,6 +556,14 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { } return false; } + if (action == MQTTPrefsRecovery::Action::DiscardTemp) { + if (fs->remove("/mqtt_prefs.tmp")) { + MESH_DEBUG_PRINTLN("MQTT: discarded incomplete legacy transaction temp"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not discard incomplete /mqtt_prefs temp; source held"); + return true; + } if (action == MQTTPrefsRecovery::Action::PromoteTemp) { if (fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) { // A usable temp is now the committed primary. Its backup is necessarily @@ -462,30 +595,31 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { return false; } -// Filesystem adapter for MQTTPrefsAtomicStore. It writes the new image to -// /mqtt_prefs.tmp and verifies its size. Publishing is a recoverable SPIFFS +// Filesystem adapter for the ConfigSerializer image. It writes to +// /mqtt.json.tmp and verifies its size and checksum. Publishing is a recoverable SPIFFS // transaction: primary -> .bak, then tmp -> primary, then best-effort backup // cleanup. A power loss at every boundary leaves at least one recoverable file. -class MQTTPrefsFileStore { +class MQTTPrefsJsonFileStore { public: - explicit MQTTPrefsFileStore(FILESYSTEM* fs) : _fs(fs) {} + explicit MQTTPrefsJsonFileStore(FILESYSTEM* fs) : _fs(fs) {} bool begin() { _finished = false; _open = false; _owns_temp = false; _bytes_written = 0; + _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; // Recovery owns stale artifacts. Do not delete them here: a failed commit // may have moved the old primary to .bak and left a verified temp that the // next boot must choose between. Refusing the save is safer than erasing an // image this firmware cannot decode. - if (_fs->exists("/mqtt_prefs.tmp") || _fs->exists("/mqtt_prefs.bak")) return false; + if (_fs->exists("/mqtt.json.tmp") || _fs->exists("/mqtt.json.bak")) return false; #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) - _file = _fs->open("/mqtt_prefs.tmp", FILE_O_WRITE); + _file = _fs->open("/mqtt.json.tmp", FILE_O_WRITE); #elif defined(RP2040_PLATFORM) - _file = _fs->open("/mqtt_prefs.tmp", "w"); + _file = _fs->open("/mqtt.json.tmp", "w"); #else - _file = _fs->open("/mqtt_prefs.tmp", "w", true); + _file = _fs->open("/mqtt.json.tmp", "w", true); #endif _open = _file; _owns_temp = _open; @@ -495,6 +629,9 @@ public: size_t write(const uint8_t* bytes, size_t size) { if (!_open) return 0; const size_t written = _file.write(bytes, size); + for (size_t i = 0; i < written; ++i) { + _expected_crc = (_expected_crc ^ bytes[i]) * MQTT_JSON_FNV1A_PRIME; + } _bytes_written += written; return written; } @@ -504,12 +641,33 @@ public: _file.close(); _open = false; #if defined(RP2040_PLATFORM) - File verify = _fs->open("/mqtt_prefs.tmp", "r"); + File verify = _fs->open("/mqtt.json.tmp", "r"); #else - File verify = _fs->open("/mqtt_prefs.tmp"); + File verify = _fs->open("/mqtt.json.tmp"); #endif if (!verify) return false; - const bool complete = verify.size() == _bytes_written; + uint32_t actual_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; + size_t actual_size = 0; + bool read_failed = false; + uint8_t buf[64]; + while (verify.available() > 0) { + // Arduino File implementations normally return a byte count, but some + // Stream implementations use -1 for a read error. Keep that sentinel + // signed so it cannot become a huge size_t and overrun this buffer. + const int count = static_cast(verify.read(buf, sizeof(buf))); + if (count <= 0) { + read_failed = count < 0; + break; + } + actual_size += static_cast(count); + for (int i = 0; i < count; ++i) { + actual_crc = (actual_crc ^ buf[i]) * MQTT_JSON_FNV1A_PRIME; + } + } + const bool complete = !read_failed && + verify.size() == _bytes_written && + actual_size == _bytes_written && + actual_crc == _expected_crc; verify.close(); if (!complete) return false; _finished = true; @@ -522,25 +680,33 @@ public: // recoverable backup first, then publish temp into the now-empty primary. // Never remove either image after a failed boundary; boot recovery selects // the completed temp or restores the backup. - if (_fs->exists("/mqtt_prefs.bak")) return false; - if (_fs->exists("/mqtt_prefs") && !_fs->rename("/mqtt_prefs", "/mqtt_prefs.bak")) { + if (_fs->exists("/mqtt.json.bak")) return false; + if (_fs->exists("/mqtt.json") && !_fs->rename("/mqtt.json", "/mqtt.json.bak")) { return false; } - if (!_fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) return false; + if (!_fs->rename("/mqtt.json.tmp", "/mqtt.json")) return false; // Cleanup failure is non-fatal: the new primary is published and recovery // will remove a known-good stale backup on a later boot. - if (_fs->exists("/mqtt_prefs.bak")) _fs->remove("/mqtt_prefs.bak"); + if (_fs->exists("/mqtt.json.bak")) _fs->remove("/mqtt.json.bak"); return true; } + void discardFinishedTemp() { + if (_open) _file.close(); + _open = false; + if (_owns_temp && _fs->exists("/mqtt.json.tmp")) _fs->remove("/mqtt.json.tmp"); + _finished = false; + _owns_temp = false; + } + void abort() { if (_open) _file.close(); _open = false; // Once finish() has verified the temp, commit may already have moved the // primary to .bak. Keep the temp on a commit failure so recovery can // publish it (or fall back to .bak) after reset. - if (_owns_temp && !_finished && _fs->exists("/mqtt_prefs.tmp")) { - _fs->remove("/mqtt_prefs.tmp"); + if (_owns_temp && !_finished && _fs->exists("/mqtt.json.tmp")) { + _fs->remove("/mqtt.json.tmp"); } _finished = false; _owns_temp = false; @@ -553,31 +719,90 @@ private: bool _finished = false; bool _owns_temp = false; size_t _bytes_written = 0; + uint32_t _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; +}; + +class MQTTPrefsStoreStream : public Stream { +public: + explicit MQTTPrefsStoreStream(MQTTPrefsJsonFileStore* store) : _store(store) {} + + size_t write(uint8_t byte) override { return write(&byte, 1); } + size_t write(const uint8_t* buffer, size_t size) override { + if (!_ok || _store == nullptr) return 0; + const size_t written = _store->write(buffer, size); + if (written != size) _ok = false; + return written; + } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + bool ok() const { return _ok; } + +private: + MQTTPrefsJsonFileStore* _store; + bool _ok = true; }; #endif // WITH_MQTT_BRIDGE #ifdef WITH_MQTT_BRIDGE -static const char* mqttPrefsSaveResultName(MQTTPrefsAtomicStore::Result result) { - switch (result) { - case MQTTPrefsAtomicStore::Result::BeginFailed: return "begin"; - case MQTTPrefsAtomicStore::Result::HeaderWriteFailed: return "header write"; - case MQTTPrefsAtomicStore::Result::PayloadWriteFailed: return "payload write"; - case MQTTPrefsAtomicStore::Result::FinishFailed: return "close"; - case MQTTPrefsAtomicStore::Result::CommitFailed: return "rename"; - case MQTTPrefsAtomicStore::Result::Committed: return "committed"; - } - return "unknown"; -} - void CommonCLI::loadMQTTPrefs( FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade) { setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = recoverMqttJsonFiles(fs); + if (_mqtt_prefs_hold && !fs->exists("/mqtt.json") && + (fs->exists("/mqtt.json.tmp") || fs->exists("/mqtt.json.bak"))) { + _legacy_tail.valid = false; + MESH_DEBUG_PRINTLN("MQTT: unresolved /mqtt.json recovery files; using defaults (files preserved)"); + return; + } + + // The JSON file is authoritative once it exists. Never fall back to the + // stale binary snapshot when JSON is corrupt, unreadable, or from a future + // schema: preserve it and run defaults until an operator resolves it. + if (fs->exists("/mqtt.json")) { + MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; + if (scratch == nullptr) { + _mqtt_prefs_hold = true; + _legacy_tail.valid = false; + MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json; source preserved"); + return; + } + const JsonPrefsLoadResult json_result = loadMqttJsonFile(fs, "/mqtt.json", scratch); + if (json_result == JsonPrefsLoadResult::Loaded || + json_result == JsonPrefsLoadResult::LoadedWithRepairs) { + memcpy(&_mqtt_prefs, scratch, sizeof(_mqtt_prefs)); + delete scratch; + _legacy_tail.valid = false; + if (json_result == JsonPrefsLoadResult::LoadedWithRepairs) { + MESH_DEBUG_PRINTLN("MQTT: repaired out-of-range values in /mqtt.json"); + if (!_mqtt_prefs_hold && !saveMQTTPrefs(fs)) { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: could not persist /mqtt.json repairs; source held"); + } + } + return; + } + delete scratch; + _mqtt_prefs_hold = true; + _legacy_tail.valid = false; + if (json_result == JsonPrefsLoadResult::UnsupportedVersion) { + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json uses a future version; using defaults (file preserved)"); + } else if (json_result == JsonPrefsLoadResult::FutureClaimed) { + MESH_DEBUG_PRINTLN( + "MQTT: /mqtt.json claims a future version but uses unknown grammar; " + "using defaults (file preserved)"); + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json is invalid or unreadable; using defaults (file preserved)"); + } + return; + } + // Complete or preserve an interrupted SPIFFS transaction before decoding. // A failed recovery leaves the artifacts untouched and blocks this boot from // replacing them with defaults through a later CLI save. - _mqtt_prefs_hold = recoverMqttPrefsFiles(fs); + _mqtt_prefs_hold = _mqtt_prefs_hold || recoverMqttPrefsFiles(fs); bool has_observer_fields = false; bool mqtt_rewrite_pending = false; bool migrated_legacy_mqtt = false; @@ -599,24 +824,34 @@ void CommonCLI::loadMQTTPrefs( } else if (plan.source == MQTTPrefsCodec::Source::Current) { file = openMqttPrefsRead(fs); MQTTPrefsHeader header; - if (!file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || - file.read((uint8_t *)&_mqtt_prefs, plan.payload_len) != plan.payload_len) { + LegacyV1MQTTPrefs* old_prefs = new (std::nothrow) LegacyV1MQTTPrefs; + if (old_prefs) memset(old_prefs, 0, sizeof(*old_prefs)); + if (!old_prefs || !file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || + file.read((uint8_t *)old_prefs, plan.payload_len) != plan.payload_len) { setMQTTPrefsDefaults(&_mqtt_prefs); _mqtt_prefs_hold = true; MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs read failed, using defaults (file preserved)"); + } else if (!MQTTPrefsCodec::isPlausibleV1(*old_prefs, plan.payload_len)) { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs v1 content failed plausibility checks; source preserved"); } else { + MQTTPrefsCodec::migrateV1(*old_prefs, plan.payload_len, &_mqtt_prefs); has_observer_fields = plan.observer_fields_present; + mqtt_rewrite_pending = true; + migrated_legacy_mqtt = true; // Written by a later build with appended fields. Everything this // binary knows loaded normally; say so, because the next `set` will // rewrite the file at this length and drop the newer settings. if (file_size - sizeof(MQTTPrefsHeader) > plan.payload_len) { MESH_DEBUG_PRINTLN( "MQTT: /mqtt_prefs written by newer firmware (%u > %u bytes); " - "config loaded, newer settings ignored and dropped on next save", + "known settings loaded; newer binary fields remain in the rollback snapshot", (unsigned)(file_size - sizeof(MQTTPrefsHeader)), (unsigned)plan.payload_len); } } + delete old_prefs; if (file) file.close(); } else if (plan.rewrite_legacy) { bool migrated = false; @@ -673,16 +908,18 @@ void CommonCLI::loadMQTTPrefs( case MQTTPrefsCodec::Source::LegacySixSlotAudience: case MQTTPrefsCodec::Source::LegacySixSlotAudienceRx: case MQTTPrefsCodec::Source::LegacySixSlot: { - Legacy6SlotMQTTPrefs old_prefs = {}; - if (file.read((uint8_t *)&old_prefs, plan.payload_len) == plan.payload_len) { + Legacy6SlotMQTTPrefs* old_prefs = new (std::nothrow) Legacy6SlotMQTTPrefs; + if (old_prefs) memset(old_prefs, 0, sizeof(*old_prefs)); + if (old_prefs && file.read((uint8_t *)old_prefs, plan.payload_len) == plan.payload_len) { if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, - (const uint8_t *)&old_prefs, plan.payload_len)) { - MQTTPrefsCodec::migrateLegacySixSlot(old_prefs, plan.source, &_mqtt_prefs); + (const uint8_t *)old_prefs, plan.payload_len)) { + MQTTPrefsCodec::migrateLegacySixSlot(*old_prefs, plan.source, &_mqtt_prefs); migrated = true; } else { MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); } } + delete old_prefs; break; } default: @@ -741,9 +978,9 @@ void CommonCLI::loadMQTTPrefs( if (mqtt_rewrite_pending) { legacy_upgrade->requireMqttRewrite(); if (migrated_legacy_mqtt) { - MESH_DEBUG_PRINTLN("MQTT: Migrating headerless /mqtt_prefs to versioned layout"); + MESH_DEBUG_PRINTLN("MQTT: Migrating binary /mqtt_prefs to /mqtt.json"); } else { - MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt_prefs before /com_prefs compaction"); + MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt.json before /com_prefs compaction"); } if (saveMQTTPrefs(fs)) { legacy_upgrade->recordMqttSave(true); @@ -753,7 +990,7 @@ void CommonCLI::loadMQTTPrefs( // untouched; the next boot can recover the tail and retry the transaction. _mqtt_prefs_hold = true; legacy_upgrade->recordMqttSave(false); - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs migration save failed; legacy files preserved and held"); + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json migration save failed; legacy files preserved and held"); } } } @@ -762,26 +999,71 @@ bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { if (_mqtt_prefs_hold) { // Loading deliberately preserved the source file. Do not replace it with this // boot's defaults after an unsupported, corrupt, or temporarily failed read. - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs held, not overwriting"); + MESH_DEBUG_PRINTLN("MQTT: observer preference source held, not overwriting /mqtt.json"); return false; } - // Write header and payload sequentially so the transaction needs no second - // full-size (2.8 KiB) staging buffer on constrained targets. The length is - // the shortest that still round-trips this config, so a node with default - // packet filters keeps writing a payload older firmware can read. - const size_t payload_len = MQTTPrefsCodec::payloadLenFor(_mqtt_prefs); - const MQTTPrefsHeader header = MQTTPrefsCodec::makeHeader(payload_len); - MQTTPrefsFileStore store(fs); - const MQTTPrefsAtomicStore::Result result = MQTTPrefsAtomicStore::write( - store, (const uint8_t *)&header, sizeof(header), - (const uint8_t *)&_mqtt_prefs, payload_len); - if (!MQTTPrefsAtomicStore::committed(result)) { - MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt_prefs save failed at %s; source preserved", - mqttPrefsSaveResultName(result)); + MQTTPrefs* repair_defaults = new (std::nothrow) MQTTPrefs; + if (repair_defaults == nullptr) { + MESH_DEBUG_PRINTLN("MQTT: no memory to normalize observer settings before save"); return false; } - return true; + setMQTTPrefsDefaults(repair_defaults); + MQTTPrefsSerializer serializer(&_mqtt_prefs, repair_defaults); + // The serializer hierarchy copies the default values it needs. Release this + // large temporary before file I/O and the independent verification scratch. + delete repair_defaults; + bool repaired = false; + if (!serializer.normalize(&repaired)) return false; + if (repaired) MESH_DEBUG_PRINTLN("MQTT: normalized out-of-range observer settings before save"); + + MQTTPrefsJsonFileStore store(fs); + bool verify_oom = false; + const MQTTPrefsAtomicStore::VerifiedImageResult result = + MQTTPrefsAtomicStore::writeVerifiedImage( + store, + [&]() -> bool { + MQTTPrefsStoreStream stream(&store); + return serializer.saveSerial(stream) && stream.ok(); + }, + [&]() -> bool { + MQTTPrefs* verify = new (std::nothrow) MQTTPrefs; + if (verify == nullptr) { + verify_oom = true; + return false; + } + const JsonPrefsLoadResult verify_result = + loadMqttJsonFile(fs, "/mqtt.json.tmp", verify); + delete verify; + // saveSerial() emits already-normalized values. Needing another + // repair here means save/load is not idempotent. + return verify_result == JsonPrefsLoadResult::Loaded; + }); + + switch (result) { + case MQTTPrefsAtomicStore::VerifiedImageResult::Committed: + return true; + case MQTTPrefsAtomicStore::VerifiedImageResult::BeginFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed at begin; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::WriteFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during write; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::FinishFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during checksum verification; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::VerifyFailed: + if (verify_oom) { + MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json temp; source preserved"); + } else { + MESH_DEBUG_PRINTLN("MQTT: generated /mqtt.json temp failed schema validation; source preserved"); + } + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::CommitFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename; recovery files preserved"); + break; + } + return false; } #endif @@ -1396,6 +1678,15 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep strcpy(reply, "Error: delay must be between 0-10000 ms"); } } else if (memcmp(config, "bridge.source ", 14) == 0) { +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* observer_rollback = new (std::nothrow) MQTTPrefs; + if (observer_rollback == nullptr) { + strcpy(reply, "Error: insufficient memory to update observer setting"); + return; + } + memcpy(observer_rollback, &_mqtt_prefs, sizeof(*observer_rollback)); + const uint8_t old_bridge_pkt_src = _prefs->bridge_pkt_src; +#endif _prefs->bridge_pkt_src = memcmp(&config[14], "rx", 2) == 0; #ifdef WITH_MQTT_BRIDGE if (_prefs->bridge_pkt_src == 1) { @@ -1405,6 +1696,15 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep _mqtt_prefs.mqtt_rx_enabled = 0; _mqtt_prefs.mqtt_tx_enabled = 1; } + _observer_prefs_rollback = observer_rollback; + if (!persistObserverPrefs(reply)) { + _prefs->bridge_pkt_src = old_bridge_pkt_src; + _observer_prefs_rollback = nullptr; + delete observer_rollback; + return; + } + _observer_prefs_rollback = nullptr; + delete observer_rollback; #endif savePrefs(); strcpy(reply, "OK"); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 7bee051b..fa3e627e 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -72,7 +72,7 @@ public: uint8_t extra_sf[4]; // NOTE: observer settings (MQTT/WiFi/timezone/SNMP/alert) are not in NodePrefs. - // They live in MQTTPrefs, persisted separately to /mqtt_prefs, so this struct + // They live in MQTTPrefs, persisted separately to /mqtt.json, so this struct // stays aligned with upstream. See struct MQTTPrefs below. private: @@ -226,6 +226,13 @@ struct LegacyObserverTail { class CommonCLICallbacks { public: virtual void savePrefs() = 0; +#ifdef WITH_MQTT_BRIDGE + virtual bool saveObserverPrefs() = 0; +#else + virtual bool saveObserverPrefs() { + return false; + } +#endif virtual const char* getFirmwareVer() = 0; virtual const char* getBuildDate() = 0; virtual const char* getRole() = 0; @@ -357,8 +364,11 @@ class CommonCLI { char tmp[PRV_KEY_SIZE*2 + 4]; #ifdef WITH_MQTT_BRIDGE MQTTPrefs _mqtt_prefs; + // Points at a per-command snapshot only while an observer setter is running. + // persistObserverPrefs() uses it to undo RAM mutations when flash commit fails. + const MQTTPrefs* _observer_prefs_rollback = nullptr; LegacyObserverTail _legacy_tail; - // /mqtt_prefs is newer, corrupt, or temporarily unreadable. The in-memory prefs + // /mqtt.json is newer, corrupt, or temporarily unreadable. The in-memory prefs // run on defaults and saveMQTTPrefs() must not overwrite the source file. bool _mqtt_prefs_hold = false; #endif @@ -382,6 +392,7 @@ class CommonCLI { // false to fall through to the base get/set parsing. bool handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply); bool handleObserverGetCmd(uint32_t sender_timestamp, const char* config, char* reply); + bool persistObserverPrefs(char* reply); // Observer-only top-level commands (ota check/update, tls.bundletest, alert test) // also live in CommonCLI_Observer.cpp; returns true if it handled the command. bool handleObserverCommand(uint32_t sender_timestamp, char* command, char* reply); @@ -391,14 +402,17 @@ public: : _board(&board), _rtc(&rtc), _sensors(&sensors), _region_map(®ion_map), _acl(&acl), _prefs(prefs), _callbacks(callbacks) { } void loadPrefs(FILESYSTEM* _fs); - bool savePrefs(FILESYSTEM* _fs, bool save_mqtt = true); + // Node preferences and observer preferences are separate transactions. + // Callers must explicitly request an observer save when they changed it. + bool savePrefs(FILESYSTEM* _fs, bool save_mqtt = false); void handleCommand(uint32_t sender_timestamp, char* command, char* reply); mesh::MainBoard* getBoard() { return _board; } uint8_t buildAdvertData(uint8_t node_type, uint8_t* app_data); #ifdef WITH_MQTT_BRIDGE - // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt_prefs. + // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // Exposed so the app can hand it to MQTTBridge/AlertReporter, which read these // fields directly (they no longer live in NodePrefs). MQTTPrefs* getObserverPrefs() const { return const_cast(&_mqtt_prefs); } + bool saveObserverPrefs(FILESYSTEM* fs) { return saveMQTTPrefs(fs); } #endif }; diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index c1b7e74a..93c25776 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -16,6 +16,7 @@ #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) #include +#include #ifdef ESP_PLATFORM #include #include @@ -109,6 +110,46 @@ static bool valueTooLong(const char* val, size_t bufsize, char* reply, const cha return false; } +static bool isObserverPrefsSetCommand(const char* config) { + return strncmp(config, "snmp", 4) == 0 || + strncmp(config, "radio.watchdog", 14) == 0 || + strncmp(config, "mqtt", 4) == 0 || + strncmp(config, "wifi.", 5) == 0 || + strncmp(config, "timezone", 8) == 0 || + strncmp(config, "alert", 5) == 0; +} + +// Keep observer setters atomic from the caller's perspective. The live object +// is restored if its JSON transaction fails, and the snapshot pointer is valid +// only for this synchronous command dispatch. +class ObserverPrefsRollbackScope { +public: + ObserverPrefsRollbackScope(const MQTTPrefs** active_snapshot, + const MQTTPrefs& live, + bool capture) + : _active_snapshot(active_snapshot) { + *_active_snapshot = nullptr; + if (capture) { + _snapshot = new (std::nothrow) MQTTPrefs; + if (_snapshot != nullptr) { + memcpy(_snapshot, &live, sizeof(*_snapshot)); + *_active_snapshot = _snapshot; + } + } + } + + ~ObserverPrefsRollbackScope() { + *_active_snapshot = nullptr; + delete _snapshot; + } + + bool available() const { return _snapshot != nullptr; } + +private: + const MQTTPrefs** _active_snapshot; + MQTTPrefs* _snapshot = nullptr; +}; + static const char* getMQTTPresetNameByIndex(int index) { if (index < MQTT_PRESET_COUNT) return MQTT_PRESETS[index].name; if (index == MQTT_PRESET_COUNT) return MQTT_PRESET_CUSTOM; @@ -161,17 +202,38 @@ static void formatMQTTPresetListReply(char* reply, size_t reply_size, int start) } #endif +bool CommonCLI::persistObserverPrefs(char* reply) { +#ifdef WITH_MQTT_BRIDGE + if (_callbacks->saveObserverPrefs()) return true; + if (_observer_prefs_rollback != nullptr) { + memcpy(&_mqtt_prefs, _observer_prefs_rollback, sizeof(_mqtt_prefs)); + } + strcpy(reply, "Error: setting not persisted; change rolled back"); + return false; +#else + (void)reply; + return false; +#endif +} + bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply) { #ifdef WITH_MQTT_BRIDGE + const bool needs_snapshot = isObserverPrefsSetCommand(config); + ObserverPrefsRollbackScope rollback_scope( + &_observer_prefs_rollback, _mqtt_prefs, needs_snapshot); + if (needs_snapshot && !rollback_scope.available()) { + strcpy(reply, "Error: insufficient memory to update observer setting"); + return true; + } bool handled = true; if (memcmp(config, "snmp.community ", 15) == 0) { if (valueTooLong(&config[15], sizeof(_mqtt_prefs.snmp_community), reply, "snmp.community")) return true; StrHelper::strncpy(_mqtt_prefs.snmp_community, &config[15], sizeof(_mqtt_prefs.snmp_community)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - restart to apply"); } else if (memcmp(config, "snmp ", 5) == 0) { _mqtt_prefs.snmp_enabled = memcmp(&config[5], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - restart to apply"); } else if (memcmp(config, "radio.watchdog ", 15) == 0) { const char* val = &config[15]; @@ -189,7 +251,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: radio.watchdog must be 0-120 minutes"); } else { _mqtt_prefs.radio_watchdog_minutes = (uint8_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; if (mins == 0) { strcpy(reply, "OK - radio watchdog disabled"); } else { @@ -200,13 +262,13 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf #ifdef WITH_MQTT_BRIDGE } else if (strcmp(config, "mqtt.origin") == 0) { _mqtt_prefs.mqtt_origin[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.origin ", 12) == 0) { if (valueTooLong(&config[12], sizeof(_mqtt_prefs.mqtt_origin), reply, "origin")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_origin, &config[12], sizeof(_mqtt_prefs.mqtt_origin)); StrHelper::stripSurroundingQuotes(_mqtt_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.iata ", 10) == 0) { const char* iata = &config[10]; @@ -215,7 +277,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // Empty clears the region code (meshcore-topic publishing stays disabled // until one is set). This keeps the pre-existing "clear IATA" capability. _mqtt_prefs.mqtt_iata[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); strcpy(reply, "OK - IATA cleared"); } else { @@ -228,22 +290,22 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf for (int i = 0; _mqtt_prefs.mqtt_iata[i]; i++) { _mqtt_prefs.mqtt_iata[i] = toupper(_mqtt_prefs.mqtt_iata[i]); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); strcpy(reply, "OK"); } } } else if (memcmp(config, "mqtt.status ", 12) == 0) { _mqtt_prefs.mqtt_status_enabled = memcmp(&config[12], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.packets ", 13) == 0) { _mqtt_prefs.mqtt_packets_enabled = memcmp(&config[13], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.raw ", 9) == 0) { _mqtt_prefs.mqtt_raw_enabled = memcmp(&config[9], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.tx ", 8) == 0) { if (memcmp(&config[8], "advert", 6) == 0) { @@ -251,17 +313,17 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { _mqtt_prefs.mqtt_tx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.rx ", 8) == 0) { _mqtt_prefs.mqtt_rx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.interval ", 14) == 0) { uint32_t minutes = _atoi(&config[14]); if (minutes >= 1 && minutes <= 60) { _mqtt_prefs.mqtt_status_interval = minutes * 60000; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); sprintf(reply, "OK - interval set to %u minutes (%lu ms), bridge restarted", minutes, (unsigned long)_mqtt_prefs.mqtt_status_interval); } else { @@ -274,7 +336,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf uint32_t hours = _atoi(&config[24]); if (hours >= MQTT_NEIGHBORS_MIN_INTERVAL_HOURS && hours <= MQTT_NEIGHBORS_MAX_INTERVAL_HOURS) { _mqtt_prefs.mqtt_neighbors_interval = hours * 3600000UL; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - neighbors interval set to %u hours (%lu ms)", (unsigned)hours, (unsigned long)_mqtt_prefs.mqtt_neighbors_interval); } else { @@ -284,7 +346,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // The mesh loop reads this live, so no bridge restart is needed; enabling it // triggers a discovery on the next eligible loop pass. _mqtt_prefs.mqtt_neighbors_enabled = memcmp(&config[15], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); #elif defined(WITH_MQTT_BRIDGE) } else if (memcmp(config, "mqtt.neighbors.interval ", 24) == 0 || @@ -303,7 +365,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { StrHelper::strncpy(_mqtt_prefs.mqtt_ntp_server, host, sizeof(_mqtt_prefs.mqtt_ntp_server)); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM // Queue a sync on the MQTT task (Core 0) but do NOT block: this handler // runs on the Arduino loop task, shared with mesh/radio processing and the @@ -325,12 +387,12 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "wifi.ssid ", 10) == 0) { if (valueTooLong(&config[10], sizeof(_mqtt_prefs.wifi_ssid), reply, "wifi.ssid")) return true; StrHelper::strncpy(_mqtt_prefs.wifi_ssid, &config[10], sizeof(_mqtt_prefs.wifi_ssid)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "wifi.pwd ", 9) == 0) { if (valueTooLong(&config[9], sizeof(_mqtt_prefs.wifi_password), reply, "wifi.pwd")) return true; StrHelper::strncpy(_mqtt_prefs.wifi_password, &config[9], sizeof(_mqtt_prefs.wifi_password)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "wifi.powersave ", 15) == 0) { const char* value = &config[15]; @@ -350,7 +412,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: must be none, min, or max"); } else { _mqtt_prefs.wifi_power_save = ps_value; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM if (WiFi.status() == WL_CONNECTED) { wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : @@ -374,13 +436,13 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "timezone ", 9) == 0) { if (valueTooLong(&config[9], sizeof(_mqtt_prefs.timezone_string), reply, "timezone")) return true; StrHelper::strncpy(_mqtt_prefs.timezone_string, &config[9], sizeof(_mqtt_prefs.timezone_string)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "timezone.offset ", 16) == 0) { int8_t offset = _atoi(&config[16]); if (offset >= -12 && offset <= 14) { _mqtt_prefs.timezone_offset = offset; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else { strcpy(reply, "Error: timezone offset must be between -12 and +14"); @@ -397,20 +459,14 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0 || strcmp(preset_name, MQTT_PRESET_NONE) == 0) { // Reject duplicate presets (except "none" and "custom") - int dup_slot = -1; - if (findMQTTPreset(preset_name) != nullptr) { - for (int s = 0; s < MAX_MQTT_SLOTS; s++) { - if (s != slot && strcmp(_mqtt_prefs.mqtt_slot_preset[s], preset_name) == 0) { - dup_slot = s; - break; - } - } - } + const int dup_slot = findMQTTPreset(preset_name) == nullptr ? -1 : + mqttAssignedPresetSlot(_mqtt_prefs.mqtt_slot_preset, + preset_name, slot); if (dup_slot >= 0) { sprintf(reply, "Error: preset '%s' is already assigned to slot %d", preset_name, dup_slot + 1); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], preset_name, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); // Check if the slot has everything it needs to connect const MQTTPresetDef* p = findMQTTPreset(preset_name); @@ -469,7 +525,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "server ", 7) == 0) { if (valueTooLong(&subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot]), reply, "server")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_host[slot], &subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; // Reconfigure the slot so the new host reaches the live connection (other // custom-slot setters do the same; without it the change only applies on // the next reboot/bridge restart). @@ -479,7 +535,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf int port = atoi(&subcmd[5]); if (port > 0 && port <= 65535) { _mqtt_prefs.mqtt_slot_port[slot] = port; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else { @@ -488,19 +544,19 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "username ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot]), reply, "username")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_username[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(subcmd, "password ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot]), reply, "password")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_password[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(subcmd, "token ", 6) == 0) { if (valueTooLong(&subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot]), reply, "token")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_token[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d token set", slot + 1); } else if (memcmp(subcmd, "topic ", 6) == 0) { @@ -510,14 +566,14 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf return true; } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_topic[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_topic[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d topic: %s", slot + 1, _mqtt_prefs.mqtt_slot_topic[slot]); } } else if (memcmp(subcmd, "audience ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot]), reply, "audience")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_audience[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); if (_mqtt_prefs.mqtt_slot_audience[slot][0] != '\0') { sprintf(reply, "OK - slot %d JWT audience: %s", slot + 1, _mqtt_prefs.mqtt_slot_audience[slot]); @@ -527,7 +583,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "audience", 8) == 0 && subcmd[8] == '\0') { // "set mqttN.audience" with no value — clear the audience _mqtt_prefs.mqtt_slot_audience[slot][0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d JWT audience cleared (using username/password auth)", slot + 1); } else if (strcmp(subcmd, "filter") == 0 || @@ -539,20 +595,18 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: filter must be all, none, or a CSV of types 0-15 / names (advert,txt_msg,...)"); } else { _mqtt_prefs.mqtt_slot_packet_filter[slot] = filter_mask; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; char filter_text[MQTTPacketFilter::kFilterTextSize]; MQTTPacketFilter::format(filter_mask, filter_text, sizeof(filter_text)); snprintf(reply, 160, "OK - slot %d packet types: %s", slot + 1, filter_text); - // A non-default filter extends /mqtt_prefs past what pre-filter - // firmware can read (see MQTTPrefsCodec::payloadLenFor), so say when - // that cost buys nothing: slots beyond the runtime array are never - // published to on this board, the same warning `preset` gives. + // Slots beyond the runtime array are never published to on this board, + // so retain the same inactive-hardware warning `preset` gives. if (slot >= RUNTIME_MQTT_SLOTS && filter_mask != MQTTPacketFilter::kAllPacketTypes) { size_t used = strlen(reply); if (used < 158) { snprintf(reply + used, 160 - used, - " (slot inactive on this hardware; blocks firmware rollback)"); + " (slot inactive on this hardware)"); } } } @@ -562,21 +616,35 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "mqtt.analyzer.us ", 17) == 0) { const int slot = 0; if (memcmp(&config[17], "on", 2) == 0) { + const int dup_slot = mqttAssignedPresetSlot( + _mqtt_prefs.mqtt_slot_preset, "analyzer-us", slot); + if (dup_slot >= 0) { + sprintf(reply, "Error: preset 'analyzer-us' is already assigned to slot %d", + dup_slot + 1); + return true; + } StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.analyzer.eu ", 17) == 0) { const int slot = 1; if (memcmp(&config[17], "on", 2) == 0) { + const int dup_slot = mqttAssignedPresetSlot( + _mqtt_prefs.mqtt_slot_preset, "analyzer-eu", slot); + if (dup_slot >= 0) { + sprintf(reply, "Error: preset 'analyzer-eu' is already assigned to slot %d", + dup_slot + 1); + return true; + } StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.owner ", 11) == 0) { @@ -585,11 +653,11 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // Owner key is optional — empty clears it (previously this errored, so a // set key could never be removed via the portal/CLI). _mqtt_prefs.mqtt_owner_public_key[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - owner key cleared"); } else if (mqttOwnerKeyValid(owner_key)) { StrHelper::strncpy(_mqtt_prefs.mqtt_owner_public_key, owner_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else { strcpy(reply, "Error: public key must be 64 hex characters (32 bytes)"); @@ -597,7 +665,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "mqtt.email ", 11) == 0) { if (valueTooLong(&config[11], sizeof(_mqtt_prefs.mqtt_email), reply, "email")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_email, &config[11], sizeof(_mqtt_prefs.mqtt_email)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); #endif } else if (memcmp(config, "alert ", 6) == 0) { @@ -605,12 +673,12 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf const char* val = &config[6]; if (memcmp(val, "on", 2) == 0 && (val[2] == 0 || val[2] == ' ')) { _mqtt_prefs.alert_enabled = 1; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alerts on"); } else if (memcmp(val, "off", 3) == 0 && (val[3] == 0 || val[3] == ' ')) { _mqtt_prefs.alert_enabled = 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alerts off"); } else { @@ -625,7 +693,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf if (len == 0) { _mqtt_prefs.alert_psk_hex[0] = '\0'; _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.psk cleared (alerts disabled until configured)"); } else if (val[0] == '#') { @@ -659,7 +727,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // The new PSK is operator-supplied, so any previously-derived // hashtag name is no longer accurate provenance — drop it. _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.psk updated"); } @@ -672,7 +740,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf if (in_len == 0) { _mqtt_prefs.alert_psk_hex[0] = '\0'; _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.hashtag cleared (alerts disabled until configured)"); } else { @@ -708,7 +776,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf mesh::Utils::toHex(hex, digest, 16); StrHelper::strncpy(_mqtt_prefs.alert_hashtag, hashtag, sizeof(_mqtt_prefs.alert_hashtag)); StrHelper::strncpy(_mqtt_prefs.alert_psk_hex, hex, sizeof(_mqtt_prefs.alert_psk_hex)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); sprintf(reply, "OK - alert.hashtag: %s", _mqtt_prefs.alert_hashtag); } @@ -726,7 +794,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf size_t len = strlen(val); if (len == 0) { _mqtt_prefs.alert_region[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.region cleared (using default scope)"); } else if (len >= sizeof(_mqtt_prefs.alert_region)) { @@ -734,7 +802,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { StrHelper::strncpy(_mqtt_prefs.alert_region, val, sizeof(_mqtt_prefs.alert_region)); StrHelper::stripSurroundingQuotes(_mqtt_prefs.alert_region, sizeof(_mqtt_prefs.alert_region)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); sprintf(reply, "OK - alert.region: %s", _mqtt_prefs.alert_region); } @@ -744,7 +812,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.wifi must be 0-1440 minutes (0=off)"); } else { _mqtt_prefs.alert_wifi_minutes = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.wifi %d min%s", mins, mins == 0 ? " (disabled)" : ""); } } else if (memcmp(config, "alert.mqtt ", 11) == 0) { @@ -753,7 +821,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.mqtt must be 0-10080 minutes (0=off)"); } else { _mqtt_prefs.alert_mqtt_minutes = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.mqtt %d min%s", mins, mins == 0 ? " (disabled)" : ""); } } else if (memcmp(config, "alert.interval ", 15) == 0) { @@ -764,7 +832,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.interval must be 60-10080 minutes"); } else { _mqtt_prefs.alert_min_interval_min = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.interval %d min", mins); } } else { diff --git a/src/helpers/ConfigSerializer.cpp b/src/helpers/ConfigSerializer.cpp index adff147f..b8788bf5 100644 --- a/src/helpers/ConfigSerializer.cpp +++ b/src/helpers/ConfigSerializer.cpp @@ -1,5 +1,9 @@ #include "ConfigSerializer.h" +#include +#include +#include + bool ConfigSerializer::saveSerial(Stream& s) { Context context(&s, OP::WRITE); _context = &context; // set the context for structure() call @@ -22,9 +26,12 @@ bool ConfigSerializer::saveSerial(Stream& s) { static bool is_whitespace(char c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; } -static bool is_key_char(char c) { +static bool is_key_start_char(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; } +static bool is_key_char(char c) { + return is_key_start_char(c) || (c >= '0' && c <= '9'); +} static bool is_value_char(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || c == '-' || c == '.'; } @@ -64,14 +71,26 @@ int ConfigSerializer::Context::readNext() { case EXPECT_KEY: if (rd_len > 0 && c == ':') { rd_buf[rd_len] = 0; rd_len = 0; rd_mode = EXPECT_VAL_OR_OBJ; return TOK_KEY; } if (rd_len == 0 && is_whitespace(c)) return TOK_WHITESPACE; - if (rd_len < CONFIG_MAX_KEYLEN-1 && is_key_char(c)) { rd_buf[rd_len++] = c; return TOK_WHITESPACE; } + if (rd_len < CONFIG_MAX_KEYLEN-1 && + ((rd_len == 0 && is_key_start_char(c)) || (rd_len > 0 && is_key_char(c)))) { + rd_buf[rd_len++] = c; + return TOK_WHITESPACE; + } return TOK_ERROR; case EXPECT_VAL_OR_OBJ: if (rd_len == 0 && is_whitespace(c)) return TOK_WHITESPACE; - if (rd_len == 0 && c == '"') { rd_mode = EXPECT_STRING_VAL; return TOK_WHITESPACE; } + if (rd_len == 0 && c == '"') { + rd_token_quoted = true; + rd_mode = EXPECT_STRING_VAL; + return TOK_WHITESPACE; + } if (rd_len == 0 && c == '{') { rd_mode = EXPECT_KEY; return TOK_START_OBJ; } - if (is_value_char(c) && rd_len < CONFIG_MAX_TOKEN_LEN-1) { rd_buf[rd_len++] = c; return TOK_WHITESPACE; } + if (is_value_char(c) && rd_len < CONFIG_MAX_TOKEN_LEN-1) { + if (rd_len == 0) rd_token_quoted = false; + rd_buf[rd_len++] = c; + return TOK_WHITESPACE; + } if (rd_len > 0 && (c == ',' || c == '}' || is_whitespace(c))) { pending = c; rd_buf[rd_len] = 0; rd_len = 0; rd_mode = EXPECT_COMMA_OR_CLOSE; return TOK_VALUE; } return TOK_ERROR; @@ -107,9 +126,19 @@ bool ConfigSerializer::loadSerial(Stream& s) { if (next_tok == TOK_KEY) { context.setKey(sp, context.getToken()); } else if (next_tok == TOK_VALUE) { + context.setValueEvent(sp, false); _depth = 1; // re-run the structure() hierarchy again (looking for specific key, at specific depth) structure(); } else if (next_tok == TOK_START_OBJ) { + // The root has no key. For every nested object, rerun the schema at the + // parent depth so strict scalar fields can reject an object value and + // object fields can confirm that their value has the expected shape. + if (sp > 0) { + context.setValueEvent(sp, true); + _depth = 1; + structure(); + if (!context.success) break; + } if (sp < CONFIG_MAX_DEPTH - 1) { sp++; } else { @@ -153,6 +182,10 @@ void ConfigSerializer::def(const char* key, void* value, size_t len) { _context->file()->print("\""); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } memset(value, 0, len); mesh::Utils::fromHex((uint8_t *)value, len, _context->getToken()); } @@ -181,6 +214,10 @@ void ConfigSerializer::def(const char* key, char* value, size_t max_len) { _context->file()->print("\""); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } strncpy(value, _context->getToken(), max_len - 1); value[max_len - 1] = 0; } @@ -195,6 +232,10 @@ void ConfigSerializer::def(const char* key, int32_t& value) { _context->file()->print(value); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -208,6 +249,10 @@ void ConfigSerializer::def(const char* key, uint32_t& value) { _context->file()->print(value); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -221,6 +266,10 @@ void ConfigSerializer::def(const char* key, int16_t& value) { _context->file()->print((int32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -234,6 +283,10 @@ void ConfigSerializer::def(const char* key, uint16_t& value) { _context->file()->print((uint32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -247,6 +300,10 @@ void ConfigSerializer::def(const char* key, uint8_t& value) { _context->file()->print((uint32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -260,6 +317,10 @@ void ConfigSerializer::def(const char* key, int8_t& value) { _context->file()->print((int32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -273,6 +334,10 @@ void ConfigSerializer::def(const char* key, bool& value) { _context->file()->print(value ? "true" : "false"); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = strcmp(_context->getToken(), "true") == 0 || atoi(_context->getToken()) != 0; // 'true' or a non-zero number } } @@ -290,6 +355,10 @@ void ConfigSerializer::def(const char* key, double& value) { } } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atof(_context->getToken()); } } @@ -307,6 +376,10 @@ void ConfigSerializer::def(const char* key, float& value) { } } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = (float) atof(_context->getToken()); } } @@ -323,9 +396,64 @@ void ConfigSerializer::def(const char* key, ConfigSerializer& sub_obj) { if (_context->file()->print("}") != 1) _context->success = false; // failure detect } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() && _context->valueDepth() == _depth) { + return; // the object itself; descendant values recurse below + } + if (_context->valueDepth() <= _depth) { + _context->success = false; // known object supplied as a scalar + return; + } sub_obj._context = _context; // inherit the Context sub_obj._depth = _depth + 1; sub_obj.structure(); // recurse into sub object } } } + +bool ConfigSerializer::defStrict(const char* key, char* value, size_t max_len, bool& seen) { + if (_context->op() == OP::WRITE) { + def(key, value, max_len); + return true; + } + if (!_context->keyMatch(_depth, key)) return false; + if (seen || max_len == 0 || _context->objectStart() || + _context->valueDepth() != _depth || !_context->tokenQuoted()) { + _context->success = false; + return false; + } + seen = true; + const char* token = _context->getToken(); + const size_t len = strlen(token); + if (len >= max_len) { + _context->success = false; + return false; + } + memcpy(value, token, len + 1); + return true; +} + +bool ConfigSerializer::defStrict(const char* key, int32_t& value, bool& seen) { + if (_context->op() == OP::WRITE) { + def(key, value); + return true; + } + if (!_context->keyMatch(_depth, key)) return false; + if (seen || _context->objectStart() || _context->valueDepth() != _depth || + _context->tokenQuoted()) { + _context->success = false; + return false; + } + seen = true; + + const char* token = _context->getToken(); + char* end = nullptr; + errno = 0; + const long long parsed = strtoll(token, &end, 10); + if (token[0] == '\0' || end == token || *end != '\0' || errno == ERANGE || + parsed < INT32_MIN || parsed > INT32_MAX) { + _context->success = false; + return false; + } + value = static_cast(parsed); + return true; +} diff --git a/src/helpers/ConfigSerializer.h b/src/helpers/ConfigSerializer.h index 7e6d6f2a..e889e637 100644 --- a/src/helpers/ConfigSerializer.h +++ b/src/helpers/ConfigSerializer.h @@ -25,17 +25,35 @@ class ConfigSerializer { OP _op; uint8_t rd_len; uint8_t rd_mode; + uint8_t rd_value_depth; char pending; + bool rd_token_quoted; + bool rd_object_start; char rd_buf[CONFIG_MAX_TOKEN_LEN]; char _keys[CONFIG_MAX_DEPTH][CONFIG_MAX_KEYLEN]; public: bool success = true; - Context(Stream* f, OP op) : _f(f), _op(op) { rd_buf[rd_len = 0] = 0; rd_mode = 0; pending = 0; } + Context(Stream* f, OP op) : _f(f), _op(op) { + rd_buf[rd_len = 0] = 0; + rd_mode = 0; + rd_value_depth = 0; + pending = 0; + rd_token_quoted = false; + rd_object_start = false; + memset(_keys, 0, sizeof(_keys)); + } OP op() const { return _op; } Stream* file() const { return _f; } int readNext(); const char* getToken() const { return rd_buf; } + bool tokenQuoted() const { return rd_token_quoted; } + uint8_t valueDepth() const { return rd_value_depth; } + bool objectStart() const { return rd_object_start; } + void setValueEvent(uint8_t depth, bool object_start) { + rd_value_depth = depth; + rd_object_start = object_start; + } bool keyMatch(int8_t depth, const char* key) { return strcmp(key, _keys[depth]) == 0; } void setKey(uint8_t depth, const char* key) { strcpy(_keys[depth], key); } }; @@ -60,6 +78,67 @@ protected: void def(const char* key, bool& value); void def(const char* key, ConfigSerializer& sub_obj); + // Strict read helpers for schemas whose values may be edited outside the + // firmware. Unlike the legacy def() overloads, these reject duplicate keys, + // truncated strings, malformed integers, and integer overflow. `seen` must + // be a field owned by the schema object and initialized false before load. + bool defStrict(const char* key, char* value, size_t max_len, bool& seen); + bool defStrict(const char* key, int32_t& value, bool& seen); + + // Literal keys are checked where a schema defines them, so adding a field + // that an older ConfigSerializer cannot tokenize is a build failure rather + // than a silent downgrade trap. Dynamic keys retain the pointer overloads. + template static void checkKey(const char (&)[N]) { + static_assert(N <= CONFIG_MAX_KEYLEN, + "ConfigSerializer key exceeds the visible-key limit"); + } + template void def(const char (&key)[N], char* value, size_t max_len) { + checkKey(key); def(static_cast(key), value, max_len); + } + template void def(const char (&key)[N], void* value, size_t len) { + checkKey(key); def(static_cast(key), value, len); + } + template void def(const char (&key)[N], int32_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], int16_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], int8_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint32_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint16_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint8_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], float& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], double& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], bool& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], ConfigSerializer& sub_obj) { + checkKey(key); def(static_cast(key), sub_obj); + } + template + bool defStrict(const char (&key)[N], char* value, size_t max_len, bool& seen) { + checkKey(key); + return defStrict(static_cast(key), value, max_len, seen); + } + template + bool defStrict(const char (&key)[N], int32_t& value, bool& seen) { + checkKey(key); + return defStrict(static_cast(key), value, seen); + } + virtual void structure() = 0; public: diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h index 5072e422..21c53bc4 100644 --- a/src/helpers/MQTTDefaults.h +++ b/src/helpers/MQTTDefaults.h @@ -8,7 +8,7 @@ #include "MQTTPacketFilter.h" #include "MQTTPresets.h" -// Compile-time defaults for fresh /mqtt_prefs (override via platformio build_flags). +// Compile-time defaults for fresh /mqtt.json (override via platformio build_flags). // Example: // -D MQTT_DEFAULT_SLOT1_PRESET='"meshcore-ca-1"' // -D MQTT_DEFAULT_IATA='"YYZ"' diff --git a/src/helpers/MQTTObserverValidation.h b/src/helpers/MQTTObserverValidation.h index 2042f226..a8d465a7 100644 --- a/src/helpers/MQTTObserverValidation.h +++ b/src/helpers/MQTTObserverValidation.h @@ -58,3 +58,20 @@ static inline bool mqttNtpHostnameValid(const char* host) { static inline bool mqttValueFits(const char* s, size_t bufsize) { return s != NULL && bufsize > 0 && strlen(s) < bufsize; } + +// Find a preset already assigned to another slot. The caller decides which +// names are singleton presets ("none" and "custom" intentionally are not). +template +static inline int mqttAssignedPresetSlot( + const char (&presets)[SlotCount][PresetSize], + const char* preset_name, + int ignored_slot) { + if (preset_name == NULL) return -1; + for (size_t slot = 0; slot < SlotCount; ++slot) { + if (static_cast(slot) != ignored_slot && + strcmp(presets[slot], preset_name) == 0) { + return static_cast(slot); + } + } + return -1; +} diff --git a/src/helpers/MQTTPacketFilter.h b/src/helpers/MQTTPacketFilter.h index df811828..47fbfe5c 100644 --- a/src/helpers/MQTTPacketFilter.h +++ b/src/helpers/MQTTPacketFilter.h @@ -71,8 +71,8 @@ inline bool resolveToken(const char* begin, size_t len, uint8_t* type_out) { } // Parse an allowlist value. Empty input means "all" so WebConfig can clear a -// field and retain the same backwards-compatible default as an older -// /mqtt_prefs file. Keywords and type names are deliberately lowercase; "all" +// field and retain the same all-packet-types default as an older preference +// file. Keywords and type names are deliberately lowercase; "all" // and "none" cannot be mixed into a list. Entries may be decimal (0..15) or // named, may carry surrounding ASCII whitespace, and may be repeated. inline bool parse(const char* input, uint16_t* mask_out) { @@ -186,8 +186,8 @@ inline uint16_t enabledUnion(const uint16_t* masks, const bool* enabled, size_t return combined; } -// True when every slot still carries the default all-types mask, i.e. nothing -// depends on the packet-filter tail of /mqtt_prefs being written. +// True when every slot still carries the default all-types mask. The legacy +// binary encoder uses this while producing migration fixtures. inline bool allMasksDefault(const uint16_t* masks, size_t count) { if (masks == nullptr) return true; for (size_t i = 0; i < count; ++i) { diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h index be77bf99..c3b3ce20 100644 --- a/src/helpers/MQTTPrefsAtomicStore.h +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -3,12 +3,51 @@ #include #include -// Transactional writer for /mqtt_prefs. The Store interface is intentionally -// narrow so host tests can exercise every failure boundary without an Arduino -// filesystem: begin(), write(), finish(), commit(), and abort(). The caller -// supplies header and payload separately, avoiding a second full-size buffer. +// Transactional writer policy for preferences. The Store interface is +// intentionally narrow so host tests can exercise every failure boundary +// without an Arduino filesystem. namespace MQTTPrefsAtomicStore { +// Production /mqtt.json flow. finish() verifies the exact bytes written, +// verify_image reparses the finished temp through the schema, and only then is +// commit() allowed to publish it. A schema-verification failure discards the +// finished temp; a commit failure preserves it for boot recovery. +enum class VerifiedImageResult : uint8_t { + Committed, + BeginFailed, + WriteFailed, + FinishFailed, + VerifyFailed, + CommitFailed, +}; + +template +inline VerifiedImageResult writeVerifiedImage(Store& store, + ImageWriter write_image, + ImageVerifier verify_image) { + if (!store.begin()) { + store.abort(); + return VerifiedImageResult::BeginFailed; + } + if (!write_image()) { + store.abort(); + return VerifiedImageResult::WriteFailed; + } + if (!store.finish()) { + store.abort(); + return VerifiedImageResult::FinishFailed; + } + if (!verify_image()) { + store.discardFinishedTemp(); + return VerifiedImageResult::VerifyFailed; + } + if (!store.commit()) { + store.abort(); + return VerifiedImageResult::CommitFailed; + } + return VerifiedImageResult::Committed; +} + enum class Result : uint8_t { Committed, BeginFailed, @@ -59,7 +98,7 @@ inline ImageResult writeImage(Store& store, ImageWriter write_image) { } // Coordinates a two-file legacy upgrade. /com_prefs must not be compacted -// until the observer tail it carries has been published into /mqtt_prefs. +// until the observer tail it carries has been published into /mqtt.json. // Keeping this state in a tiny pure helper lets host tests cover power-cut // boundaries without an Arduino filesystem. class LegacyUpgradeGate { diff --git a/src/helpers/MQTTPrefsCodec.h b/src/helpers/MQTTPrefsCodec.h index 7bbfe30a..5ac073c0 100644 --- a/src/helpers/MQTTPrefsCodec.h +++ b/src/helpers/MQTTPrefsCodec.h @@ -53,6 +53,9 @@ static const size_t kEncodedSize = sizeof(MQTTPrefsHeader) + kV1BaselinePayloadS // refuses to overwrite the file, so the node cannot be recovered over the air. // Touching any filter opts that node into the longer payload — a deliberate, // operator-initiated trade rather than a side effect of upgrading. +// The encoder below is retained for host migration fixtures and downgrade +// compatibility tests. Production firmware reads legacy binary files but only +// writes the JSON schema. inline size_t payloadLenFor(const MQTTPrefs& prefs) { return MQTTPacketFilter::allMasksDefault(prefs.mqtt_slot_packet_filter, MQTT_PREFS_SLOT_COUNT) @@ -68,13 +71,60 @@ inline MQTTPrefsHeader makeHeader(size_t payload_len) { return header; } +inline void freezeV1(const MQTTPrefs& prefs, LegacyV1MQTTPrefs* frozen) { + if (frozen == nullptr) return; + memset(frozen, 0, sizeof(*frozen)); + memcpy(frozen->mqtt_origin, prefs.mqtt_origin, sizeof(frozen->mqtt_origin)); + memcpy(frozen->mqtt_iata, prefs.mqtt_iata, sizeof(frozen->mqtt_iata)); + frozen->mqtt_status_enabled = prefs.mqtt_status_enabled; + frozen->mqtt_packets_enabled = prefs.mqtt_packets_enabled; + frozen->mqtt_raw_enabled = prefs.mqtt_raw_enabled; + frozen->mqtt_tx_enabled = prefs.mqtt_tx_enabled; + frozen->mqtt_status_interval = prefs.mqtt_status_interval; + memcpy(frozen->wifi_ssid, prefs.wifi_ssid, sizeof(frozen->wifi_ssid)); + memcpy(frozen->wifi_password, prefs.wifi_password, sizeof(frozen->wifi_password)); + frozen->wifi_power_save = prefs.wifi_power_save; + memcpy(frozen->timezone_string, prefs.timezone_string, sizeof(frozen->timezone_string)); + frozen->timezone_offset = prefs.timezone_offset; + memcpy(frozen->mqtt_slot_preset, prefs.mqtt_slot_preset, sizeof(frozen->mqtt_slot_preset)); + memcpy(frozen->mqtt_slot_host, prefs.mqtt_slot_host, sizeof(frozen->mqtt_slot_host)); + memcpy(frozen->mqtt_slot_port, prefs.mqtt_slot_port, sizeof(frozen->mqtt_slot_port)); + memcpy(frozen->mqtt_slot_username, prefs.mqtt_slot_username, sizeof(frozen->mqtt_slot_username)); + memcpy(frozen->mqtt_slot_password, prefs.mqtt_slot_password, sizeof(frozen->mqtt_slot_password)); + memcpy(frozen->mqtt_owner_public_key, prefs.mqtt_owner_public_key, + sizeof(frozen->mqtt_owner_public_key)); + memcpy(frozen->mqtt_email, prefs.mqtt_email, sizeof(frozen->mqtt_email)); + memcpy(frozen->mqtt_slot_token, prefs.mqtt_slot_token, sizeof(frozen->mqtt_slot_token)); + memcpy(frozen->mqtt_slot_topic, prefs.mqtt_slot_topic, sizeof(frozen->mqtt_slot_topic)); + memcpy(frozen->mqtt_slot_audience, prefs.mqtt_slot_audience, + sizeof(frozen->mqtt_slot_audience)); + frozen->mqtt_rx_enabled = prefs.mqtt_rx_enabled; + memcpy(frozen->mqtt_ntp_server, prefs.mqtt_ntp_server, sizeof(frozen->mqtt_ntp_server)); + frozen->snmp_enabled = prefs.snmp_enabled; + memcpy(frozen->snmp_community, prefs.snmp_community, sizeof(frozen->snmp_community)); + frozen->radio_watchdog_minutes = prefs.radio_watchdog_minutes; + frozen->alert_enabled = prefs.alert_enabled; + memcpy(frozen->alert_psk_hex, prefs.alert_psk_hex, sizeof(frozen->alert_psk_hex)); + frozen->alert_wifi_minutes = prefs.alert_wifi_minutes; + frozen->alert_mqtt_minutes = prefs.alert_mqtt_minutes; + frozen->alert_min_interval_min = prefs.alert_min_interval_min; + memcpy(frozen->alert_hashtag, prefs.alert_hashtag, sizeof(frozen->alert_hashtag)); + memcpy(frozen->alert_region, prefs.alert_region, sizeof(frozen->alert_region)); + frozen->mqtt_neighbors_enabled = prefs.mqtt_neighbors_enabled; + frozen->mqtt_neighbors_interval = prefs.mqtt_neighbors_interval; + memcpy(frozen->mqtt_slot_packet_filter, prefs.mqtt_slot_packet_filter, + sizeof(frozen->mqtt_slot_packet_filter)); +} + inline size_t encode(const MQTTPrefs& prefs, uint8_t* output, size_t output_size) { const size_t payload_len = payloadLenFor(prefs); const size_t encoded_size = sizeof(MQTTPrefsHeader) + payload_len; if (output == nullptr || output_size < encoded_size) return 0; const MQTTPrefsHeader header = makeHeader(payload_len); + LegacyV1MQTTPrefs frozen; + freezeV1(prefs, &frozen); memcpy(output, &header, sizeof(header)); - memcpy(output + sizeof(header), &prefs, payload_len); + memcpy(output + sizeof(header), &frozen, payload_len); return encoded_size; } @@ -346,6 +396,120 @@ inline bool isPlausibleLegacy(Source source, const uint8_t* input, size_t size) } } +// A valid v1 header proves the intended layout, not that a torn or corrupted +// payload still contains bounded C strings. Validate every field present at a +// shipped v1 boundary before copying it into the runtime object; otherwise the +// later semantic validators and JSON writer could scan beyond a fixed array. +inline bool isPlausibleV1(const LegacyV1MQTTPrefs& prefs, size_t payload_len) { + if (payload_len != kV1PreObserverPayloadSize && + payload_len != kV1PreNeighborsPayloadSize && + payload_len != kV1PreFilterPayloadSize && + payload_len != kV1BaselinePayloadSize) { + return false; + } + const uint8_t* input = reinterpret_cast(&prefs); + // A v1 header already identifies the layout. At this stage only reject + // fields that could make later C-string validation/serialization read past a + // fixed array; numeric values are safely repaired by MQTTPrefsSerializer. + if (!hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_origin), sizeof(prefs.mqtt_origin)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_iata), sizeof(prefs.mqtt_iata)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, wifi_ssid), sizeof(prefs.wifi_ssid)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, wifi_password), sizeof(prefs.wifi_password)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, timezone_string), sizeof(prefs.timezone_string)) || + !hasPlausibleSharedAuth(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_owner_public_key), + offsetof(LegacyV1MQTTPrefs, mqtt_email)) || + !hasPlausibleSlotText(input, payload_len, MQTT_PREFS_SLOT_COUNT, + offsetof(LegacyV1MQTTPrefs, mqtt_slot_preset), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_host), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_username), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_password), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_token), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_topic), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_audience)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_ntp_server), + sizeof(prefs.mqtt_ntp_server))) { + return false; + } + if (payload_len >= kV1PreNeighborsPayloadSize) { + if (!hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, snmp_community), + sizeof(prefs.snmp_community)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_psk_hex), + sizeof(prefs.alert_psk_hex)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_hashtag), + sizeof(prefs.alert_hashtag)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_region), + sizeof(prefs.alert_region))) { + return false; + } + } + return true; +} + +inline void migrateV1(const LegacyV1MQTTPrefs& old_prefs, size_t payload_len, + MQTTPrefs* prefs) { + if (prefs == nullptr || payload_len < kV1PreObserverPayloadSize) return; + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_slot_preset, old_prefs.mqtt_slot_preset, sizeof(prefs->mqtt_slot_preset)); + memcpy(prefs->mqtt_slot_host, old_prefs.mqtt_slot_host, sizeof(prefs->mqtt_slot_host)); + memcpy(prefs->mqtt_slot_port, old_prefs.mqtt_slot_port, sizeof(prefs->mqtt_slot_port)); + memcpy(prefs->mqtt_slot_username, old_prefs.mqtt_slot_username, sizeof(prefs->mqtt_slot_username)); + memcpy(prefs->mqtt_slot_password, old_prefs.mqtt_slot_password, sizeof(prefs->mqtt_slot_password)); + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + memcpy(prefs->mqtt_slot_token, old_prefs.mqtt_slot_token, sizeof(prefs->mqtt_slot_token)); + memcpy(prefs->mqtt_slot_topic, old_prefs.mqtt_slot_topic, sizeof(prefs->mqtt_slot_topic)); + memcpy(prefs->mqtt_slot_audience, old_prefs.mqtt_slot_audience, + sizeof(prefs->mqtt_slot_audience)); + prefs->mqtt_rx_enabled = old_prefs.mqtt_rx_enabled; + memcpy(prefs->mqtt_ntp_server, old_prefs.mqtt_ntp_server, sizeof(prefs->mqtt_ntp_server)); + + if (payload_len >= kV1PreNeighborsPayloadSize) { + prefs->snmp_enabled = old_prefs.snmp_enabled; + memcpy(prefs->snmp_community, old_prefs.snmp_community, sizeof(prefs->snmp_community)); + prefs->radio_watchdog_minutes = old_prefs.radio_watchdog_minutes; + prefs->alert_enabled = old_prefs.alert_enabled; + memcpy(prefs->alert_psk_hex, old_prefs.alert_psk_hex, sizeof(prefs->alert_psk_hex)); + prefs->alert_wifi_minutes = old_prefs.alert_wifi_minutes; + prefs->alert_mqtt_minutes = old_prefs.alert_mqtt_minutes; + prefs->alert_min_interval_min = old_prefs.alert_min_interval_min; + memcpy(prefs->alert_hashtag, old_prefs.alert_hashtag, sizeof(prefs->alert_hashtag)); + memcpy(prefs->alert_region, old_prefs.alert_region, sizeof(prefs->alert_region)); + // The pre-neighbors payload includes the old zero padding byte at this + // offset; copying it preserves the deployed format's disabled default. + prefs->mqtt_neighbors_enabled = old_prefs.mqtt_neighbors_enabled; + } + if (payload_len >= kV1PreFilterPayloadSize) { + prefs->mqtt_neighbors_interval = old_prefs.mqtt_neighbors_interval; + } + if (payload_len >= kV1BaselinePayloadSize) { + memcpy(prefs->mqtt_slot_packet_filter, old_prefs.mqtt_slot_packet_filter, + sizeof(prefs->mqtt_slot_packet_filter)); + } +} + inline void migratePreSlot(const OldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); diff --git a/src/helpers/MQTTPrefsRecovery.h b/src/helpers/MQTTPrefsRecovery.h index 7534678e..37073209 100644 --- a/src/helpers/MQTTPrefsRecovery.h +++ b/src/helpers/MQTTPrefsRecovery.h @@ -2,23 +2,29 @@ #include -// Pure recovery policy for the three MQTT preference transaction files. The +// Pure recovery policy for the three MQTT preference transaction files. The // writer first moves the old primary to .bak, then moves the verified .tmp to -// the primary name. On a reset, the loader uses this policy before decoding -// /mqtt_prefs. "Preserve" is deliberately distinct from "Usable": it covers -// an unsupported newer layout, corruption, or an unreadable file and must -// never be replaced by an older image. +// the primary name. On a reset, the loader uses this policy before decoding +// the primary. FutureUsable is syntactically valid but belongs to newer +// firmware. FutureClaimed and Indeterminate cannot be safely classified by +// this firmware, so recovery may use a known-good backup but must retain the +// uncertain image and hold further writes. Preserve is definitively invalid or +// unsupported and may be discarded only where the transaction policy permits. namespace MQTTPrefsRecovery { enum class FileState : uint8_t { Missing, Usable, + FutureUsable, + FutureClaimed, + Indeterminate, Preserve, }; enum class Action : uint8_t { None, KeepPrimary, + DiscardTemp, PromoteTemp, PromoteBackup, }; @@ -30,13 +36,23 @@ inline Action select(FileState primary, FileState temp, FileState backup) { if (primary != FileState::Missing) return Action::KeepPrimary; // A completed temp is the new image and wins over the old backup. - if (temp == FileState::Usable) return Action::PromoteTemp; + if (temp == FileState::Usable || temp == FileState::FutureUsable) { + return Action::PromoteTemp; + } - // If temp is opaque but a known-good backup exists, boot from the backup. - // The caller may discard the opaque temp once that usable backup has become - // primary. Otherwise, rename the opaque temp into the empty primary name so - // the normal loader can hold it. + // Preserve is definitively invalid, so it was never a committed image. If + // no backup exists, discard the interrupted temp and leave the primary name + // absent; this is what lets a first JSON migration retry from /mqtt_prefs. + // If a backup exists, it is the prior committed image and wins regardless + // of whether this firmware understands its schema. if (temp == FileState::Preserve) { + return backup == FileState::Missing ? Action::DiscardTemp : Action::PromoteBackup; + } + + // FutureClaimed and Indeterminate may be a completed image this firmware + // cannot classify. A known-good backup can run this boot, but without one + // preserve the only candidate under the authoritative name and hold writes. + if (temp == FileState::FutureClaimed || temp == FileState::Indeterminate) { return backup == FileState::Usable ? Action::PromoteBackup : Action::PromoteTemp; } @@ -46,4 +62,8 @@ inline Action select(FileState primary, FileState temp, FileState backup) { return Action::None; } +inline bool uncertain(FileState state) { + return state == FileState::FutureClaimed || state == FileState::Indeterminate; +} + } // namespace MQTTPrefsRecovery diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h new file mode 100644 index 00000000..b4d12b18 --- /dev/null +++ b/src/helpers/MQTTPrefsSerializer.h @@ -0,0 +1,412 @@ +#pragma once + +#include +#include +#include +#include + +#ifdef WITH_MQTT_BRIDGE + +static const int32_t MQTT_PREFS_JSON_FORMAT_VERSION = 1; + +// Reads only the mandatory root version while ignoring every other schema +// field. Recovery uses this before the v1 decoder so a syntactically valid +// future file remains opaque even when that schema changes a v1 field's type. +class MQTTPrefsVersionProbe : public ConfigSerializer { + int32_t _version = 0; + bool _seen_version = false; +protected: + void structure() override { defStrict("version", _version, _seen_version); } +public: + bool hasFutureVersion() const { + return _seen_version && _version > MQTT_PREFS_JSON_FORMAT_VERSION; + } +}; + +// ConfigSerializer adapter for the observer preference POD. Keeping the +// serializer separate avoids making the runtime object layout part of the JSON +// format and adds no permanent per-slot serializer state to CommonCLI. +class MQTTPrefsSerializer : public ConfigSerializer { + + class WifiPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _power_save; + bool _seen_ssid = false, _seen_password = false, _seen_power_save = false; + protected: + void structure() override { + defStrict("ssid", _prefs->wifi_ssid, sizeof(_prefs->wifi_ssid), _seen_ssid); + defStrict("password", _prefs->wifi_password, sizeof(_prefs->wifi_password), _seen_password); + defStrict("power_save", _power_save, _seen_power_save); + } + public: + explicit WifiPrefs(MQTTPrefs* prefs) : _prefs(prefs), _power_save(prefs->wifi_power_save) {} + bool apply(bool* repaired) { + if (_power_save < 0 || _power_save > 2) { + _power_save = 1; + *repaired = true; + } + _prefs->wifi_power_save = static_cast(_power_save); + return true; + } + }; + + class TimePrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _utc_offset, _default_utc_offset; + char _default_ntp_server[sizeof(MQTTPrefs::mqtt_ntp_server)]; + bool _seen_timezone = false, _seen_utc_offset = false, _seen_ntp_server = false; + protected: + void structure() override { + defStrict("timezone", _prefs->timezone_string, sizeof(_prefs->timezone_string), _seen_timezone); + defStrict("utc_offset", _utc_offset, _seen_utc_offset); + defStrict("ntp_server", _prefs->mqtt_ntp_server, sizeof(_prefs->mqtt_ntp_server), _seen_ntp_server); + } + public: + TimePrefs(MQTTPrefs* prefs, const MQTTPrefs* defaults) + : _prefs(prefs), _utc_offset(prefs->timezone_offset), + _default_utc_offset(defaults->timezone_offset) { + if (_default_utc_offset < -12 || _default_utc_offset > 14) { + _default_utc_offset = 0; + } + memcpy(_default_ntp_server, defaults->mqtt_ntp_server, sizeof(_default_ntp_server)); + if (_default_ntp_server[0] != '\0' && + !mqttNtpHostnameValid(_default_ntp_server)) { + _default_ntp_server[0] = '\0'; + } + } + bool apply(bool* repaired) { + if (_utc_offset < -12 || _utc_offset > 14) { + _utc_offset = _default_utc_offset; + *repaired = true; + } + if (_prefs->mqtt_ntp_server[0] != '\0' && + !mqttNtpHostnameValid(_prefs->mqtt_ntp_server)) { + memcpy(_prefs->mqtt_ntp_server, _default_ntp_server, sizeof(_default_ntp_server)); + *repaired = true; + } + _prefs->timezone_offset = static_cast(_utc_offset); + return true; + } + }; + + class StatusPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _interval_ms; + bool _seen_enabled = false, _seen_interval = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("interval_ms", _interval_ms, _seen_interval); + } + public: + explicit StatusPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->mqtt_status_enabled), + _interval_ms(static_cast(prefs->mqtt_status_interval)) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 1; *repaired = true; } + if (_interval_ms < 60000 || _interval_ms > 3600000) { + _interval_ms = 300000; + *repaired = true; + } + _prefs->mqtt_status_enabled = static_cast(_enabled); + _prefs->mqtt_status_interval = static_cast(_interval_ms); + } + }; + + class NeighborPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _interval_ms; + bool _seen_enabled = false, _seen_interval = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("interval_ms", _interval_ms, _seen_interval); + } + public: + explicit NeighborPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->mqtt_neighbors_enabled), + _interval_ms(static_cast(prefs->mqtt_neighbors_interval)) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + if (_interval_ms < static_cast(MQTT_NEIGHBORS_MIN_INTERVAL_MS) || + _interval_ms > static_cast(MQTT_NEIGHBORS_MAX_INTERVAL_MS)) { + _interval_ms = static_cast(MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS); + *repaired = true; + } + _prefs->mqtt_neighbors_enabled = static_cast(_enabled); + _prefs->mqtt_neighbors_interval = static_cast(_interval_ms); + } + }; + + class OwnerPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + bool _seen_public_key = false, _seen_email = false; + protected: + void structure() override { + defStrict("public_key", _prefs->mqtt_owner_public_key, + sizeof(_prefs->mqtt_owner_public_key), _seen_public_key); + defStrict("email", _prefs->mqtt_email, sizeof(_prefs->mqtt_email), _seen_email); + } + public: + explicit OwnerPrefs(MQTTPrefs* prefs) : _prefs(prefs) {} + void apply(bool* repaired) { + if (_prefs->mqtt_owner_public_key[0] != '\0' && + !mqttOwnerKeyValid(_prefs->mqtt_owner_public_key)) { + _prefs->mqtt_owner_public_key[0] = '\0'; + *repaired = true; + } + } + }; + + class SlotPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int _index; + int32_t _port, _filter; + bool _seen_preset = false, _seen_host = false, _seen_port = false; + bool _seen_username = false, _seen_password = false, _seen_token = false; + bool _seen_topic = false, _seen_audience = false, _seen_filter = false; + protected: + void structure() override { + defStrict("preset", _prefs->mqtt_slot_preset[_index], + sizeof(_prefs->mqtt_slot_preset[_index]), _seen_preset); + defStrict("host", _prefs->mqtt_slot_host[_index], + sizeof(_prefs->mqtt_slot_host[_index]), _seen_host); + defStrict("port", _port, _seen_port); + defStrict("username", _prefs->mqtt_slot_username[_index], + sizeof(_prefs->mqtt_slot_username[_index]), _seen_username); + defStrict("password", _prefs->mqtt_slot_password[_index], + sizeof(_prefs->mqtt_slot_password[_index]), _seen_password); + defStrict("token", _prefs->mqtt_slot_token[_index], + sizeof(_prefs->mqtt_slot_token[_index]), _seen_token); + defStrict("topic", _prefs->mqtt_slot_topic[_index], + sizeof(_prefs->mqtt_slot_topic[_index]), _seen_topic); + defStrict("audience", _prefs->mqtt_slot_audience[_index], + sizeof(_prefs->mqtt_slot_audience[_index]), _seen_audience); + defStrict("packet_filter", _filter, _seen_filter); + } + public: + SlotPrefs(MQTTPrefs* prefs, int index) + : _prefs(prefs), _index(index), _port(prefs->mqtt_slot_port[index]), + _filter(prefs->mqtt_slot_packet_filter[index]) {} + void apply(bool* repaired) { + if (_port < 0 || _port > 65535) { _port = 0; *repaired = true; } + if (_filter < 0 || _filter > 65535) { _filter = 0xffff; *repaired = true; } + const char* preset = _prefs->mqtt_slot_preset[_index]; + if (strcmp(preset, MQTT_PRESET_NONE) != 0 && + strcmp(preset, MQTT_PRESET_CUSTOM) != 0 && findMQTTPreset(preset) == nullptr) { + strcpy(_prefs->mqtt_slot_preset[_index], MQTT_PRESET_NONE); + *repaired = true; + } + _prefs->mqtt_slot_port[_index] = static_cast(_port); + _prefs->mqtt_slot_packet_filter[_index] = static_cast(_filter); + } + }; + + class MqttPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _packets_enabled, _raw_enabled, _tx_enabled, _rx_enabled; + char _default_iata[sizeof(MQTTPrefs::mqtt_iata)]; + bool _seen_origin = false, _seen_iata = false, _seen_packets = false; + bool _seen_raw = false, _seen_tx = false, _seen_rx = false; + StatusPrefs _status; + NeighborPrefs _neighbors; + OwnerPrefs _owner; + static_assert(MQTT_PREFS_SLOT_COUNT == 6, + "MQTTPrefsSerializer slot members and keys must be updated"); + SlotPrefs _slot1, _slot2, _slot3, _slot4, _slot5, _slot6; + protected: + void structure() override { + defStrict("origin", _prefs->mqtt_origin, sizeof(_prefs->mqtt_origin), _seen_origin); + defStrict("iata", _prefs->mqtt_iata, sizeof(_prefs->mqtt_iata), _seen_iata); + defStrict("packets_enabled", _packets_enabled, _seen_packets); + defStrict("raw_enabled", _raw_enabled, _seen_raw); + defStrict("tx_enabled", _tx_enabled, _seen_tx); + defStrict("rx_enabled", _rx_enabled, _seen_rx); + def("status", _status); + def("neighbors", _neighbors); + def("owner", _owner); + def("slot1", _slot1); + def("slot2", _slot2); + def("slot3", _slot3); + def("slot4", _slot4); + def("slot5", _slot5); + def("slot6", _slot6); + } + public: + MqttPrefs(MQTTPrefs* prefs, const MQTTPrefs* defaults) + : _prefs(prefs), _packets_enabled(prefs->mqtt_packets_enabled), + _raw_enabled(prefs->mqtt_raw_enabled), _tx_enabled(prefs->mqtt_tx_enabled), + _rx_enabled(prefs->mqtt_rx_enabled), _status(prefs), _neighbors(prefs), + _owner(prefs), _slot1(prefs, 0), _slot2(prefs, 1), + _slot3(prefs, 2), _slot4(prefs, 3), + _slot5(prefs, 4), _slot6(prefs, 5) { + memcpy(_default_iata, defaults->mqtt_iata, sizeof(_default_iata)); + if (_default_iata[0] != '\0' && !mqttIataValid(_default_iata)) { + _default_iata[0] = '\0'; + } + for (char* p = _default_iata; *p; ++p) { + if (*p >= 'a' && *p <= 'z') *p = static_cast(*p - ('a' - 'A')); + } + } + void apply(bool* repaired) { + if (_packets_enabled < 0 || _packets_enabled > 1) { _packets_enabled = 1; *repaired = true; } + if (_raw_enabled < 0 || _raw_enabled > 1) { _raw_enabled = 0; *repaired = true; } + if (_tx_enabled < 0 || _tx_enabled > 2) { _tx_enabled = 2; *repaired = true; } + if (_rx_enabled < 0 || _rx_enabled > 1) { _rx_enabled = 1; *repaired = true; } + _prefs->mqtt_packets_enabled = static_cast(_packets_enabled); + _prefs->mqtt_raw_enabled = static_cast(_raw_enabled); + _prefs->mqtt_tx_enabled = static_cast(_tx_enabled); + _prefs->mqtt_rx_enabled = static_cast(_rx_enabled); + if (_prefs->mqtt_iata[0] != '\0') { + if (!mqttIataValid(_prefs->mqtt_iata)) { + memcpy(_prefs->mqtt_iata, _default_iata, sizeof(_default_iata)); + *repaired = true; + } else { + for (char* p = _prefs->mqtt_iata; *p; ++p) { + if (*p >= 'a' && *p <= 'z') { + *p = static_cast(*p - ('a' - 'A')); + *repaired = true; + } + } + } + } + _status.apply(repaired); + _neighbors.apply(repaired); + _owner.apply(repaired); + _slot1.apply(repaired); _slot2.apply(repaired); _slot3.apply(repaired); + _slot4.apply(repaired); _slot5.apply(repaired); _slot6.apply(repaired); + } + }; + + class SnmpPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled; + bool _seen_enabled = false, _seen_community = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("community", _prefs->snmp_community, + sizeof(_prefs->snmp_community), _seen_community); + } + public: + explicit SnmpPrefs(MQTTPrefs* prefs) : _prefs(prefs), _enabled(prefs->snmp_enabled) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + _prefs->snmp_enabled = static_cast(_enabled); + } + }; + + class RadioPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _watchdog_min; + bool _seen_watchdog = false; + protected: + void structure() override { defStrict("watchdog_min", _watchdog_min, _seen_watchdog); } + public: + explicit RadioPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _watchdog_min(prefs->radio_watchdog_minutes) {} + void apply(bool* repaired) { + if (_watchdog_min < 0 || _watchdog_min > 120) { _watchdog_min = 5; *repaired = true; } + _prefs->radio_watchdog_minutes = static_cast(_watchdog_min); + } + }; + + class AlertPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _wifi_minutes, _mqtt_minutes, _rate_limit_min; + bool _seen_enabled = false, _seen_psk = false, _seen_wifi = false; + bool _seen_mqtt = false, _seen_rate = false, _seen_hashtag = false, _seen_region = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("psk_hex", _prefs->alert_psk_hex, sizeof(_prefs->alert_psk_hex), _seen_psk); + defStrict("wifi_minutes", _wifi_minutes, _seen_wifi); + defStrict("mqtt_minutes", _mqtt_minutes, _seen_mqtt); + defStrict("rate_limit_min", _rate_limit_min, _seen_rate); + defStrict("hashtag", _prefs->alert_hashtag, sizeof(_prefs->alert_hashtag), _seen_hashtag); + defStrict("region", _prefs->alert_region, sizeof(_prefs->alert_region), _seen_region); + } + public: + explicit AlertPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->alert_enabled), + _wifi_minutes(prefs->alert_wifi_minutes), _mqtt_minutes(prefs->alert_mqtt_minutes), + _rate_limit_min(prefs->alert_min_interval_min) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + if (_wifi_minutes < 0 || _wifi_minutes > 1440) { _wifi_minutes = 30; *repaired = true; } + if (_mqtt_minutes < 0 || _mqtt_minutes > 10080) { _mqtt_minutes = 240; *repaired = true; } + if (_rate_limit_min < 60 || _rate_limit_min > 10080) { _rate_limit_min = 60; *repaired = true; } + if (_prefs->alert_psk_hex[0] != '\0') { + const size_t len = strlen(_prefs->alert_psk_hex); + bool valid_hex = len == 32; + for (size_t i = 0; valid_hex && i < len; ++i) { + const char c = _prefs->alert_psk_hex[i]; + valid_hex = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || + (c >= 'a' && c <= 'f'); + } + if (!valid_hex) { + _prefs->alert_psk_hex[0] = '\0'; + _prefs->alert_hashtag[0] = '\0'; + *repaired = true; + } + } + _prefs->alert_enabled = static_cast(_enabled); + _prefs->alert_wifi_minutes = static_cast(_wifi_minutes); + _prefs->alert_mqtt_minutes = static_cast(_mqtt_minutes); + _prefs->alert_min_interval_min = static_cast(_rate_limit_min); + } + }; + + MQTTPrefs* _prefs; + int32_t _version = MQTT_PREFS_JSON_FORMAT_VERSION; + bool _seen_version = false; + WifiPrefs _wifi; + TimePrefs _time; + MqttPrefs _mqtt; + SnmpPrefs _snmp; + RadioPrefs _radio; + AlertPrefs _alert; + +protected: + void structure() override { + defStrict("version", _version, _seen_version); + def("wifi", _wifi); + def("time", _time); + def("mqtt", _mqtt); + def("snmp", _snmp); + def("radio", _radio); + def("alert", _alert); + } + +public: + explicit MQTTPrefsSerializer(MQTTPrefs* prefs, const MQTTPrefs* repair_defaults = nullptr) + : _prefs(prefs), _wifi(prefs), + _time(prefs, repair_defaults ? repair_defaults : prefs), + _mqtt(prefs, repair_defaults ? repair_defaults : prefs), _snmp(prefs), + _radio(prefs), _alert(prefs) {} + + bool hasSupportedVersion() const { + return _seen_version && _version == MQTT_PREFS_JSON_FORMAT_VERSION; + } + bool hasFutureVersion() const { + return _seen_version && _version > MQTT_PREFS_JSON_FORMAT_VERSION; + } + + bool normalize(bool* repaired) { + if (repaired == nullptr) return false; + *repaired = false; + _wifi.apply(repaired); + _time.apply(repaired); + _mqtt.apply(repaired); + _snmp.apply(repaired); + _radio.apply(repaired); + _alert.apply(repaired); + return true; + } + + bool apply(bool* repaired) { + return hasSupportedVersion() && normalize(repaired); + } +}; + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h index e3c6a098..0bc0867e 100644 --- a/src/helpers/MQTTPrefsStorage.h +++ b/src/helpers/MQTTPrefsStorage.h @@ -62,10 +62,9 @@ struct PreWifiPowerOldMQTTPrefs { char mqtt_email[64]; }; -// MQTT preferences stored separately from NodePrefs to avoid upstream layout -// conflicts. The full layout is the frozen v1 payload baseline. The prefix -// before observer settings is also an explicitly supported v1 payload: it was -// used before the observer fields were appended. +// Runtime MQTT preferences, stored separately from NodePrefs to avoid upstream +// layout conflicts. JSON persistence is field-based; this type is deliberately +// free to evolve independently of the frozen binary migration layouts below. struct MQTTPrefs { char mqtt_origin[32]; char mqtt_iata[8]; @@ -110,17 +109,57 @@ struct MQTTPrefs { char alert_region[31]; // Neighbors publishing (PSRAM boards only). Appended at the end of the - // observer tail so a shorter (pre-neighbors) /mqtt_prefs payload from earlier - // firmware still loads with these defaulting off/24h; keeps the format at - // VERSION 1. Field order and sizes are kept byte-identical to the flex - // neighbors build so a /mqtt_prefs written by either firmware is - // interchangeable (see the offsetof static_asserts below). + // observer tail in the former binary format. Binary compatibility is now + // represented only by LegacyV1MQTTPrefs below. uint8_t mqtt_neighbors_enabled; uint32_t mqtt_neighbors_interval; // Per-slot payload-type allow masks. Bit N controls MeshCore packet type N - // for both packets and raw MQTT topics. Appended so older v1 payloads load - // with the default all-types masks intact. + // for both packets and raw MQTT topics. + uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; +}; + +// Frozen payload written by the version-1 binary format. Keep this distinct +// from the runtime MQTTPrefs type: JSON persistence must not make the runtime +// object's padding or member order an on-flash ABI again. Legacy decoding reads +// into this type and field-copies into a defaulted runtime object. +struct LegacyV1MQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; + uint8_t snmp_enabled; + char snmp_community[24]; + uint8_t radio_watchdog_minutes; + uint8_t alert_enabled; + char alert_psk_hex[33]; + uint16_t alert_wifi_minutes; + uint16_t alert_mqtt_minutes; + uint16_t alert_min_interval_min; + char alert_hashtag[24]; + char alert_region[31]; + uint8_t mqtt_neighbors_enabled; + uint32_t mqtt_neighbors_interval; uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; }; @@ -272,20 +311,20 @@ static const size_t LEGACY6_AUDIENCE_RX_SIZE = 2840; // Frozen on-flash layouts; every firmware and native fixture build checks them. static_assert(sizeof(MQTTPrefsHeader) == 8, "versioned /mqtt_prefs header must stay 8 bytes"); -static_assert(offsetof(MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, "v1 pre-observer /mqtt_prefs boundary changed"); -static_assert(sizeof(MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, +static_assert(sizeof(LegacyV1MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, "v1 /mqtt_prefs payload layout changed"); // Lock the neighbors tail to the flex neighbors build's layout so a /mqtt_prefs // written by either firmware is byte-for-byte interchangeable. The enable flag // lands in the old struct's zeroed trailing padding (offset 2857), and the // interval begins exactly at the pre-neighbors payload size (2860) so a // pre-neighbors read stops right before it and the interval keeps its default. -static_assert(offsetof(MQTTPrefs, mqtt_neighbors_enabled) == 2857, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_neighbors_enabled) == 2857, "neighbors enable flag must sit at the flex-compatible offset"); -static_assert(offsetof(MQTTPrefs, mqtt_neighbors_interval) == MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_neighbors_interval) == MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE, "neighbors interval offset must equal the pre-neighbors payload size"); -static_assert(offsetof(MQTTPrefs, mqtt_slot_packet_filter) == MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_slot_packet_filter) == MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE, "packet filters must begin at the pre-filter payload boundary"); static_assert(sizeof(OldMQTTPrefs) == 472, "frozen pre-slot /mqtt_prefs layout changed"); static_assert(sizeof(PreWifiPowerOldMQTTPrefs) == 472, "frozen pre-WiFi-power /mqtt_prefs layout changed"); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 143aae36..aa2660cf 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -468,7 +468,7 @@ private: LifecycleOps _lifecycle_ops; MQTTLifecycle::Coordinator _lifecycle; - // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt_prefs. + // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // _prefs (held by BridgeBase) still provides upstream fields (freq/sf/node_name…). MQTTPrefs* _obs = nullptr; diff --git a/test/README.md b/test/README.md index 711e6484..80182510 100644 --- a/test/README.md +++ b/test/README.md @@ -33,7 +33,8 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_packet_filter` | `src/helpers/MQTTPacketFilter.h` | per-slot 0-15 allowlist parsing/formatting, numeric and named spellings; exact bounds; membership; candidate/eligible split and retry-completion policy; pre-queue union gate; default-mask detection | | `test_mqtt_runtime_buffer_lifecycle` | `src/helpers/MQTTRuntimeBufferLifecycle.h` | idempotent allocation/release; partial-allocation degradation; retry of only missing buffers | | `test_mqtt_prefs_codec` | `src/helpers/MQTTPrefsStorage.h`, `src/helpers/MQTTPrefsCodec.h` | binary pre-slot/3-slot/6-slot migration fixtures; v1 header integrity; downgrade preservation; shortest-payload write policy (default filters stay downgrade-readable) | -| `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h` | transactional MQTT writes and legacy `/node_prefs` handoff; exact short-write detection; begin/finish/rename failure cleanup; original-file preservation | +| `test_mqtt_prefs_serializer` | `src/helpers/MQTTPrefsSerializer.h`, `src/helpers/ConfigSerializer.*` | semantic nested `/mqtt.json` round trips; numeric slot keys; required/future version handling; strict length/overflow/duplicate rejection; safe semantic repair; scratch-before-live loading | +| `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h`, `src/helpers/MQTTPrefsRecovery.h` | production JSON begin/write/checksum-finish/schema-verify/commit orchestration; first-migration and rename-boundary recovery; legacy `/node_prefs` handoff; failure cleanup and original-file preservation | | `test_mqtt_payload_builder` | `src/helpers/MQTTPayloadBuilder.cpp` | status/packet/raw JSON contracts; optional fields; escaping; RX metrics and path; score handling; exact buffer bounds; maximum representative payloads | | `test_utils` | `src/Utils.cpp` | `Utils::toHex` (upstream) | diff --git a/test/test_config_serializer/test_config_serializer.cpp b/test/test_config_serializer/test_config_serializer.cpp index 7a13f487..3a5d215f 100644 --- a/test/test_config_serializer/test_config_serializer.cpp +++ b/test/test_config_serializer/test_config_serializer.cpp @@ -171,6 +171,23 @@ TEST(ConfigSerializer, LoadSerial_IgnoreUnknowns) { EXPECT_TRUE(match); } +TEST(ConfigSerializer, LoadSerial_KeyDigitsAfterFirstCharacter) { + MockInputStream s("{age:1,flags:2,name:\"ok\",slot1:7}"); + TestStruct data; + data.age = data.flags = 0; + strcpy(data.name, "before"); + EXPECT_TRUE(data.loadSerial(s)); + EXPECT_EQ(1, data.age); + EXPECT_EQ(2, data.flags); + EXPECT_STREQ("ok", data.name); +} + +TEST(ConfigSerializer, LoadSerial_RejectsLeadingDigitKey) { + MockInputStream s("{1slot:7}"); + TestStruct data; + EXPECT_FALSE(data.loadSerial(s)); +} + // ── main ─────────────────────────────────────────────────────── diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp index 67880a82..4d2ae120 100644 --- a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -20,6 +20,7 @@ enum class FailurePoint { PayloadWrite, ImageWrite, Finish, + Verify, Commit, }; @@ -44,7 +45,8 @@ public: ++write_calls; if (!_open) return 0; const bool should_fail = (write_calls == 1 && _failure == FailurePoint::HeaderWrite) || - (write_calls == 2 && _failure == FailurePoint::PayloadWrite); + (write_calls == 2 && _failure == FailurePoint::PayloadWrite) || + _failure == FailurePoint::ImageWrite; const size_t written = should_fail && size > 0 ? size - 1 : size; _staging.insert(_staging.end(), bytes, bytes + written); return written; @@ -68,6 +70,18 @@ public: return true; } + bool verify() { + ++verify_calls; + return _failure != FailurePoint::Verify; + } + + void discardFinishedTemp() { + ++discard_calls; + _files.erase("/mqtt_prefs.tmp"); + _finished = false; + _owns_temp = false; + } + void abort() { ++abort_calls; _open = false; @@ -88,6 +102,8 @@ public: int finish_calls = 0; int commit_calls = 0; int abort_calls = 0; + int verify_calls = 0; + int discard_calls = 0; private: FailurePoint _failure; @@ -113,6 +129,16 @@ AtomicStore::Result runWithObserverTail(InMemoryStore* store) { return AtomicStore::write(*store, header, sizeof(header), payload, sizeof(payload)); } +AtomicStore::VerifiedImageResult runVerifiedJson(InMemoryStore* store) { + const uint8_t json[] = "{version:1,wifi:{ssid:\"mesh\"}}"; + return AtomicStore::writeVerifiedImage( + *store, + [store, &json]() { + return store->write(json, sizeof(json) - 1) == sizeof(json) - 1; + }, + [store]() { return store->verify(); }); +} + class LegacyComPrefs { public: LegacyComPrefs() : bytes({'l', 'e', 'g', 'a', 'c', 'y', '-', 'c', 'o', 'm'}) {} @@ -286,9 +312,14 @@ public: } return; } + if (action == Recovery::Action::DiscardTemp) { + _files.erase("/mqtt_prefs.tmp"); + return; + } if (action == Recovery::Action::PromoteBackup) { rename("/mqtt_prefs.bak", "/mqtt_prefs"); - if (backup == Recovery::FileState::Usable && temp != Recovery::FileState::Missing) { + if (backup == Recovery::FileState::Usable && !Recovery::uncertain(temp) && + temp != Recovery::FileState::Missing) { _files.erase("/mqtt_prefs.tmp"); } return; @@ -298,12 +329,13 @@ public: // incomplete transaction artifact. It only preserves artifacts when the // primary itself is opaque. if (had_primary && primary == Recovery::FileState::Usable) { - _files.erase("/mqtt_prefs.tmp"); - _files.erase("/mqtt_prefs.bak"); + if (!Recovery::uncertain(temp)) _files.erase("/mqtt_prefs.tmp"); + if (!Recovery::uncertain(backup)) _files.erase("/mqtt_prefs.bak"); } } bool has(const char* path) const { return _files.count(path) != 0; } + void removePrimary() { _files.erase("/mqtt_prefs"); } bool canStartSave() const { return !has("/mqtt_prefs.tmp") && !has("/mqtt_prefs.bak"); } const std::vector& primary() const { return _files.at("/mqtt_prefs"); } static std::vector oldImage() { return {'o', 'l', 'd'}; } @@ -337,6 +369,50 @@ TEST(MQTTPrefsAtomicStore, CommitPublishesExactHeaderThenPayload) { EXPECT_EQ(0, store.abort_calls); } +TEST(MQTTPrefsAtomicStore, ProductionJsonPolicyCoversEveryVerificationBoundary) { + const std::vector old_source = { + 'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + const struct { + FailurePoint point; + AtomicStore::VerifiedImageResult expected; + int writes; + int finishes; + int verifies; + int commits; + int aborts; + int discards; + bool keeps_temp; + } cases[] = { + {FailurePoint::None, AtomicStore::VerifiedImageResult::Committed, + 1, 1, 1, 1, 0, 0, false}, + {FailurePoint::Begin, AtomicStore::VerifiedImageResult::BeginFailed, + 0, 0, 0, 0, 1, 0, false}, + {FailurePoint::ImageWrite, AtomicStore::VerifiedImageResult::WriteFailed, + 1, 0, 0, 0, 1, 0, false}, + {FailurePoint::Finish, AtomicStore::VerifiedImageResult::FinishFailed, + 1, 1, 0, 0, 1, 0, false}, + {FailurePoint::Verify, AtomicStore::VerifiedImageResult::VerifyFailed, + 1, 1, 1, 0, 0, 1, false}, + {FailurePoint::Commit, AtomicStore::VerifiedImageResult::CommitFailed, + 1, 1, 1, 1, 1, 0, true}, + }; + + for (const auto& test_case : cases) { + InMemoryStore store(test_case.point); + EXPECT_EQ(test_case.expected, runVerifiedJson(&store)); + EXPECT_EQ(test_case.writes, store.write_calls); + EXPECT_EQ(test_case.finishes, store.finish_calls); + EXPECT_EQ(test_case.verifies, store.verify_calls); + EXPECT_EQ(test_case.commits, store.commit_calls); + EXPECT_EQ(test_case.aborts, store.abort_calls); + EXPECT_EQ(test_case.discards, store.discard_calls); + EXPECT_EQ(test_case.keeps_temp, store.tempExists()); + if (test_case.point != FailurePoint::None) { + EXPECT_EQ(old_source, store.source()); + } + } +} + TEST(MQTTPrefsAtomicStore, AnyFailureAbortsAndPreservesExistingSource) { const std::vector source = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; const struct { @@ -546,6 +622,23 @@ TEST(MQTTPrefsAtomicStore, PowerCutDuringTempWriteKeepsPrimaryAndAllowsNextSave) EXPECT_TRUE(store.canStartSave()); } +TEST(MQTTPrefsAtomicStore, TornFirstMigrationTempIsDiscardedSoLegacyCanRetry) { + SpiffsMqttTransaction store; + store.removePrimary(); + store.cutDuringTempWrite(); + + EXPECT_EQ(Recovery::Action::DiscardTemp, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::Preserve, + Recovery::FileState::Missing)); + store.recover(Recovery::FileState::Missing, + Recovery::FileState::Preserve, + Recovery::FileState::Missing); + EXPECT_FALSE(store.has("/mqtt_prefs")); + EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.canStartSave()); +} + TEST(MQTTPrefsAtomicStore, RecoveredUsablePrimaryClearsOpaqueTransactionArtifacts) { { SpiffsMqttTransaction store; @@ -605,8 +698,14 @@ TEST(MQTTPrefsAtomicStore, RecoveryNeverOverwritesOpaqueNewerLayout) { EXPECT_EQ(Recovery::Action::KeepPrimary, Recovery::select(Recovery::FileState::Preserve, Recovery::FileState::Usable, Recovery::FileState::Usable)); - // If there is no primary, a usable backup wins over an opaque temp. Once - // promoted, production treats the backup as authoritative and clears temp. + // A syntactically valid future temp may already have passed verification and + // reached the rename phase. It wins over the stale supported backup and is + // promoted into the authoritative name, where older firmware will hold it. + EXPECT_EQ(Recovery::Action::PromoteTemp, + Recovery::select(Recovery::FileState::Missing, Recovery::FileState::FutureUsable, + Recovery::FileState::Usable)); + // A corrupt or incomplete temp is different: the supported backup remains + // the last known committed image. EXPECT_EQ(Recovery::Action::PromoteBackup, Recovery::select(Recovery::FileState::Missing, Recovery::FileState::Preserve, Recovery::FileState::Usable)); @@ -617,6 +716,34 @@ TEST(MQTTPrefsAtomicStore, RecoveryNeverOverwritesOpaqueNewerLayout) { Recovery::FileState::Preserve)); } +TEST(MQTTPrefsAtomicStore, AmbiguousFutureOrOomTempIsNeverDeleted) { + for (const Recovery::FileState uncertain : { + Recovery::FileState::FutureClaimed, + Recovery::FileState::Indeterminate}) { + SpiffsMqttTransaction store; + store.cutAt(SpiffsMqttTransaction::Boundary::AfterBackupRename); + store.recover(Recovery::FileState::Missing, uncertain, + Recovery::FileState::Usable); + + // The supported backup can run this boot, but the candidate that this + // firmware could not classify remains available to newer firmware or a + // later boot with more heap. Its presence also blocks a new transaction. + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_FALSE(store.canStartSave()); + } +} + +TEST(MQTTPrefsAtomicStore, UsablePrimaryDoesNotCleanIndeterminateArtifact) { + SpiffsMqttTransaction store; + store.cutDuringTempWrite(); + store.recover(Recovery::FileState::Usable, + Recovery::FileState::Indeterminate); + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_FALSE(store.canStartSave()); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp index 7bf49746..73ec5ec8 100644 --- a/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp +++ b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp @@ -28,6 +28,7 @@ MQTTPrefs defaults() { prefs.alert_wifi_minutes = 30; prefs.alert_mqtt_minutes = 240; prefs.alert_min_interval_min = 60; + prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; return prefs; } @@ -59,6 +60,21 @@ Codec::DecodePlan classify(const std::vector& bytes) { return Codec::classify(bytes.data(), prefix_size, bytes.size()); } +void writeV1Payload(std::vector* bytes, const MQTTPrefs& prefs, size_t payload_len) { + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(prefs, &frozen); + ASSERT_GE(bytes->size(), sizeof(MQTTPrefsHeader) + payload_len); + memcpy(bytes->data() + sizeof(MQTTPrefsHeader), &frozen, payload_len); +} + +MQTTPrefs decodeV1(const std::vector& bytes, const Codec::DecodePlan& plan) { + LegacyV1MQTTPrefs frozen = {}; + memcpy(&frozen, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = defaults(); + Codec::migrateV1(frozen, plan.payload_len, &loaded); + return loaded; +} + void fillHighEntropy(std::vector* bytes) { uint32_t state = 0x89abcdef; for (size_t i = 0; i < bytes->size(); ++i) { @@ -244,8 +260,7 @@ TEST(MQTTPrefsCodec, CurrentVersionedPayloadRoundTripsExactly) { ASSERT_EQ(Codec::Source::Current, plan.source); ASSERT_FALSE(plan.preserve_file); ASSERT_TRUE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -277,8 +292,7 @@ TEST(MQTTPrefsCodec, DefaultFiltersKeepTheDowngradeReadablePayloadLength) { ASSERT_EQ(Codec::kV1PreFilterPayloadSize, plan.payload_len); ASSERT_FALSE(plan.preserve_file); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -299,8 +313,7 @@ TEST(MQTTPrefsCodec, AnyNonDefaultFilterOptsIntoTheLongerPayload) { const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::kV1BaselinePayloadSize, plan.payload_len); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(mask, loaded.mqtt_slot_packet_filter[slot]) << slot; EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -338,7 +351,7 @@ TEST(MQTTPrefsCodec, PreFilterV1PayloadDefaultsEverySlotToAllTypes) { std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreFilterPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreFilterPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreFilterPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreFilterPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -346,8 +359,7 @@ TEST(MQTTPrefsCodec, PreFilterV1PayloadDefaultsEverySlotToAllTypes) { ASSERT_TRUE(plan.observer_fields_present); ASSERT_FALSE(plan.preserve_file); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("pre-filter-node", loaded.mqtt_origin); EXPECT_EQ(1u, loaded.mqtt_neighbors_enabled); EXPECT_EQ(MQTT_NEIGHBORS_MAX_INTERVAL_MS, loaded.mqtt_neighbors_interval); @@ -368,7 +380,7 @@ TEST(MQTTPrefsCodec, CompatibleShortV1PayloadPreservesDefaultsBeyondObserverBoun std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreObserverPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreObserverPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreObserverPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreObserverPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -376,8 +388,7 @@ TEST(MQTTPrefsCodec, CompatibleShortV1PayloadPreservesDefaultsBeyondObserverBoun ASSERT_FALSE(plan.preserve_file); ASSERT_FALSE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("short-v1-node", loaded.mqtt_origin); EXPECT_STREQ("ntp.short.example", loaded.mqtt_ntp_server); EXPECT_EQ(0, loaded.snmp_enabled); @@ -400,7 +411,7 @@ TEST(MQTTPrefsCodec, PreNeighborsV1PayloadLoadsObserverFieldsAndDefaultsNeighbor std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreNeighborsPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreNeighborsPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreNeighborsPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreNeighborsPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -408,10 +419,7 @@ TEST(MQTTPrefsCodec, PreNeighborsV1PayloadLoadsObserverFieldsAndDefaultsNeighbor ASSERT_FALSE(plan.preserve_file); ASSERT_TRUE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - loaded.mqtt_neighbors_enabled = 1; // pretend stale - loaded.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; // caller's defaulted tail - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("pre-neighbors-node", loaded.mqtt_origin); EXPECT_STREQ("PNW", loaded.alert_region); @@ -475,6 +483,34 @@ TEST(MQTTPrefsCodec, CorruptOrShortVersionedInputsArePreserved) { EXPECT_TRUE(plan.preserve_file); } +TEST(MQTTPrefsCodec, HeaderValidV1StillRequiresBoundedPayloadStrings) { + MQTTPrefs source = defaults(); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + ASSERT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); + + memset(frozen.mqtt_slot_topic[5], 'x', sizeof(frozen.mqtt_slot_topic[5])); + EXPECT_FALSE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); +} + +TEST(MQTTPrefsCodec, HeaderValidV1AllowsNumericValuesForSerializerRepair) { + MQTTPrefs source = defaults(); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + + // A valid v1 header fixes the layout. Numeric bytes cannot trigger an + // out-of-bounds read and are normalized after migration, so plausibility + // must not reject deployed files merely because these values need repair. + frozen.timezone_offset = 99; + frozen.mqtt_status_enabled = 0xff; + frozen.mqtt_rx_enabled = 0xff; + frozen.snmp_enabled = 0xff; + frozen.alert_enabled = 0xff; + frozen.mqtt_neighbors_enabled = 0xa5; + EXPECT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); + EXPECT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1PreNeighborsPayloadSize)); +} + TEST(MQTTPrefsCodec, LegacyPlausibilityRejectsHighEntropyBytesAtEveryWhitelistedSize) { // A headerless raw struct has no checksum, so this only reduces false // migrations; it cannot prove that a plausible-looking file is authentic. @@ -531,7 +567,9 @@ TEST(MQTTPrefsCodec, LongerSameVersionPayloadLoadsTheBaselineAndIgnoresTheTail) const size_t payload_len = Codec::kV1BaselinePayloadSize + kTail; std::vector bytes(sizeof(MQTTPrefsHeader) + payload_len, 0xA5); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(payload_len)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, sizeof(source)); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &frozen, sizeof(frozen)); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -542,9 +580,11 @@ TEST(MQTTPrefsCodec, LongerSameVersionPayloadLoadsTheBaselineAndIgnoresTheTail) // Reading plan.payload_len bytes recovers this build's whole struct exactly, // and cannot run past it into the unknown tail. + LegacyV1MQTTPrefs loaded_frozen = {}; + memcpy(&loaded_frozen, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + ASSERT_TRUE(Codec::isPlausibleV1(loaded_frozen, plan.payload_len)); MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); - EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); + Codec::migrateV1(loaded_frozen, plan.payload_len, &loaded); EXPECT_STREQ("future-node", loaded.mqtt_origin); EXPECT_STREQ("field-ssid", loaded.wifi_ssid); EXPECT_STREQ("field-secret", loaded.wifi_password); diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp new file mode 100644 index 00000000..ec2544e7 --- /dev/null +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -0,0 +1,358 @@ +#include + +#include + +#define WITH_MQTT_BRIDGE 1 +#define PROGMEM +#include "helpers/MQTTPrefsSerializer.h" + +class InputStream : public Stream { +public: + explicit InputStream(const std::string& text) : _text(text) {} + int available() override { return static_cast(_text.size() - _pos); } + int read() override { return _pos < _text.size() ? _text[_pos++] : -1; } + int peek() override { return _pos < _text.size() ? _text[_pos] : -1; } +private: + std::string _text; + size_t _pos = 0; +}; + +class OutputStream : public Stream { +public: + size_t write(uint8_t byte) override { _text.push_back(static_cast(byte)); return 1; } + size_t print(int value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned int value, int = DEC) override { return appendNumber(value); } + size_t print(long value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned long value, int = DEC) override { return appendNumber(value); } + size_t print(long long value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned long long value, int = DEC) override { return appendNumber(value); } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + const std::string& text() const { return _text; } +private: + template size_t appendNumber(T value) { + const std::string number = std::to_string(value); + _text += number; + return number.size(); + } + std::string _text; +}; + +class StickyFailingStream : public Stream { +public: + explicit StickyFailingStream(size_t limit) : _limit(limit) {} + size_t write(uint8_t) override { + if (_failed || _written >= _limit) { + _failed = true; + return 0; + } + ++_written; + return 1; + } + size_t print(int value, int = DEC) override { + const std::string number = std::to_string(value); + return Print::print(number.c_str()); + } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + bool failed() const { return _failed; } +private: + size_t _limit; + size_t _written = 0; + bool _failed = false; +}; + +static MQTTPrefs defaults() { + MQTTPrefs prefs = {}; + prefs.mqtt_status_enabled = 1; + prefs.mqtt_packets_enabled = 1; + prefs.mqtt_tx_enabled = 2; + prefs.mqtt_rx_enabled = 1; + prefs.mqtt_status_interval = 300000; + prefs.wifi_power_save = 1; + prefs.timezone_offset = -7; + prefs.radio_watchdog_minutes = 5; + prefs.alert_wifi_minutes = 30; + prefs.alert_mqtt_minutes = 240; + prefs.alert_min_interval_min = 60; + prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; + strcpy(prefs.snmp_community, "public"); + for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { + strcpy(prefs.mqtt_slot_preset[i], "none"); + prefs.mqtt_slot_packet_filter[i] = 0xffff; + } + return prefs; +} + +TEST(MQTTPrefsSerializer, RoundTripsEveryGroupAndNumericSlotKeys) { + MQTTPrefs source = defaults(); + strcpy(source.wifi_ssid, "mesh-net"); + strcpy(source.wifi_password, "p\\\"ass\nword"); + strcpy(source.timezone_string, "MST7MDT,M3.2.0"); + strcpy(source.mqtt_ntp_server, "time.example"); + strcpy(source.mqtt_origin, "observer-one"); + strcpy(source.mqtt_iata, "SEA"); + source.mqtt_neighbors_enabled = 1; + source.mqtt_neighbors_interval = MQTT_NEIGHBORS_MAX_INTERVAL_MS; + strcpy(source.mqtt_owner_public_key, + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); + strcpy(source.mqtt_email, "owner@example.com"); + strcpy(source.mqtt_slot_preset[5], "custom"); + strcpy(source.mqtt_slot_host[5], "broker.example"); + source.mqtt_slot_port[5] = 65535; + strcpy(source.mqtt_slot_username[5], "user-six"); + strcpy(source.mqtt_slot_password[5], "secret-six"); + strcpy(source.mqtt_slot_token[5], "token-six"); + strcpy(source.mqtt_slot_topic[5], "mesh/{iata}/{type}"); + strcpy(source.mqtt_slot_audience[5], "audience-six"); + source.mqtt_slot_packet_filter[5] = 0x8001; + source.snmp_enabled = 1; + source.radio_watchdog_minutes = 120; + source.alert_enabled = 1; + strcpy(source.alert_psk_hex, "0123456789abcdef0123456789abcdef"); + strcpy(source.alert_hashtag, "#ops"); + strcpy(source.alert_region, "PNW"); + + OutputStream output; + MQTTPrefsSerializer writer(&source); + ASSERT_TRUE(writer.saveSerial(output)); + EXPECT_NE(std::string::npos, output.text().find("slot6:{")) << output.text(); + EXPECT_NE(std::string::npos, output.text().find("packet_filter:32769")) << output.text(); + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)) << output.text(); + bool repaired = true; + ASSERT_TRUE(reader.apply(&repaired)); + EXPECT_FALSE(repaired); + EXPECT_STREQ(source.wifi_password, loaded.wifi_password); + EXPECT_STREQ(source.mqtt_owner_public_key, loaded.mqtt_owner_public_key); + EXPECT_STREQ(source.mqtt_slot_host[5], loaded.mqtt_slot_host[5]); + EXPECT_EQ(65535, loaded.mqtt_slot_port[5]); + EXPECT_EQ(0x8001, loaded.mqtt_slot_packet_filter[5]); + EXPECT_EQ(MQTT_NEIGHBORS_MAX_INTERVAL_MS, loaded.mqtt_neighbors_interval); + EXPECT_STREQ("PNW", loaded.alert_region); +} + +TEST(MQTTPrefsSerializer, MissingOptionalKeysKeepDefaults) { + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,mqtt:{origin:\"changed\"}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = true; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_FALSE(repaired); + EXPECT_STREQ("changed", prefs.mqtt_origin); + EXPECT_EQ(1, prefs.mqtt_packets_enabled); + EXPECT_EQ(300000u, prefs.mqtt_status_interval); + EXPECT_EQ(0xffff, prefs.mqtt_slot_packet_filter[5]); +} + +TEST(MQTTPrefsSerializer, RequiresSupportedVersion) { + MQTTPrefs missing = defaults(); + InputStream no_version("{wifi:{ssid:\"x\"}}"); + MQTTPrefsSerializer missing_serializer(&missing); + ASSERT_TRUE(missing_serializer.loadSerial(no_version)); + bool repaired = false; + EXPECT_FALSE(missing_serializer.apply(&repaired)); + + MQTTPrefs future = defaults(); + InputStream future_input("{version:2,wifi:{ssid:\"x\"}}"); + MQTTPrefsSerializer future_serializer(&future); + ASSERT_TRUE(future_serializer.loadSerial(future_input)); + EXPECT_TRUE(future_serializer.hasFutureVersion()); + EXPECT_FALSE(future_serializer.apply(&repaired)); +} + +TEST(MQTTPrefsSerializer, FutureVersionProbeIgnoresV1FieldTypeChanges) { + InputStream probe_input("{version:2,wifi:{power_save:\"max\"}}"); + MQTTPrefsVersionProbe probe; + ASSERT_TRUE(probe.loadSerial(probe_input)); + EXPECT_TRUE(probe.hasFutureVersion()); + + // The same image is not valid under v1, demonstrating why recovery must + // probe the version before invoking the current schema. + MQTTPrefs prefs = defaults(); + InputStream v1_input("{version:2,wifi:{power_save:\"max\"}}"); + MQTTPrefsSerializer v1(&prefs); + EXPECT_FALSE(v1.loadSerial(v1_input)); + + for (const char* future_text : { + "{version:2,this_key_is_too_long:1}", + "{version:2,x:[1]}", + "{version:2,wifi:{ssid:\"torn\"}"}) { + InputStream future_input(future_text); + MQTTPrefsVersionProbe future_probe; + EXPECT_FALSE(future_probe.loadSerial(future_input)) << future_text; + EXPECT_TRUE(future_probe.hasFutureVersion()) << future_text; + } +} + +TEST(MQTTPrefsSerializer, RejectsDuplicateKnownKey) { + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,mqtt:{origin:\"one\",origin:\"two\"}}"); + MQTTPrefsSerializer serializer(&prefs); + EXPECT_FALSE(serializer.loadSerial(input)); +} + +TEST(MQTTPrefsSerializer, RejectsOverlongStringAndIntegerOverflow) { + MQTTPrefs prefs = defaults(); + InputStream long_string( + "{version:1,wifi:{ssid:\"12345678901234567890123456789012\"}}"); + MQTTPrefsSerializer string_serializer(&prefs); + EXPECT_FALSE(string_serializer.loadSerial(long_string)); + + prefs = defaults(); + InputStream overflow("{version:1,mqtt:{slot1:{port:999999999999}}}"); + MQTTPrefsSerializer number_serializer(&prefs); + EXPECT_FALSE(number_serializer.loadSerial(overflow)); + + prefs = defaults(); + InputStream quoted_number("{version:\"1\"}"); + MQTTPrefsSerializer quoted_number_serializer(&prefs); + EXPECT_FALSE(quoted_number_serializer.loadSerial(quoted_number)); + + prefs = defaults(); + InputStream bare_string("{version:1,wifi:{ssid:meshnet}}"); + MQTTPrefsSerializer bare_string_serializer(&prefs); + EXPECT_FALSE(bare_string_serializer.loadSerial(bare_string)); +} + +TEST(MQTTPrefsSerializer, RejectsScalarObjectShapeMismatches) { + MQTTPrefs prefs = defaults(); + InputStream object_version("{version:{x:1}}"); + MQTTPrefsSerializer object_version_serializer(&prefs); + EXPECT_FALSE(object_version_serializer.loadSerial(object_version)); + + prefs = defaults(); + InputStream object_port("{version:1,mqtt:{slot1:{port:{x:1883}}}}"); + MQTTPrefsSerializer object_port_serializer(&prefs); + EXPECT_FALSE(object_port_serializer.loadSerial(object_port)); + + prefs = defaults(); + InputStream scalar_mqtt("{version:1,mqtt:1}"); + MQTTPrefsSerializer scalar_mqtt_serializer(&prefs); + EXPECT_FALSE(scalar_mqtt_serializer.loadSerial(scalar_mqtt)); + + prefs = defaults(); + InputStream scalar_slot("{version:1,mqtt:{slot1:1}}"); + MQTTPrefsSerializer scalar_slot_serializer(&prefs); + EXPECT_FALSE(scalar_slot_serializer.loadSerial(scalar_slot)); +} + +TEST(MQTTPrefsSerializer, RepairsSemanticRanges) { + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,wifi:{power_save:9},time:{utc_offset:99}," + "mqtt:{tx_enabled:7,status:{enabled:3,interval_ms:10}," + "neighbors:{enabled:2,interval_ms:100},slot1:{port:-1,packet_filter:-2}}," + "radio:{watchdog_min:121},alert:{rate_limit_min:1}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(1, prefs.wifi_power_save); + EXPECT_EQ(-7, prefs.timezone_offset); + EXPECT_EQ(2, prefs.mqtt_tx_enabled); + EXPECT_EQ(300000u, prefs.mqtt_status_interval); + EXPECT_EQ(MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS, prefs.mqtt_neighbors_interval); + EXPECT_EQ(0, prefs.mqtt_slot_port[0]); + EXPECT_EQ(0xffff, prefs.mqtt_slot_packet_filter[0]); + EXPECT_EQ(5, prefs.radio_watchdog_minutes); + EXPECT_EQ(60, prefs.alert_min_interval_min); +} + +TEST(MQTTPrefsSerializer, RepairsTextValuesToSafeDefaults) { + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,time:{ntp_server:\"bad/host\"},mqtt:{iata:\"sea\"," + "owner:{public_key:\"not-a-key\"},slot1:{preset:\"not-a-preset\"}," + "slot2:{preset:\"analyzer-us\"},slot3:{preset:\"analyzer-us\"}}," + "alert:{psk_hex:\"not-hex\",hashtag:\"#stale\"}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_STREQ("SEA", prefs.mqtt_iata); + EXPECT_STREQ("", prefs.mqtt_ntp_server); + EXPECT_STREQ("", prefs.mqtt_owner_public_key); + EXPECT_STREQ("none", prefs.mqtt_slot_preset[0]); + EXPECT_STREQ("analyzer-us", prefs.mqtt_slot_preset[1]); + // Historical firmware allowed duplicate aliases. Preserve them on load; + // current setters prevent creating new duplicates without silently changing + // a deployed configuration during migration. + EXPECT_STREQ("analyzer-us", prefs.mqtt_slot_preset[2]); + EXPECT_STREQ("", prefs.alert_psk_hex); + EXPECT_STREQ("", prefs.alert_hashtag); +} + +TEST(MQTTPrefsSerializer, LateParseOrVersionFailureCannotMutateLivePrefs) { + MQTTPrefs live = defaults(); + strcpy(live.wifi_ssid, "live-network"); + + MQTTPrefs scratch = defaults(); + InputStream truncated("{wifi:{ssid:\"uncommitted\"},version:1"); + MQTTPrefsSerializer truncated_serializer(&scratch); + EXPECT_FALSE(truncated_serializer.loadSerial(truncated)); + EXPECT_STREQ("live-network", live.wifi_ssid); + + scratch = defaults(); + InputStream future("{wifi:{ssid:\"future-network\"},version:2}"); + MQTTPrefsSerializer future_serializer(&scratch); + ASSERT_TRUE(future_serializer.loadSerial(future)); + EXPECT_TRUE(future_serializer.hasFutureVersion()); + bool repaired = false; + EXPECT_FALSE(future_serializer.apply(&repaired)); + EXPECT_STREQ("live-network", live.wifi_ssid); +} + +TEST(MQTTPrefsSerializer, StickyShortWriteFailsTheCompleteSave) { + MQTTPrefs prefs = defaults(); + MQTTPrefsSerializer serializer(&prefs); + StickyFailingStream output(20); + EXPECT_FALSE(serializer.saveSerial(output)); + EXPECT_TRUE(output.failed()); +} + +TEST(MQTTPrefsSerializer, SaveNormalizationIsIdempotentAgainstKnownDefaults) { + MQTTPrefs prefs = defaults(); + prefs.timezone_offset = 99; + strcpy(prefs.mqtt_ntp_server, "bad/host"); + strcpy(prefs.mqtt_iata, "not-iata"); + strcpy(prefs.mqtt_slot_preset[0], "not-a-preset"); + + MQTTPrefs repair_defaults = defaults(); + repair_defaults.timezone_offset = -7; + strcpy(repair_defaults.mqtt_iata, "sea"); + strcpy(repair_defaults.mqtt_slot_preset[0], "analyzer-us"); + + MQTTPrefsSerializer writer(&prefs, &repair_defaults); + bool repaired = false; + ASSERT_TRUE(writer.normalize(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(-7, prefs.timezone_offset); + EXPECT_STREQ("", prefs.mqtt_ntp_server); + EXPECT_STREQ("SEA", prefs.mqtt_iata); + EXPECT_STREQ("none", prefs.mqtt_slot_preset[0]); + + OutputStream output; + ASSERT_TRUE(writer.saveSerial(output)); + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)); + repaired = true; + ASSERT_TRUE(reader.apply(&repaired)); + EXPECT_FALSE(repaired) << output.text(); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_observer_validation/test_observer_validation.cpp b/test/test_observer_validation/test_observer_validation.cpp index 3045caed..8e9913ca 100644 --- a/test/test_observer_validation/test_observer_validation.cpp +++ b/test/test_observer_validation/test_observer_validation.cpp @@ -129,6 +129,18 @@ TEST(ValueFits, RejectsNullOrZeroBuffer) { EXPECT_FALSE(mqttValueFits("abc", 0)); } +TEST(AssignedPresetSlot, FindsDuplicatesOutsideTheTargetSlot) { + const char presets[4][24] = { + "analyzer-us", "none", "analyzer-eu", "analyzer-us" + }; + + EXPECT_EQ(3, mqttAssignedPresetSlot(presets, "analyzer-us", 0)); + EXPECT_EQ(0, mqttAssignedPresetSlot(presets, "analyzer-us", 3)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, "analyzer-eu", 2)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, "missing", 0)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, nullptr, 0)); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_webconfig_batch/test_webconfig_batch.cpp b/test/test_webconfig_batch/test_webconfig_batch.cpp index 813412c5..670c6ead 100644 --- a/test/test_webconfig_batch/test_webconfig_batch.cpp +++ b/test/test_webconfig_batch/test_webconfig_batch.cpp @@ -230,6 +230,7 @@ TEST(WebConfigBatch, CliFailureRepliesAreRecognisedInEveryShapeCommonCLIEmits) { "Err - bad params", // MyMesh setperm "ERR: bad pubkey", // neighbor.remove "Error: IATA code must be exactly 3 letters",// observer setters + "Error: setting not persisted; change rolled back", // observer storage failure "(ERR: clock cannot go backwards)", // clock sync, parenthesised "Unknown command", // top-level fallthrough "unknown config: mqtt.nope", // set fallthrough From d489159c99bf30d773789d21a1222daffd9b74d6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 08:19:34 -0700 Subject: [PATCH 02/93] docs: refresh firmware-notes.html for v1.17.1 The rolling-release body is what the flasher dropdown serves as changelog. Drop the stale v1.16.0 experimental blurb and describe the current observer surface: MeshCore 1.17.1, web config, in-channel OTA, and neighbors. --- firmware-notes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firmware-notes.html b/firmware-notes.html index eeda7b07..a6d5a786 100644 --- a/firmware-notes.html +++ b/firmware-notes.html @@ -1 +1 @@ -MQTT Observer v1.16.0 (experimental)
  • Up to 6 MQTT broker slots with built-in presets
  • Presets: Analyzer, MeshMapper, MeshRank, Waev, Meshomatic, CascadiaMesh, TennMesh, NashMesh, and more
  • JWT (Ed25519) and username/password authentication
  • Automatic reconnection with exponential backoff
  • After flashing, configure via serial console (115200 baud)

See setup guide for configuration instructions.

+MQTT Observer v1.17.1
  • Based on MeshCore 1.17.1
  • Up to 6 MQTT broker slots with built-in community presets
  • Presets include Analyzer, MeshMapper, MeshRank, Waev, Meshomatic, CascadiaMesh, TennMesh, NashMesh, IdahoMesh, and more
  • JWT (Ed25519) and username/password authentication
  • Web config portal on the device, plus serial console (115200 baud)
  • Over-the-air updates within the channel you flashed
  • Neighbor discovery uplink on supported boards

See setup guide for configuration instructions.

From 40635a53c40048be9095bb5243545bf9e82ff351 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 09:05:05 -0700 Subject: [PATCH 03/93] test(native): declare stdlib in the Arduino mock so ConfigSerializer builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The real Arduino.h includes stdlib.h, so ConfigSerializer.cpp reaches atoi, atol and atof through it and compiles on device. The mock supplied only cstdint, cmath and Stream.h, leaving those undeclared — and since the native env compiles ConfigSerializer.cpp into every suite via build_src_filter, all 21 suites errored rather than just its own. Fixing the mock keeps src/ identical to upstream and covers any other source relying on the same transitive include. pio test -e native: 297 test cases, 297 succeeded. --- test/mocks/Arduino.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/mocks/Arduino.h b/test/mocks/Arduino.h index 77499fe4..a3d5b276 100644 --- a/test/mocks/Arduino.h +++ b/test/mocks/Arduino.h @@ -2,8 +2,15 @@ #include #include +// The real Arduino.h pulls in stdlib.h, so device code reaches atoi/atol/atof/strtoul +// without including it. Mirror that here or those sources fail only on the native build. +#include #include "Stream.h" +using std::atof; +using std::atoi; +using std::atol; + inline uint32_t g_mock_millis = 0; using std::isnan; From 75f3e446e0f2dcbd3865c1356017dbb09b2ff342 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 9 Aug 2026 12:04:55 -0700 Subject: [PATCH 04/93] feat(mqtt): persist observer preferences as JSON --- MQTT_IMPLEMENTATION.md | 2 +- MQTT_INTERNALS.md | 139 +++--- examples/simple_repeater/MyMesh.cpp | 4 +- examples/simple_repeater/MyMesh.h | 5 + examples/simple_room_server/MyMesh.cpp | 4 +- examples/simple_room_server/MyMesh.h | 5 + src/helpers/CommonCLI.cpp | 420 +++++++++++++++--- src/helpers/CommonCLI.h | 22 +- src/helpers/CommonCLI_Observer.cpp | 190 +++++--- src/helpers/ConfigSerializer.cpp | 136 +++++- src/helpers/ConfigSerializer.h | 81 +++- src/helpers/MQTTDefaults.h | 2 +- src/helpers/MQTTObserverValidation.h | 17 + src/helpers/MQTTPacketFilter.h | 8 +- src/helpers/MQTTPrefsAtomicStore.h | 49 +- src/helpers/MQTTPrefsCodec.h | 166 ++++++- src/helpers/MQTTPrefsRecovery.h | 40 +- src/helpers/MQTTPrefsSerializer.h | 412 +++++++++++++++++ src/helpers/MQTTPrefsStorage.h | 71 ++- src/helpers/bridges/MQTTBridge.h | 2 +- test/README.md | 3 +- .../test_config_serializer.cpp | 17 + .../test_mqtt_prefs_atomic_store.cpp | 139 +++++- .../test_mqtt_prefs_codec.cpp | 80 +++- .../test_mqtt_prefs_serializer.cpp | 358 +++++++++++++++ .../test_observer_validation.cpp | 12 + .../test_webconfig_batch.cpp | 1 + 27 files changed, 2131 insertions(+), 254 deletions(-) create mode 100644 src/helpers/MQTTPrefsSerializer.h create mode 100644 test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 0df96e67..f18fed75 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -285,7 +285,7 @@ flash regardless of target. #### Compile-time fresh-install defaults (`src/helpers/MQTTDefaults.h`) -Optional PlatformIO `build_flags` override defaults written when `/mqtt_prefs` is first created. They do **not** change existing saved prefs on upgrade or reflash (unless `/mqtt_prefs` is erased). +Optional PlatformIO `build_flags` override defaults used when `/mqtt.json` is first created. They do **not** change existing saved prefs on upgrade or reflash (unless `/mqtt.json` is erased). | Macro | Default | Notes | |-------|---------|-------| diff --git a/MQTT_INTERNALS.md b/MQTT_INTERNALS.md index d0de0398..005de530 100644 --- a/MQTT_INTERNALS.md +++ b/MQTT_INTERNALS.md @@ -8,7 +8,8 @@ Developer-facing notes on how the MQTT observer feature is structured in the cod - `src/helpers/bridges/MQTTBridge.h` - MQTT bridge class definition - `src/helpers/bridges/MQTTBridge.cpp` - MQTT bridge implementation - `src/helpers/MQTTPresets.h` - Preset definitions, CA certificates, and lookup functions -- `src/helpers/MQTTDefaults.h` - Compile-time defaults for fresh `/mqtt_prefs` +- `src/helpers/MQTTDefaults.h` - Compile-time defaults for fresh `/mqtt.json` +- `src/helpers/MQTTPrefsSerializer.h` - Versioned, semantic observer JSON schema - `src/helpers/MQTTMessageBuilder.h` - JSON message formatting utilities - `src/helpers/MQTTMessageBuilder.cpp` - JSON message formatting implementation - `src/helpers/JWTHelper.h` - JWT token generation for Ed25519-based authentication @@ -30,8 +31,9 @@ The observer feature is kept out of upstream-tracked files through three mechani `resolveAlertScope`, `beginDeferredOtaUpdate`). The example apps override them behind `#ifdef WITH_MQTT_BRIDGE`. - **Separate settings file** — observer settings (MQTT slots, WiFi, timezone, SNMP, - radio watchdog, fault alerts) live in the `MQTTPrefs` struct persisted to - `/mqtt_prefs`, keeping `NodePrefs` / `/com_prefs` aligned with the upstream layout. + radio watchdog, fault alerts) live in the runtime `MQTTPrefs` object and are + field-serialized to `/mqtt.json`, keeping `NodePrefs` / `/prefs.json` aligned with + upstream. The two files are independent transactions, not one atomic snapshot. Remaining integration points in upstream files: - `examples/simple_repeater/MyMesh.{h,cpp}`, `examples/simple_room_server/MyMesh.{h,cpp}` - @@ -164,75 +166,104 @@ which packet events non-MQTT bridges capture. The MQTT bridge ignores `bridge.so favour of independent `mqtt.rx` / `mqtt.tx` controls. Everything MQTT-specific lives under `mqtt.*` (shared settings), `mqttN.*` (per-slot broker config), `wifi.*`, and `timezone.*`. -### `/mqtt_prefs` file format +### `/mqtt.json` file format -`/mqtt_prefs` is written with an 8-byte `MQTTPrefsHeader` (`magic`, `version`, -`payload_len`) followed by the raw `MQTTPrefs` payload. The magic is -`{0xF5, 'M', 'Q', 'P'}` — its leading non-ASCII byte can never collide with the first -bytes of a legacy (headerless) file, whose payload begins with the `mqtt_origin` -string. Bump `MQTT_PREFS_VERSION` when the payload layout changes incompatibly; a file -whose version this firmware doesn't recognize is left untouched and the in-memory prefs -fall back to defaults (no downgrade, no misread). `saveMQTTPrefs()` also refuses to -write while such a file is present (`_mqtt_prefs_hold`), so a `set` command after a -firmware downgrade can't clobber the newer config — observer settings changed in that -state simply don't persist. The frozen legacy layouts are pinned with `static_assert`s -in `MQTTPrefsStorage.h`, so every target build re-verifies the fleet's file offsets. +Observer preferences use the same `ConfigSerializer` object notation as upstream +`/prefs.json`: semantic unquoted keys, quoted strings, decimal numbers, and nested +objects. Schema version 1 requires the root `version:1` field. The main shape is: -Adding a field to the current version stays backward compatible: append it to the end -of `MQTTPrefs`, give the older exact payload length an explicit decoder boundary, and -leave the missing tail at its default. The packet-filter addition follows that rule: -the prior 2864-byte v1 payload loads with all six filters set to `all`, while the -full payload is 2876 bytes. +```text +{version:1, + wifi:{ssid:"...",password:"...",power_save:1}, + time:{timezone:"...",utc_offset:0,ntp_server:"..."}, + mqtt:{origin:"...",iata:"SEA",packets_enabled:1,raw_enabled:0, + tx_enabled:2,rx_enabled:1, + status:{enabled:1,interval_ms:300000}, + neighbors:{enabled:0,interval_ms:86400000}, + owner:{public_key:"...",email:"..."}, + slot1:{preset:"analyzer-us",host:"",port:0,username:"",password:"", + token:"",topic:"",audience:"",packet_filter:65535}, + ... slot2 through slot6 ...}, + snmp:{enabled:0,community:"public"}, + radio:{watchdog_min:5}, + alert:{enabled:0,psk_hex:"",wifi_minutes:30,mqtt_minutes:240, + rate_limit_min:60,hashtag:"",region:""}} +``` -#### The downgrade contract +Keys may contain digits after their first character, which permits the readable +`slot1` ... `slot6` names. Every schema key fits `ConfigSerializer`'s 15-character +visible-key limit. Known strings and numbers use strict parsing: duplicates, overlong +strings, malformed decimals, and overflow reject the complete file. A supported file +with a semantically out-of-range value is repaired to that field's safe default and +rewritten atomically. Unknown fields are ignored only within the serializer's general +limits (15-character keys, 127-byte decoded values, and six nested object levels below the root). -**Within a version tag the layout is append-only, and a longer payload is always -readable.** A file written by a later build starts with this binary's exact baseline, -so `classify()` reads that prefix and ignores the tail. A downgraded node keeps its -WiFi credentials, broker slots, and every other setting it understands; the only thing -it loses is the settings the newer build added. +Loading always starts with defaults and parses into a separate heap scratch object. +The live preferences change only after the complete file parses, has an explicit +supported version, and passes validation. A missing/future version, syntax error, +overlength value, or allocation/read failure leaves `/mqtt.json` untouched, runs this +boot on defaults, and holds observer saves so a later CLI/WebConfig write cannot erase +the opaque source. If an observer setter cannot commit its JSON transaction, it restores +the pre-command in-memory preferences and does not restart or apply a bridge change as +though the setting were durable. -That asymmetry is the whole point. Refusing the file costs the operator the network -itself — `/mqtt_prefs` holds `wifi_ssid`/`wifi_password` as well as the broker config, -so a node that falls back to defaults has no WiFi, no portal, and no OTA, recoverable -only over serial. Reading it costs a feature's settings. Losing later settings is the -acceptable half of that trade; losing the node is not. +Saves stream to `/mqtt.json.tmp` through a sticky short-write detector while computing +size and checksum. The firmware closes and rereads the temp, verifies size/checksum, +parses it into another scratch object, then publishes with +`/mqtt.json` -> `/mqtt.json.bak` and temp -> primary. Boot recovery selects the usable +primary/temp/backup without overwriting an opaque future or corrupt primary. A valid +future-version temp that reached the rename phase wins over the stale backup and is held +for newer firmware. If a temp claims a future version but uses grammar this firmware +cannot parse, or cannot be classified because scratch allocation fails, recovery may run +the last usable backup but retains the uncertain temp and holds all observer writes. A +definitively corrupt/incomplete current-version temp may be discarded for that backup. +If power fails during the very first migration, there is no JSON primary or backup yet; +recovery discards a definitively invalid temp so the intact `/mqtt_prefs` source can be +loaded and the migration retried. An uncertain/future temp is still preserved and held. -The tail survives until something actually writes. `saveMQTTPrefs()` rewrites at this -binary's own length, so a rollback that changes no observer setting and is later rolled -forward keeps the newer fields intact — only an explicit `set` while downgraded drops -them. The boot log says so when it happens. +Unknown slot preset names are repaired to `none`, never to a build-specific default +broker. Duplicate known presets from historical firmware are preserved on load; current +CLI and WebConfig setters prevent creating new duplicates without silently changing an +existing deployment during migration. -**A change that is not a pure append MUST bump `MQTT_PREFS_VERSION`.** The version -check is what makes the rule above safe: a different tag is still refused outright and -the file preserved, because the bytes may no longer mean what this binary thinks. Never -reorder, resize, or repurpose an existing field within a version. +Schema changes that reinterpret existing names or values must increment `version`. +Additive fields may remain in version 1 only when old readers can safely ignore them +within the limits above. A future version is treated as opaque rather than partially +loading its known-looking fields. Literal schema keys are compile-time checked against +the 15-character visible-key limit so a new version-1 field cannot accidentally violate +that downgrade contract. -Note the rule is only as old as the build that implements it. Firmware already deployed -carries the *previous* decoder, which rejects any longer v1 payload — so rolling back -from this build to one shipped before it still falls back to defaults. -`MQTTPrefsCodec::payloadLenFor()` covers that gap from the writing side: it returns the -shortest length that still round-trips the configuration, so a node keeps writing 2864 -bytes until a packet filter actually holds something, and clearing the last non-default -filter puts it back. That mitigation can be retired once no supported downgrade target -predates the contract; the contract itself is the durable half. +#### Downgrade and rollback -Shorter payloads keep their existing, stricter treatment: a short length must match a -boundary that really shipped (`MQTT_PREFS_V1_*_PAYLOAD_SIZE`), because raw prefs have -no checksum and an arbitrary short size cannot be trusted to mean anything. +The old `/mqtt_prefs` binary is read only for one-time migration and is deliberately +not updated or deleted. When it exists and is usable, it is the exact pre-migration +rollback snapshot. Firmware older than this JSON change will therefore see stale +observer settings after a downgrade. A fresh JSON-only install has no binary observer +configuration to recover on downgrade. + +The two files are intentionally not reconciled. If an operator changes observer settings +while running old firmware, those changes update only `/mqtt_prefs`. Rolling forward to +this firmware makes the existing `/mqtt.json` authoritative again, so rollback-era edits +are ignored unless the operator exports and reapplies them. Conversely, JSON-only settings +cannot be recovered by old firmware. This is a rollback snapshot, not bidirectional sync. + +`LegacyV1MQTTPrefs` and the older pre-slot/3-slot/6-slot structs are frozen migration +ABIs with size/offset assertions. The runtime `MQTTPrefs` layout is not an on-flash ABI. ### Settings upgrade / migration `loadPrefs()` handles every historical on-device format one-time at boot: -- **`/mqtt_prefs`** — if the file has the version header it is read directly. Otherwise +- **`/mqtt_prefs` -> `/mqtt.json`** — if the legacy file has the version header its + frozen v1 layout is field-copied. Otherwise it is a legacy headerless file and its layout is detected by size: pre-slot (`OldMQTTPrefs`), 3-slot (`ThreeSlotMQTTPrefs`), or the 6-slot layout shipped on `observer-firmware` back when it was named `mqtt-bridge-implementation-flex` (`Legacy6SlotMQTTPrefs`). Each is field-copied into - the current compact `MQTTPrefs` and re-saved with the version header — which also + the runtime `MQTTPrefs` and saved as schema-versioned JSON — which also drops the vestigial `_legacy_*` fields the flex layout carried mid-struct. This is a one-time rewrite; every deployed device performs it on its first boot of versioned - firmware, after which all reads take the header path. + firmware, after which `/mqtt.json` is authoritative. The binary source remains as + a rollback snapshot and is never dual-written. The pre-slot (`OldMQTTPrefs`) copy maps the old single-broker keys onto slots: `mqtt.analyzer.us = on` → slot 1 `analyzer-us`, `mqtt.analyzer.eu = on` → slot 2 `analyzer-eu`, and a configured `mqtt.server` / `mqtt.port` / `mqtt.username` / @@ -241,7 +272,7 @@ no checksum and an arbitrary short size cannot be trusted to mean anything. - **`/com_prefs`** — a file written by fork firmware that predates the `MQTTPrefs` split (a zero-filled MQTT gap plus a trailing observer block) is detected by size; the trailing SNMP / radio-watchdog / fault-alert settings and the `rx_boosted_gain` / - `flood_max_*` fields are recovered, carried into `/mqtt_prefs`, and both files are + `flood_max_*` fields are recovered, carried into `/mqtt.json`, and the active files are rewritten in the current formats. - Settings the pre-split firmware stored *inside* the `/com_prefs` MQTT gap (the MQTT slot/WiFi config itself) are **not** recovered — users upgrading from firmware that diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 7e209cfb..22645503 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1050,7 +1050,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc // _prefs.agc_reset_interval = 7; // 28 seconds (secs/4) #endif // Observer defaults (radio_watchdog, alert.*, snmp.*) moved to applyMQTTDefaults() - // in MQTTDefaults.h — they live in /mqtt_prefs now, not NodePrefs. + // in MQTTDefaults.h — they live in /mqtt.json now, not NodePrefs. // bridge defaults _prefs.bridge_enabled = 1; // enabled @@ -1066,7 +1066,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.gps_interval = 0; _prefs.advert_loc_policy = ADVERT_LOC_PREFS; - // MQTT/WiFi/timezone/radio_watchdog defaults live in /mqtt_prefs now (see applyMQTTDefaults). + // MQTT/WiFi/timezone/radio_watchdog defaults live in /mqtt.json now (see applyMQTTDefaults). _prefs.adc_multiplier = 0.0f; // 0.0f means use default board multiplier diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index a232ef52..e4c3a3f7 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -285,6 +285,11 @@ public: void savePrefs() override { _cli.savePrefs(_fs); } +#ifdef WITH_MQTT_BRIDGE + bool saveObserverPrefs() override { + return _cli.saveObserverPrefs(_fs); + } +#endif void sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 9fe5a9c8..02efbd43 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -891,7 +891,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.radio_fem_txgain = 0; // Observer defaults (alert.*, etc.) moved to applyMQTTDefaults() — they live - // in /mqtt_prefs now, not NodePrefs. + // in /mqtt.json now, not NodePrefs. // bridge defaults (same as repeater) _prefs.bridge_enabled = 1; // enabled @@ -900,7 +900,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _prefs.bridge_baud = 115200; // baud rate _prefs.bridge_channel = 1; // channel 1 - // MQTT/WiFi/timezone defaults live in /mqtt_prefs now (see applyMQTTDefaults). + // MQTT/WiFi/timezone defaults live in /mqtt.json now (see applyMQTTDefaults). next_post_idx = 0; next_client_idx = 0; diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index b0eb4e8f..65de77ed 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -289,6 +289,11 @@ public: void savePrefs() override { _cli.savePrefs(_fs); } +#ifdef WITH_MQTT_BRIDGE + bool saveObserverPrefs() override { + return _cli.saveObserverPrefs(_fs); + } +#endif void sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size); diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 427d456b..cf2e52eb 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -6,6 +6,7 @@ #include "MQTTPrefsAtomicStore.h" #include #include +#include #ifndef BRIDGE_MAX_BAUD #define BRIDGE_MAX_BAUD 115200 @@ -23,6 +24,7 @@ #include "MQTTDefaults.h" #include "MQTTPrefsCodec.h" #include "MQTTPrefsRecovery.h" +#include "MQTTPrefsSerializer.h" #endif // Believe it or not, this std C function is busted on some platforms! @@ -94,7 +96,8 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { _prefs->bridge_pkt_src = 1; // Default to RX (logRx) for new installs } #ifdef WITH_MQTT_BRIDGE - // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt_prefs. + // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt.json, + // migrating the old /mqtt_prefs binary when JSON does not exist yet. // Readers (MQTTBridge, AlertReporter, observer CLI) use _mqtt_prefs directly — // these fields no longer exist in NodePrefs, so there is nothing to sync. MQTTPrefsAtomicStore::LegacyUpgradeGate legacy_upgrade( @@ -119,13 +122,13 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { } } // mqtt_rx_enabled: new field appended to end of MQTTPrefs. On upgrade from older firmware, - // the shorter /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) + // a shorter legacy /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) // set by setMQTTPrefsDefaults(). No explicit migration needed. #endif // Republish legacy binary prefs as /prefs.json. Old-format files also carried a // trailing observer block, which loadPrefsInt() recovered into _legacy_tail; wait - // for loadMQTTPrefs() to commit that to /mqtt_prefs first. The legacy file is left + // for loadMQTTPrefs() to commit that to /mqtt.json first. The legacy file is left // on flash either way, so a deferred or failed save just retries on the next boot. #ifdef WITH_MQTT_BRIDGE if (loaded_from_legacy || _com_prefs_needs_upgrade) { @@ -134,7 +137,7 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { legacy_upgrade.recordComPrefsRewrite(); _com_prefs_needs_upgrade = false; } else { - MESH_DEBUG_PRINTLN("Prefs: deferring /prefs.json migration until /mqtt_prefs commits"); + MESH_DEBUG_PRINTLN("Prefs: deferring /prefs.json migration until /mqtt.json commits"); } } #else @@ -376,13 +379,17 @@ bool CommonCLI::savePrefs(FILESYSTEM* fs, bool save_mqtt) { #ifdef WITH_MQTT_BRIDGE // Observer config (MQTT/WiFi/timezone/SNMP/alert) is persisted separately. The // observer CLI writes _mqtt_prefs directly, so no NodePrefs->MQTTPrefs sync runs. - // Runs regardless of the NodePrefs result so a failed JSON write cannot strand it. + // Ordinary NodePrefs callers leave save_mqtt false; migration is the only + // workflow that may explicitly combine these independent transactions. if (save_mqtt) saveMQTTPrefs(fs); #endif return success; } #ifdef WITH_MQTT_BRIDGE +static const uint32_t MQTT_JSON_FNV1A_OFFSET_BASIS = 2166136261u; +static const uint32_t MQTT_JSON_FNV1A_PRIME = 16777619u; + // Set default values for MQTT preferences (used when file doesn't exist or is corrupted) static void setMQTTPrefsDefaults(MQTTPrefs* prefs) { applyMQTTDefaults(prefs); @@ -396,6 +403,124 @@ static File openMqttPrefsRead(FILESYSTEM* fs, const char* path = "/mqtt_prefs") #endif } +enum class JsonPrefsLoadResult : uint8_t { + Loaded, + LoadedWithRepairs, + UnsupportedVersion, + FutureClaimed, + Invalid, +}; + +static JsonPrefsLoadResult loadMqttJsonFile(FILESYSTEM* fs, const char* path, + MQTTPrefs* output) { + if (output == nullptr) return JsonPrefsLoadResult::Invalid; + applyMQTTDefaults(output); + + // Probe the root version without applying the v1 schema. A future version + // may legitimately change an existing field's type, and must still be held + // opaquely rather than misclassified as corrupt and rolled back. + File version_file = openMqttPrefsRead(fs, path); + if (!version_file) return JsonPrefsLoadResult::Invalid; + MQTTPrefsVersionProbe version_probe; + const bool version_parsed = version_probe.loadSerial(version_file); + version_file.close(); + if (version_probe.hasFutureVersion()) { + return version_parsed ? JsonPrefsLoadResult::UnsupportedVersion + : JsonPrefsLoadResult::FutureClaimed; + } + + File file = openMqttPrefsRead(fs, path); + if (!file) return JsonPrefsLoadResult::Invalid; + MQTTPrefsSerializer serializer(output); + const bool parsed = serializer.loadSerial(file); + file.close(); + if (!parsed) return JsonPrefsLoadResult::Invalid; + if (serializer.hasFutureVersion()) return JsonPrefsLoadResult::UnsupportedVersion; + bool repaired = false; + if (!serializer.apply(&repaired)) return JsonPrefsLoadResult::Invalid; + return repaired ? JsonPrefsLoadResult::LoadedWithRepairs : JsonPrefsLoadResult::Loaded; +} + +static MQTTPrefsRecovery::FileState mqttJsonFileState(FILESYSTEM* fs, const char* path) { + if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; + MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; + if (scratch == nullptr) return MQTTPrefsRecovery::FileState::Indeterminate; + const JsonPrefsLoadResult result = loadMqttJsonFile(fs, path, scratch); + delete scratch; + if (result == JsonPrefsLoadResult::UnsupportedVersion) { + // Syntax and the mandatory version field were valid, so this can be a + // fully verified transaction written by newer firmware. Keep it distinct + // from a torn/corrupt temp during recovery. + return MQTTPrefsRecovery::FileState::FutureUsable; + } + if (result == JsonPrefsLoadResult::FutureClaimed) { + return MQTTPrefsRecovery::FileState::FutureClaimed; + } + return result == JsonPrefsLoadResult::Loaded || + result == JsonPrefsLoadResult::LoadedWithRepairs + ? MQTTPrefsRecovery::FileState::Usable + : MQTTPrefsRecovery::FileState::Preserve; +} + +static bool recoverMqttJsonFiles(FILESYSTEM* fs) { + const MQTTPrefsRecovery::FileState primary = mqttJsonFileState(fs, "/mqtt.json"); + const MQTTPrefsRecovery::FileState temp = mqttJsonFileState(fs, "/mqtt.json.tmp"); + const MQTTPrefsRecovery::FileState backup = mqttJsonFileState(fs, "/mqtt.json.bak"); + const MQTTPrefsRecovery::Action action = MQTTPrefsRecovery::select(primary, temp, backup); + + if (action == MQTTPrefsRecovery::Action::KeepPrimary) { + if (primary == MQTTPrefsRecovery::FileState::Usable) { + if (!MQTTPrefsRecovery::uncertain(temp) && + temp != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.tmp"); + } + if (!MQTTPrefsRecovery::uncertain(backup) && + backup != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.bak"); + } + } + return MQTTPrefsRecovery::uncertain(primary) || + MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + if (action == MQTTPrefsRecovery::Action::DiscardTemp) { + if (fs->remove("/mqtt.json.tmp")) { + MESH_DEBUG_PRINTLN("MQTT: discarded incomplete first-migration JSON temp"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not discard incomplete /mqtt.json temp; source held"); + return true; + } + if (action == MQTTPrefsRecovery::Action::PromoteTemp) { + if (fs->rename("/mqtt.json.tmp", "/mqtt.json")) { + if (temp == MQTTPrefsRecovery::FileState::Usable && + backup != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.bak"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction temp"); + return MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json temp; files preserved"); + return true; + } + if (action == MQTTPrefsRecovery::Action::PromoteBackup) { + if (fs->rename("/mqtt.json.bak", "/mqtt.json")) { + if (backup == MQTTPrefsRecovery::FileState::Usable && + !MQTTPrefsRecovery::uncertain(temp) && + temp != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt.json.tmp"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction backup"); + return MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup); + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json backup; files preserved"); + return true; + } + return false; +} + static MQTTPrefsRecovery::FileState mqttPrefsFileState(FILESYSTEM* fs, const char* path) { if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; File file = openMqttPrefsRead(fs, path); @@ -432,6 +557,14 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { } return false; } + if (action == MQTTPrefsRecovery::Action::DiscardTemp) { + if (fs->remove("/mqtt_prefs.tmp")) { + MESH_DEBUG_PRINTLN("MQTT: discarded incomplete legacy transaction temp"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not discard incomplete /mqtt_prefs temp; source held"); + return true; + } if (action == MQTTPrefsRecovery::Action::PromoteTemp) { if (fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) { // A usable temp is now the committed primary. Its backup is necessarily @@ -463,30 +596,31 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { return false; } -// Filesystem adapter for MQTTPrefsAtomicStore. It writes the new image to -// /mqtt_prefs.tmp and verifies its size. Publishing is a recoverable SPIFFS +// Filesystem adapter for the ConfigSerializer image. It writes to +// /mqtt.json.tmp and verifies its size and checksum. Publishing is a recoverable SPIFFS // transaction: primary -> .bak, then tmp -> primary, then best-effort backup // cleanup. A power loss at every boundary leaves at least one recoverable file. -class MQTTPrefsFileStore { +class MQTTPrefsJsonFileStore { public: - explicit MQTTPrefsFileStore(FILESYSTEM* fs) : _fs(fs) {} + explicit MQTTPrefsJsonFileStore(FILESYSTEM* fs) : _fs(fs) {} bool begin() { _finished = false; _open = false; _owns_temp = false; _bytes_written = 0; + _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; // Recovery owns stale artifacts. Do not delete them here: a failed commit // may have moved the old primary to .bak and left a verified temp that the // next boot must choose between. Refusing the save is safer than erasing an // image this firmware cannot decode. - if (_fs->exists("/mqtt_prefs.tmp") || _fs->exists("/mqtt_prefs.bak")) return false; + if (_fs->exists("/mqtt.json.tmp") || _fs->exists("/mqtt.json.bak")) return false; #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) - _file = _fs->open("/mqtt_prefs.tmp", FILE_O_WRITE); + _file = _fs->open("/mqtt.json.tmp", FILE_O_WRITE); #elif defined(RP2040_PLATFORM) - _file = _fs->open("/mqtt_prefs.tmp", "w"); + _file = _fs->open("/mqtt.json.tmp", "w"); #else - _file = _fs->open("/mqtt_prefs.tmp", "w", true); + _file = _fs->open("/mqtt.json.tmp", "w", true); #endif _open = _file; _owns_temp = _open; @@ -496,6 +630,9 @@ public: size_t write(const uint8_t* bytes, size_t size) { if (!_open) return 0; const size_t written = _file.write(bytes, size); + for (size_t i = 0; i < written; ++i) { + _expected_crc = (_expected_crc ^ bytes[i]) * MQTT_JSON_FNV1A_PRIME; + } _bytes_written += written; return written; } @@ -505,12 +642,33 @@ public: _file.close(); _open = false; #if defined(RP2040_PLATFORM) - File verify = _fs->open("/mqtt_prefs.tmp", "r"); + File verify = _fs->open("/mqtt.json.tmp", "r"); #else - File verify = _fs->open("/mqtt_prefs.tmp"); + File verify = _fs->open("/mqtt.json.tmp"); #endif if (!verify) return false; - const bool complete = verify.size() == _bytes_written; + uint32_t actual_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; + size_t actual_size = 0; + bool read_failed = false; + uint8_t buf[64]; + while (verify.available() > 0) { + // Arduino File implementations normally return a byte count, but some + // Stream implementations use -1 for a read error. Keep that sentinel + // signed so it cannot become a huge size_t and overrun this buffer. + const int count = static_cast(verify.read(buf, sizeof(buf))); + if (count <= 0) { + read_failed = count < 0; + break; + } + actual_size += static_cast(count); + for (int i = 0; i < count; ++i) { + actual_crc = (actual_crc ^ buf[i]) * MQTT_JSON_FNV1A_PRIME; + } + } + const bool complete = !read_failed && + verify.size() == _bytes_written && + actual_size == _bytes_written && + actual_crc == _expected_crc; verify.close(); if (!complete) return false; _finished = true; @@ -523,25 +681,33 @@ public: // recoverable backup first, then publish temp into the now-empty primary. // Never remove either image after a failed boundary; boot recovery selects // the completed temp or restores the backup. - if (_fs->exists("/mqtt_prefs.bak")) return false; - if (_fs->exists("/mqtt_prefs") && !_fs->rename("/mqtt_prefs", "/mqtt_prefs.bak")) { + if (_fs->exists("/mqtt.json.bak")) return false; + if (_fs->exists("/mqtt.json") && !_fs->rename("/mqtt.json", "/mqtt.json.bak")) { return false; } - if (!_fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) return false; + if (!_fs->rename("/mqtt.json.tmp", "/mqtt.json")) return false; // Cleanup failure is non-fatal: the new primary is published and recovery // will remove a known-good stale backup on a later boot. - if (_fs->exists("/mqtt_prefs.bak")) _fs->remove("/mqtt_prefs.bak"); + if (_fs->exists("/mqtt.json.bak")) _fs->remove("/mqtt.json.bak"); return true; } + void discardFinishedTemp() { + if (_open) _file.close(); + _open = false; + if (_owns_temp && _fs->exists("/mqtt.json.tmp")) _fs->remove("/mqtt.json.tmp"); + _finished = false; + _owns_temp = false; + } + void abort() { if (_open) _file.close(); _open = false; // Once finish() has verified the temp, commit may already have moved the // primary to .bak. Keep the temp on a commit failure so recovery can // publish it (or fall back to .bak) after reset. - if (_owns_temp && !_finished && _fs->exists("/mqtt_prefs.tmp")) { - _fs->remove("/mqtt_prefs.tmp"); + if (_owns_temp && !_finished && _fs->exists("/mqtt.json.tmp")) { + _fs->remove("/mqtt.json.tmp"); } _finished = false; _owns_temp = false; @@ -554,31 +720,90 @@ private: bool _finished = false; bool _owns_temp = false; size_t _bytes_written = 0; + uint32_t _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; +}; + +class MQTTPrefsStoreStream : public Stream { +public: + explicit MQTTPrefsStoreStream(MQTTPrefsJsonFileStore* store) : _store(store) {} + + size_t write(uint8_t byte) override { return write(&byte, 1); } + size_t write(const uint8_t* buffer, size_t size) override { + if (!_ok || _store == nullptr) return 0; + const size_t written = _store->write(buffer, size); + if (written != size) _ok = false; + return written; + } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + bool ok() const { return _ok; } + +private: + MQTTPrefsJsonFileStore* _store; + bool _ok = true; }; #endif // WITH_MQTT_BRIDGE #ifdef WITH_MQTT_BRIDGE -static const char* mqttPrefsSaveResultName(MQTTPrefsAtomicStore::Result result) { - switch (result) { - case MQTTPrefsAtomicStore::Result::BeginFailed: return "begin"; - case MQTTPrefsAtomicStore::Result::HeaderWriteFailed: return "header write"; - case MQTTPrefsAtomicStore::Result::PayloadWriteFailed: return "payload write"; - case MQTTPrefsAtomicStore::Result::FinishFailed: return "close"; - case MQTTPrefsAtomicStore::Result::CommitFailed: return "rename"; - case MQTTPrefsAtomicStore::Result::Committed: return "committed"; - } - return "unknown"; -} - void CommonCLI::loadMQTTPrefs( FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade) { setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = recoverMqttJsonFiles(fs); + if (_mqtt_prefs_hold && !fs->exists("/mqtt.json") && + (fs->exists("/mqtt.json.tmp") || fs->exists("/mqtt.json.bak"))) { + _legacy_tail.valid = false; + MESH_DEBUG_PRINTLN("MQTT: unresolved /mqtt.json recovery files; using defaults (files preserved)"); + return; + } + + // The JSON file is authoritative once it exists. Never fall back to the + // stale binary snapshot when JSON is corrupt, unreadable, or from a future + // schema: preserve it and run defaults until an operator resolves it. + if (fs->exists("/mqtt.json")) { + MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; + if (scratch == nullptr) { + _mqtt_prefs_hold = true; + _legacy_tail.valid = false; + MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json; source preserved"); + return; + } + const JsonPrefsLoadResult json_result = loadMqttJsonFile(fs, "/mqtt.json", scratch); + if (json_result == JsonPrefsLoadResult::Loaded || + json_result == JsonPrefsLoadResult::LoadedWithRepairs) { + memcpy(&_mqtt_prefs, scratch, sizeof(_mqtt_prefs)); + delete scratch; + _legacy_tail.valid = false; + if (json_result == JsonPrefsLoadResult::LoadedWithRepairs) { + MESH_DEBUG_PRINTLN("MQTT: repaired out-of-range values in /mqtt.json"); + if (!_mqtt_prefs_hold && !saveMQTTPrefs(fs)) { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: could not persist /mqtt.json repairs; source held"); + } + } + return; + } + delete scratch; + _mqtt_prefs_hold = true; + _legacy_tail.valid = false; + if (json_result == JsonPrefsLoadResult::UnsupportedVersion) { + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json uses a future version; using defaults (file preserved)"); + } else if (json_result == JsonPrefsLoadResult::FutureClaimed) { + MESH_DEBUG_PRINTLN( + "MQTT: /mqtt.json claims a future version but uses unknown grammar; " + "using defaults (file preserved)"); + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json is invalid or unreadable; using defaults (file preserved)"); + } + return; + } + // Complete or preserve an interrupted SPIFFS transaction before decoding. // A failed recovery leaves the artifacts untouched and blocks this boot from // replacing them with defaults through a later CLI save. - _mqtt_prefs_hold = recoverMqttPrefsFiles(fs); + _mqtt_prefs_hold = _mqtt_prefs_hold || recoverMqttPrefsFiles(fs); bool has_observer_fields = false; bool mqtt_rewrite_pending = false; bool migrated_legacy_mqtt = false; @@ -600,24 +825,34 @@ void CommonCLI::loadMQTTPrefs( } else if (plan.source == MQTTPrefsCodec::Source::Current) { file = openMqttPrefsRead(fs); MQTTPrefsHeader header; - if (!file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || - file.read((uint8_t *)&_mqtt_prefs, plan.payload_len) != plan.payload_len) { + LegacyV1MQTTPrefs* old_prefs = new (std::nothrow) LegacyV1MQTTPrefs; + if (old_prefs) memset(old_prefs, 0, sizeof(*old_prefs)); + if (!old_prefs || !file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || + file.read((uint8_t *)old_prefs, plan.payload_len) != plan.payload_len) { setMQTTPrefsDefaults(&_mqtt_prefs); _mqtt_prefs_hold = true; MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs read failed, using defaults (file preserved)"); + } else if (!MQTTPrefsCodec::isPlausibleV1(*old_prefs, plan.payload_len)) { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs v1 content failed plausibility checks; source preserved"); } else { + MQTTPrefsCodec::migrateV1(*old_prefs, plan.payload_len, &_mqtt_prefs); has_observer_fields = plan.observer_fields_present; + mqtt_rewrite_pending = true; + migrated_legacy_mqtt = true; // Written by a later build with appended fields. Everything this // binary knows loaded normally; say so, because the next `set` will // rewrite the file at this length and drop the newer settings. if (file_size - sizeof(MQTTPrefsHeader) > plan.payload_len) { MESH_DEBUG_PRINTLN( "MQTT: /mqtt_prefs written by newer firmware (%u > %u bytes); " - "config loaded, newer settings ignored and dropped on next save", + "known settings loaded; newer binary fields remain in the rollback snapshot", (unsigned)(file_size - sizeof(MQTTPrefsHeader)), (unsigned)plan.payload_len); } } + delete old_prefs; if (file) file.close(); } else if (plan.rewrite_legacy) { bool migrated = false; @@ -674,16 +909,18 @@ void CommonCLI::loadMQTTPrefs( case MQTTPrefsCodec::Source::LegacySixSlotAudience: case MQTTPrefsCodec::Source::LegacySixSlotAudienceRx: case MQTTPrefsCodec::Source::LegacySixSlot: { - Legacy6SlotMQTTPrefs old_prefs = {}; - if (file.read((uint8_t *)&old_prefs, plan.payload_len) == plan.payload_len) { + Legacy6SlotMQTTPrefs* old_prefs = new (std::nothrow) Legacy6SlotMQTTPrefs; + if (old_prefs) memset(old_prefs, 0, sizeof(*old_prefs)); + if (old_prefs && file.read((uint8_t *)old_prefs, plan.payload_len) == plan.payload_len) { if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, - (const uint8_t *)&old_prefs, plan.payload_len)) { - MQTTPrefsCodec::migrateLegacySixSlot(old_prefs, plan.source, &_mqtt_prefs); + (const uint8_t *)old_prefs, plan.payload_len)) { + MQTTPrefsCodec::migrateLegacySixSlot(*old_prefs, plan.source, &_mqtt_prefs); migrated = true; } else { MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); } } + delete old_prefs; break; } default: @@ -742,9 +979,9 @@ void CommonCLI::loadMQTTPrefs( if (mqtt_rewrite_pending) { legacy_upgrade->requireMqttRewrite(); if (migrated_legacy_mqtt) { - MESH_DEBUG_PRINTLN("MQTT: Migrating headerless /mqtt_prefs to versioned layout"); + MESH_DEBUG_PRINTLN("MQTT: Migrating binary /mqtt_prefs to /mqtt.json"); } else { - MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt_prefs before /com_prefs compaction"); + MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt.json before /com_prefs compaction"); } if (saveMQTTPrefs(fs)) { legacy_upgrade->recordMqttSave(true); @@ -754,7 +991,7 @@ void CommonCLI::loadMQTTPrefs( // untouched; the next boot can recover the tail and retry the transaction. _mqtt_prefs_hold = true; legacy_upgrade->recordMqttSave(false); - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs migration save failed; legacy files preserved and held"); + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json migration save failed; legacy files preserved and held"); } } } @@ -763,26 +1000,71 @@ bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { if (_mqtt_prefs_hold) { // Loading deliberately preserved the source file. Do not replace it with this // boot's defaults after an unsupported, corrupt, or temporarily failed read. - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs held, not overwriting"); + MESH_DEBUG_PRINTLN("MQTT: observer preference source held, not overwriting /mqtt.json"); return false; } - // Write header and payload sequentially so the transaction needs no second - // full-size (2.8 KiB) staging buffer on constrained targets. The length is - // the shortest that still round-trips this config, so a node with default - // packet filters keeps writing a payload older firmware can read. - const size_t payload_len = MQTTPrefsCodec::payloadLenFor(_mqtt_prefs); - const MQTTPrefsHeader header = MQTTPrefsCodec::makeHeader(payload_len); - MQTTPrefsFileStore store(fs); - const MQTTPrefsAtomicStore::Result result = MQTTPrefsAtomicStore::write( - store, (const uint8_t *)&header, sizeof(header), - (const uint8_t *)&_mqtt_prefs, payload_len); - if (!MQTTPrefsAtomicStore::committed(result)) { - MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt_prefs save failed at %s; source preserved", - mqttPrefsSaveResultName(result)); + MQTTPrefs* repair_defaults = new (std::nothrow) MQTTPrefs; + if (repair_defaults == nullptr) { + MESH_DEBUG_PRINTLN("MQTT: no memory to normalize observer settings before save"); return false; } - return true; + setMQTTPrefsDefaults(repair_defaults); + MQTTPrefsSerializer serializer(&_mqtt_prefs, repair_defaults); + // The serializer hierarchy copies the default values it needs. Release this + // large temporary before file I/O and the independent verification scratch. + delete repair_defaults; + bool repaired = false; + if (!serializer.normalize(&repaired)) return false; + if (repaired) MESH_DEBUG_PRINTLN("MQTT: normalized out-of-range observer settings before save"); + + MQTTPrefsJsonFileStore store(fs); + bool verify_oom = false; + const MQTTPrefsAtomicStore::VerifiedImageResult result = + MQTTPrefsAtomicStore::writeVerifiedImage( + store, + [&]() -> bool { + MQTTPrefsStoreStream stream(&store); + return serializer.saveSerial(stream) && stream.ok(); + }, + [&]() -> bool { + MQTTPrefs* verify = new (std::nothrow) MQTTPrefs; + if (verify == nullptr) { + verify_oom = true; + return false; + } + const JsonPrefsLoadResult verify_result = + loadMqttJsonFile(fs, "/mqtt.json.tmp", verify); + delete verify; + // saveSerial() emits already-normalized values. Needing another + // repair here means save/load is not idempotent. + return verify_result == JsonPrefsLoadResult::Loaded; + }); + + switch (result) { + case MQTTPrefsAtomicStore::VerifiedImageResult::Committed: + return true; + case MQTTPrefsAtomicStore::VerifiedImageResult::BeginFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed at begin; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::WriteFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during write; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::FinishFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during checksum verification; source preserved"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::VerifyFailed: + if (verify_oom) { + MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json temp; source preserved"); + } else { + MESH_DEBUG_PRINTLN("MQTT: generated /mqtt.json temp failed schema validation; source preserved"); + } + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::CommitFailed: + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename; recovery files preserved"); + break; + } + return false; } #endif @@ -1419,6 +1701,15 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep strcpy(reply, "Error: delay must be between 0-10000 ms"); } } else if (memcmp(config, "bridge.source ", 14) == 0) { +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* observer_rollback = new (std::nothrow) MQTTPrefs; + if (observer_rollback == nullptr) { + strcpy(reply, "Error: insufficient memory to update observer setting"); + return; + } + memcpy(observer_rollback, &_mqtt_prefs, sizeof(*observer_rollback)); + const uint8_t old_bridge_pkt_src = _prefs->bridge_pkt_src; +#endif _prefs->bridge_pkt_src = memcmp(&config[14], "rx", 2) == 0; #ifdef WITH_MQTT_BRIDGE if (_prefs->bridge_pkt_src == 1) { @@ -1428,6 +1719,15 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep _mqtt_prefs.mqtt_rx_enabled = 0; _mqtt_prefs.mqtt_tx_enabled = 1; } + _observer_prefs_rollback = observer_rollback; + if (!persistObserverPrefs(reply)) { + _prefs->bridge_pkt_src = old_bridge_pkt_src; + _observer_prefs_rollback = nullptr; + delete observer_rollback; + return; + } + _observer_prefs_rollback = nullptr; + delete observer_rollback; #endif savePrefs(); strcpy(reply, "OK"); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index e0a0a71b..2e2f02ef 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -73,7 +73,7 @@ public: uint8_t extra_sf[4]; // NOTE: observer settings (MQTT/WiFi/timezone/SNMP/alert) are not in NodePrefs. - // They live in MQTTPrefs, persisted separately to /mqtt_prefs, so this struct + // They live in MQTTPrefs, persisted separately to /mqtt.json, so this struct // stays aligned with upstream. See struct MQTTPrefs below. private: @@ -228,6 +228,13 @@ struct LegacyObserverTail { class CommonCLICallbacks { public: virtual void savePrefs() = 0; +#ifdef WITH_MQTT_BRIDGE + virtual bool saveObserverPrefs() = 0; +#else + virtual bool saveObserverPrefs() { + return false; + } +#endif virtual const char* getFirmwareVer() = 0; virtual const char* getBuildDate() = 0; virtual const char* getRole() = 0; @@ -359,8 +366,11 @@ class CommonCLI { char tmp[PRV_KEY_SIZE*2 + 4]; #ifdef WITH_MQTT_BRIDGE MQTTPrefs _mqtt_prefs; + // Points at a per-command snapshot only while an observer setter is running. + // persistObserverPrefs() uses it to undo RAM mutations when flash commit fails. + const MQTTPrefs* _observer_prefs_rollback = nullptr; LegacyObserverTail _legacy_tail; - // /mqtt_prefs is newer, corrupt, or temporarily unreadable. The in-memory prefs + // /mqtt.json is newer, corrupt, or temporarily unreadable. The in-memory prefs // run on defaults and saveMQTTPrefs() must not overwrite the source file. bool _mqtt_prefs_hold = false; #endif @@ -384,6 +394,7 @@ class CommonCLI { // false to fall through to the base get/set parsing. bool handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply); bool handleObserverGetCmd(uint32_t sender_timestamp, const char* config, char* reply); + bool persistObserverPrefs(char* reply); // Observer-only top-level commands (ota check/update, tls.bundletest, alert test) // also live in CommonCLI_Observer.cpp; returns true if it handled the command. bool handleObserverCommand(uint32_t sender_timestamp, char* command, char* reply); @@ -393,14 +404,17 @@ public: : _board(&board), _rtc(&rtc), _sensors(&sensors), _region_map(®ion_map), _acl(&acl), _prefs(prefs), _callbacks(callbacks) { } void loadPrefs(FILESYSTEM* _fs); - bool savePrefs(FILESYSTEM* _fs, bool save_mqtt = true); + // Node preferences and observer preferences are separate transactions. + // Callers must explicitly request an observer save when they changed it. + bool savePrefs(FILESYSTEM* _fs, bool save_mqtt = false); void handleCommand(uint32_t sender_timestamp, char* command, char* reply); mesh::MainBoard* getBoard() { return _board; } uint8_t buildAdvertData(uint8_t node_type, uint8_t* app_data); #ifdef WITH_MQTT_BRIDGE - // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt_prefs. + // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // Exposed so the app can hand it to MQTTBridge/AlertReporter, which read these // fields directly (they no longer live in NodePrefs). MQTTPrefs* getObserverPrefs() const { return const_cast(&_mqtt_prefs); } + bool saveObserverPrefs(FILESYSTEM* fs) { return saveMQTTPrefs(fs); } #endif }; diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index c1b7e74a..93c25776 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -16,6 +16,7 @@ #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) #include +#include #ifdef ESP_PLATFORM #include #include @@ -109,6 +110,46 @@ static bool valueTooLong(const char* val, size_t bufsize, char* reply, const cha return false; } +static bool isObserverPrefsSetCommand(const char* config) { + return strncmp(config, "snmp", 4) == 0 || + strncmp(config, "radio.watchdog", 14) == 0 || + strncmp(config, "mqtt", 4) == 0 || + strncmp(config, "wifi.", 5) == 0 || + strncmp(config, "timezone", 8) == 0 || + strncmp(config, "alert", 5) == 0; +} + +// Keep observer setters atomic from the caller's perspective. The live object +// is restored if its JSON transaction fails, and the snapshot pointer is valid +// only for this synchronous command dispatch. +class ObserverPrefsRollbackScope { +public: + ObserverPrefsRollbackScope(const MQTTPrefs** active_snapshot, + const MQTTPrefs& live, + bool capture) + : _active_snapshot(active_snapshot) { + *_active_snapshot = nullptr; + if (capture) { + _snapshot = new (std::nothrow) MQTTPrefs; + if (_snapshot != nullptr) { + memcpy(_snapshot, &live, sizeof(*_snapshot)); + *_active_snapshot = _snapshot; + } + } + } + + ~ObserverPrefsRollbackScope() { + *_active_snapshot = nullptr; + delete _snapshot; + } + + bool available() const { return _snapshot != nullptr; } + +private: + const MQTTPrefs** _active_snapshot; + MQTTPrefs* _snapshot = nullptr; +}; + static const char* getMQTTPresetNameByIndex(int index) { if (index < MQTT_PRESET_COUNT) return MQTT_PRESETS[index].name; if (index == MQTT_PRESET_COUNT) return MQTT_PRESET_CUSTOM; @@ -161,17 +202,38 @@ static void formatMQTTPresetListReply(char* reply, size_t reply_size, int start) } #endif +bool CommonCLI::persistObserverPrefs(char* reply) { +#ifdef WITH_MQTT_BRIDGE + if (_callbacks->saveObserverPrefs()) return true; + if (_observer_prefs_rollback != nullptr) { + memcpy(&_mqtt_prefs, _observer_prefs_rollback, sizeof(_mqtt_prefs)); + } + strcpy(reply, "Error: setting not persisted; change rolled back"); + return false; +#else + (void)reply; + return false; +#endif +} + bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply) { #ifdef WITH_MQTT_BRIDGE + const bool needs_snapshot = isObserverPrefsSetCommand(config); + ObserverPrefsRollbackScope rollback_scope( + &_observer_prefs_rollback, _mqtt_prefs, needs_snapshot); + if (needs_snapshot && !rollback_scope.available()) { + strcpy(reply, "Error: insufficient memory to update observer setting"); + return true; + } bool handled = true; if (memcmp(config, "snmp.community ", 15) == 0) { if (valueTooLong(&config[15], sizeof(_mqtt_prefs.snmp_community), reply, "snmp.community")) return true; StrHelper::strncpy(_mqtt_prefs.snmp_community, &config[15], sizeof(_mqtt_prefs.snmp_community)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - restart to apply"); } else if (memcmp(config, "snmp ", 5) == 0) { _mqtt_prefs.snmp_enabled = memcmp(&config[5], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - restart to apply"); } else if (memcmp(config, "radio.watchdog ", 15) == 0) { const char* val = &config[15]; @@ -189,7 +251,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: radio.watchdog must be 0-120 minutes"); } else { _mqtt_prefs.radio_watchdog_minutes = (uint8_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; if (mins == 0) { strcpy(reply, "OK - radio watchdog disabled"); } else { @@ -200,13 +262,13 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf #ifdef WITH_MQTT_BRIDGE } else if (strcmp(config, "mqtt.origin") == 0) { _mqtt_prefs.mqtt_origin[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.origin ", 12) == 0) { if (valueTooLong(&config[12], sizeof(_mqtt_prefs.mqtt_origin), reply, "origin")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_origin, &config[12], sizeof(_mqtt_prefs.mqtt_origin)); StrHelper::stripSurroundingQuotes(_mqtt_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.iata ", 10) == 0) { const char* iata = &config[10]; @@ -215,7 +277,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // Empty clears the region code (meshcore-topic publishing stays disabled // until one is set). This keeps the pre-existing "clear IATA" capability. _mqtt_prefs.mqtt_iata[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); strcpy(reply, "OK - IATA cleared"); } else { @@ -228,22 +290,22 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf for (int i = 0; _mqtt_prefs.mqtt_iata[i]; i++) { _mqtt_prefs.mqtt_iata[i] = toupper(_mqtt_prefs.mqtt_iata[i]); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); strcpy(reply, "OK"); } } } else if (memcmp(config, "mqtt.status ", 12) == 0) { _mqtt_prefs.mqtt_status_enabled = memcmp(&config[12], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.packets ", 13) == 0) { _mqtt_prefs.mqtt_packets_enabled = memcmp(&config[13], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.raw ", 9) == 0) { _mqtt_prefs.mqtt_raw_enabled = memcmp(&config[9], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.tx ", 8) == 0) { if (memcmp(&config[8], "advert", 6) == 0) { @@ -251,17 +313,17 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { _mqtt_prefs.mqtt_tx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.rx ", 8) == 0) { _mqtt_prefs.mqtt_rx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.interval ", 14) == 0) { uint32_t minutes = _atoi(&config[14]); if (minutes >= 1 && minutes <= 60) { _mqtt_prefs.mqtt_status_interval = minutes * 60000; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridge(); sprintf(reply, "OK - interval set to %u minutes (%lu ms), bridge restarted", minutes, (unsigned long)_mqtt_prefs.mqtt_status_interval); } else { @@ -274,7 +336,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf uint32_t hours = _atoi(&config[24]); if (hours >= MQTT_NEIGHBORS_MIN_INTERVAL_HOURS && hours <= MQTT_NEIGHBORS_MAX_INTERVAL_HOURS) { _mqtt_prefs.mqtt_neighbors_interval = hours * 3600000UL; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - neighbors interval set to %u hours (%lu ms)", (unsigned)hours, (unsigned long)_mqtt_prefs.mqtt_neighbors_interval); } else { @@ -284,7 +346,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // The mesh loop reads this live, so no bridge restart is needed; enabling it // triggers a discovery on the next eligible loop pass. _mqtt_prefs.mqtt_neighbors_enabled = memcmp(&config[15], "on", 2) == 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); #elif defined(WITH_MQTT_BRIDGE) } else if (memcmp(config, "mqtt.neighbors.interval ", 24) == 0 || @@ -303,7 +365,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { StrHelper::strncpy(_mqtt_prefs.mqtt_ntp_server, host, sizeof(_mqtt_prefs.mqtt_ntp_server)); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM // Queue a sync on the MQTT task (Core 0) but do NOT block: this handler // runs on the Arduino loop task, shared with mesh/radio processing and the @@ -325,12 +387,12 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "wifi.ssid ", 10) == 0) { if (valueTooLong(&config[10], sizeof(_mqtt_prefs.wifi_ssid), reply, "wifi.ssid")) return true; StrHelper::strncpy(_mqtt_prefs.wifi_ssid, &config[10], sizeof(_mqtt_prefs.wifi_ssid)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "wifi.pwd ", 9) == 0) { if (valueTooLong(&config[9], sizeof(_mqtt_prefs.wifi_password), reply, "wifi.pwd")) return true; StrHelper::strncpy(_mqtt_prefs.wifi_password, &config[9], sizeof(_mqtt_prefs.wifi_password)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "wifi.powersave ", 15) == 0) { const char* value = &config[15]; @@ -350,7 +412,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: must be none, min, or max"); } else { _mqtt_prefs.wifi_power_save = ps_value; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM if (WiFi.status() == WL_CONNECTED) { wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : @@ -374,13 +436,13 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "timezone ", 9) == 0) { if (valueTooLong(&config[9], sizeof(_mqtt_prefs.timezone_string), reply, "timezone")) return true; StrHelper::strncpy(_mqtt_prefs.timezone_string, &config[9], sizeof(_mqtt_prefs.timezone_string)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "timezone.offset ", 16) == 0) { int8_t offset = _atoi(&config[16]); if (offset >= -12 && offset <= 14) { _mqtt_prefs.timezone_offset = offset; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else { strcpy(reply, "Error: timezone offset must be between -12 and +14"); @@ -397,20 +459,14 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0 || strcmp(preset_name, MQTT_PRESET_NONE) == 0) { // Reject duplicate presets (except "none" and "custom") - int dup_slot = -1; - if (findMQTTPreset(preset_name) != nullptr) { - for (int s = 0; s < MAX_MQTT_SLOTS; s++) { - if (s != slot && strcmp(_mqtt_prefs.mqtt_slot_preset[s], preset_name) == 0) { - dup_slot = s; - break; - } - } - } + const int dup_slot = findMQTTPreset(preset_name) == nullptr ? -1 : + mqttAssignedPresetSlot(_mqtt_prefs.mqtt_slot_preset, + preset_name, slot); if (dup_slot >= 0) { sprintf(reply, "Error: preset '%s' is already assigned to slot %d", preset_name, dup_slot + 1); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], preset_name, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); // Check if the slot has everything it needs to connect const MQTTPresetDef* p = findMQTTPreset(preset_name); @@ -469,7 +525,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "server ", 7) == 0) { if (valueTooLong(&subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot]), reply, "server")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_host[slot], &subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; // Reconfigure the slot so the new host reaches the live connection (other // custom-slot setters do the same; without it the change only applies on // the next reboot/bridge restart). @@ -479,7 +535,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf int port = atoi(&subcmd[5]); if (port > 0 && port <= 65535) { _mqtt_prefs.mqtt_slot_port[slot] = port; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else { @@ -488,19 +544,19 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "username ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot]), reply, "username")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_username[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(subcmd, "password ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot]), reply, "password")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_password[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(subcmd, "token ", 6) == 0) { if (valueTooLong(&subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot]), reply, "token")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_token[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d token set", slot + 1); } else if (memcmp(subcmd, "topic ", 6) == 0) { @@ -510,14 +566,14 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf return true; } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_topic[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_topic[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d topic: %s", slot + 1, _mqtt_prefs.mqtt_slot_topic[slot]); } } else if (memcmp(subcmd, "audience ", 9) == 0) { if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot]), reply, "audience")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_slot_audience[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot])); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); if (_mqtt_prefs.mqtt_slot_audience[slot][0] != '\0') { sprintf(reply, "OK - slot %d JWT audience: %s", slot + 1, _mqtt_prefs.mqtt_slot_audience[slot]); @@ -527,7 +583,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "audience", 8) == 0 && subcmd[8] == '\0') { // "set mqttN.audience" with no value — clear the audience _mqtt_prefs.mqtt_slot_audience[slot][0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); sprintf(reply, "OK - slot %d JWT audience cleared (using username/password auth)", slot + 1); } else if (strcmp(subcmd, "filter") == 0 || @@ -539,20 +595,18 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: filter must be all, none, or a CSV of types 0-15 / names (advert,txt_msg,...)"); } else { _mqtt_prefs.mqtt_slot_packet_filter[slot] = filter_mask; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; char filter_text[MQTTPacketFilter::kFilterTextSize]; MQTTPacketFilter::format(filter_mask, filter_text, sizeof(filter_text)); snprintf(reply, 160, "OK - slot %d packet types: %s", slot + 1, filter_text); - // A non-default filter extends /mqtt_prefs past what pre-filter - // firmware can read (see MQTTPrefsCodec::payloadLenFor), so say when - // that cost buys nothing: slots beyond the runtime array are never - // published to on this board, the same warning `preset` gives. + // Slots beyond the runtime array are never published to on this board, + // so retain the same inactive-hardware warning `preset` gives. if (slot >= RUNTIME_MQTT_SLOTS && filter_mask != MQTTPacketFilter::kAllPacketTypes) { size_t used = strlen(reply); if (used < 158) { snprintf(reply + used, 160 - used, - " (slot inactive on this hardware; blocks firmware rollback)"); + " (slot inactive on this hardware)"); } } } @@ -562,21 +616,35 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "mqtt.analyzer.us ", 17) == 0) { const int slot = 0; if (memcmp(&config[17], "on", 2) == 0) { + const int dup_slot = mqttAssignedPresetSlot( + _mqtt_prefs.mqtt_slot_preset, "analyzer-us", slot); + if (dup_slot >= 0) { + sprintf(reply, "Error: preset 'analyzer-us' is already assigned to slot %d", + dup_slot + 1); + return true; + } StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.analyzer.eu ", 17) == 0) { const int slot = 1; if (memcmp(&config[17], "on", 2) == 0) { + const int dup_slot = mqttAssignedPresetSlot( + _mqtt_prefs.mqtt_slot_preset, "analyzer-eu", slot); + if (dup_slot >= 0) { + sprintf(reply, "Error: preset 'analyzer-eu' is already assigned to slot %d", + dup_slot + 1); + return true; + } StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } else { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); } - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.owner ", 11) == 0) { @@ -585,11 +653,11 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // Owner key is optional — empty clears it (previously this errored, so a // set key could never be removed via the portal/CLI). _mqtt_prefs.mqtt_owner_public_key[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK - owner key cleared"); } else if (mqttOwnerKeyValid(owner_key)) { StrHelper::strncpy(_mqtt_prefs.mqtt_owner_public_key, owner_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else { strcpy(reply, "Error: public key must be 64 hex characters (32 bytes)"); @@ -597,7 +665,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "mqtt.email ", 11) == 0) { if (valueTooLong(&config[11], sizeof(_mqtt_prefs.mqtt_email), reply, "email")) return true; StrHelper::strncpy(_mqtt_prefs.mqtt_email, &config[11], sizeof(_mqtt_prefs.mqtt_email)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); #endif } else if (memcmp(config, "alert ", 6) == 0) { @@ -605,12 +673,12 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf const char* val = &config[6]; if (memcmp(val, "on", 2) == 0 && (val[2] == 0 || val[2] == ' ')) { _mqtt_prefs.alert_enabled = 1; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alerts on"); } else if (memcmp(val, "off", 3) == 0 && (val[3] == 0 || val[3] == ' ')) { _mqtt_prefs.alert_enabled = 0; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alerts off"); } else { @@ -625,7 +693,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf if (len == 0) { _mqtt_prefs.alert_psk_hex[0] = '\0'; _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.psk cleared (alerts disabled until configured)"); } else if (val[0] == '#') { @@ -659,7 +727,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // The new PSK is operator-supplied, so any previously-derived // hashtag name is no longer accurate provenance — drop it. _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.psk updated"); } @@ -672,7 +740,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf if (in_len == 0) { _mqtt_prefs.alert_psk_hex[0] = '\0'; _mqtt_prefs.alert_hashtag[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.hashtag cleared (alerts disabled until configured)"); } else { @@ -708,7 +776,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf mesh::Utils::toHex(hex, digest, 16); StrHelper::strncpy(_mqtt_prefs.alert_hashtag, hashtag, sizeof(_mqtt_prefs.alert_hashtag)); StrHelper::strncpy(_mqtt_prefs.alert_psk_hex, hex, sizeof(_mqtt_prefs.alert_psk_hex)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); sprintf(reply, "OK - alert.hashtag: %s", _mqtt_prefs.alert_hashtag); } @@ -726,7 +794,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf size_t len = strlen(val); if (len == 0) { _mqtt_prefs.alert_region[0] = '\0'; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); strcpy(reply, "OK - alert.region cleared (using default scope)"); } else if (len >= sizeof(_mqtt_prefs.alert_region)) { @@ -734,7 +802,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else { StrHelper::strncpy(_mqtt_prefs.alert_region, val, sizeof(_mqtt_prefs.alert_region)); StrHelper::stripSurroundingQuotes(_mqtt_prefs.alert_region, sizeof(_mqtt_prefs.alert_region)); - savePrefs(); + if (!persistObserverPrefs(reply)) return true; _callbacks->onAlertConfigChanged(); sprintf(reply, "OK - alert.region: %s", _mqtt_prefs.alert_region); } @@ -744,7 +812,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.wifi must be 0-1440 minutes (0=off)"); } else { _mqtt_prefs.alert_wifi_minutes = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.wifi %d min%s", mins, mins == 0 ? " (disabled)" : ""); } } else if (memcmp(config, "alert.mqtt ", 11) == 0) { @@ -753,7 +821,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.mqtt must be 0-10080 minutes (0=off)"); } else { _mqtt_prefs.alert_mqtt_minutes = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.mqtt %d min%s", mins, mins == 0 ? " (disabled)" : ""); } } else if (memcmp(config, "alert.interval ", 15) == 0) { @@ -764,7 +832,7 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, "Error: alert.interval must be 60-10080 minutes"); } else { _mqtt_prefs.alert_min_interval_min = (uint16_t)mins; - savePrefs(); + if (!persistObserverPrefs(reply)) return true; sprintf(reply, "OK - alert.interval %d min", mins); } } else { diff --git a/src/helpers/ConfigSerializer.cpp b/src/helpers/ConfigSerializer.cpp index adff147f..b8788bf5 100644 --- a/src/helpers/ConfigSerializer.cpp +++ b/src/helpers/ConfigSerializer.cpp @@ -1,5 +1,9 @@ #include "ConfigSerializer.h" +#include +#include +#include + bool ConfigSerializer::saveSerial(Stream& s) { Context context(&s, OP::WRITE); _context = &context; // set the context for structure() call @@ -22,9 +26,12 @@ bool ConfigSerializer::saveSerial(Stream& s) { static bool is_whitespace(char c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; } -static bool is_key_char(char c) { +static bool is_key_start_char(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; } +static bool is_key_char(char c) { + return is_key_start_char(c) || (c >= '0' && c <= '9'); +} static bool is_value_char(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || c == '-' || c == '.'; } @@ -64,14 +71,26 @@ int ConfigSerializer::Context::readNext() { case EXPECT_KEY: if (rd_len > 0 && c == ':') { rd_buf[rd_len] = 0; rd_len = 0; rd_mode = EXPECT_VAL_OR_OBJ; return TOK_KEY; } if (rd_len == 0 && is_whitespace(c)) return TOK_WHITESPACE; - if (rd_len < CONFIG_MAX_KEYLEN-1 && is_key_char(c)) { rd_buf[rd_len++] = c; return TOK_WHITESPACE; } + if (rd_len < CONFIG_MAX_KEYLEN-1 && + ((rd_len == 0 && is_key_start_char(c)) || (rd_len > 0 && is_key_char(c)))) { + rd_buf[rd_len++] = c; + return TOK_WHITESPACE; + } return TOK_ERROR; case EXPECT_VAL_OR_OBJ: if (rd_len == 0 && is_whitespace(c)) return TOK_WHITESPACE; - if (rd_len == 0 && c == '"') { rd_mode = EXPECT_STRING_VAL; return TOK_WHITESPACE; } + if (rd_len == 0 && c == '"') { + rd_token_quoted = true; + rd_mode = EXPECT_STRING_VAL; + return TOK_WHITESPACE; + } if (rd_len == 0 && c == '{') { rd_mode = EXPECT_KEY; return TOK_START_OBJ; } - if (is_value_char(c) && rd_len < CONFIG_MAX_TOKEN_LEN-1) { rd_buf[rd_len++] = c; return TOK_WHITESPACE; } + if (is_value_char(c) && rd_len < CONFIG_MAX_TOKEN_LEN-1) { + if (rd_len == 0) rd_token_quoted = false; + rd_buf[rd_len++] = c; + return TOK_WHITESPACE; + } if (rd_len > 0 && (c == ',' || c == '}' || is_whitespace(c))) { pending = c; rd_buf[rd_len] = 0; rd_len = 0; rd_mode = EXPECT_COMMA_OR_CLOSE; return TOK_VALUE; } return TOK_ERROR; @@ -107,9 +126,19 @@ bool ConfigSerializer::loadSerial(Stream& s) { if (next_tok == TOK_KEY) { context.setKey(sp, context.getToken()); } else if (next_tok == TOK_VALUE) { + context.setValueEvent(sp, false); _depth = 1; // re-run the structure() hierarchy again (looking for specific key, at specific depth) structure(); } else if (next_tok == TOK_START_OBJ) { + // The root has no key. For every nested object, rerun the schema at the + // parent depth so strict scalar fields can reject an object value and + // object fields can confirm that their value has the expected shape. + if (sp > 0) { + context.setValueEvent(sp, true); + _depth = 1; + structure(); + if (!context.success) break; + } if (sp < CONFIG_MAX_DEPTH - 1) { sp++; } else { @@ -153,6 +182,10 @@ void ConfigSerializer::def(const char* key, void* value, size_t len) { _context->file()->print("\""); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } memset(value, 0, len); mesh::Utils::fromHex((uint8_t *)value, len, _context->getToken()); } @@ -181,6 +214,10 @@ void ConfigSerializer::def(const char* key, char* value, size_t max_len) { _context->file()->print("\""); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } strncpy(value, _context->getToken(), max_len - 1); value[max_len - 1] = 0; } @@ -195,6 +232,10 @@ void ConfigSerializer::def(const char* key, int32_t& value) { _context->file()->print(value); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -208,6 +249,10 @@ void ConfigSerializer::def(const char* key, uint32_t& value) { _context->file()->print(value); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -221,6 +266,10 @@ void ConfigSerializer::def(const char* key, int16_t& value) { _context->file()->print((int32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atol(_context->getToken()); } } @@ -234,6 +283,10 @@ void ConfigSerializer::def(const char* key, uint16_t& value) { _context->file()->print((uint32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -247,6 +300,10 @@ void ConfigSerializer::def(const char* key, uint8_t& value) { _context->file()->print((uint32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -260,6 +317,10 @@ void ConfigSerializer::def(const char* key, int8_t& value) { _context->file()->print((int32_t) value, 10); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atoi(_context->getToken()); } } @@ -273,6 +334,10 @@ void ConfigSerializer::def(const char* key, bool& value) { _context->file()->print(value ? "true" : "false"); } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = strcmp(_context->getToken(), "true") == 0 || atoi(_context->getToken()) != 0; // 'true' or a non-zero number } } @@ -290,6 +355,10 @@ void ConfigSerializer::def(const char* key, double& value) { } } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = atof(_context->getToken()); } } @@ -307,6 +376,10 @@ void ConfigSerializer::def(const char* key, float& value) { } } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() || _context->valueDepth() != _depth) { + _context->success = false; + return; + } value = (float) atof(_context->getToken()); } } @@ -323,9 +396,64 @@ void ConfigSerializer::def(const char* key, ConfigSerializer& sub_obj) { if (_context->file()->print("}") != 1) _context->success = false; // failure detect } else { if (_context->keyMatch(_depth, key)) { + if (_context->objectStart() && _context->valueDepth() == _depth) { + return; // the object itself; descendant values recurse below + } + if (_context->valueDepth() <= _depth) { + _context->success = false; // known object supplied as a scalar + return; + } sub_obj._context = _context; // inherit the Context sub_obj._depth = _depth + 1; sub_obj.structure(); // recurse into sub object } } } + +bool ConfigSerializer::defStrict(const char* key, char* value, size_t max_len, bool& seen) { + if (_context->op() == OP::WRITE) { + def(key, value, max_len); + return true; + } + if (!_context->keyMatch(_depth, key)) return false; + if (seen || max_len == 0 || _context->objectStart() || + _context->valueDepth() != _depth || !_context->tokenQuoted()) { + _context->success = false; + return false; + } + seen = true; + const char* token = _context->getToken(); + const size_t len = strlen(token); + if (len >= max_len) { + _context->success = false; + return false; + } + memcpy(value, token, len + 1); + return true; +} + +bool ConfigSerializer::defStrict(const char* key, int32_t& value, bool& seen) { + if (_context->op() == OP::WRITE) { + def(key, value); + return true; + } + if (!_context->keyMatch(_depth, key)) return false; + if (seen || _context->objectStart() || _context->valueDepth() != _depth || + _context->tokenQuoted()) { + _context->success = false; + return false; + } + seen = true; + + const char* token = _context->getToken(); + char* end = nullptr; + errno = 0; + const long long parsed = strtoll(token, &end, 10); + if (token[0] == '\0' || end == token || *end != '\0' || errno == ERANGE || + parsed < INT32_MIN || parsed > INT32_MAX) { + _context->success = false; + return false; + } + value = static_cast(parsed); + return true; +} diff --git a/src/helpers/ConfigSerializer.h b/src/helpers/ConfigSerializer.h index 7e6d6f2a..e889e637 100644 --- a/src/helpers/ConfigSerializer.h +++ b/src/helpers/ConfigSerializer.h @@ -25,17 +25,35 @@ class ConfigSerializer { OP _op; uint8_t rd_len; uint8_t rd_mode; + uint8_t rd_value_depth; char pending; + bool rd_token_quoted; + bool rd_object_start; char rd_buf[CONFIG_MAX_TOKEN_LEN]; char _keys[CONFIG_MAX_DEPTH][CONFIG_MAX_KEYLEN]; public: bool success = true; - Context(Stream* f, OP op) : _f(f), _op(op) { rd_buf[rd_len = 0] = 0; rd_mode = 0; pending = 0; } + Context(Stream* f, OP op) : _f(f), _op(op) { + rd_buf[rd_len = 0] = 0; + rd_mode = 0; + rd_value_depth = 0; + pending = 0; + rd_token_quoted = false; + rd_object_start = false; + memset(_keys, 0, sizeof(_keys)); + } OP op() const { return _op; } Stream* file() const { return _f; } int readNext(); const char* getToken() const { return rd_buf; } + bool tokenQuoted() const { return rd_token_quoted; } + uint8_t valueDepth() const { return rd_value_depth; } + bool objectStart() const { return rd_object_start; } + void setValueEvent(uint8_t depth, bool object_start) { + rd_value_depth = depth; + rd_object_start = object_start; + } bool keyMatch(int8_t depth, const char* key) { return strcmp(key, _keys[depth]) == 0; } void setKey(uint8_t depth, const char* key) { strcpy(_keys[depth], key); } }; @@ -60,6 +78,67 @@ protected: void def(const char* key, bool& value); void def(const char* key, ConfigSerializer& sub_obj); + // Strict read helpers for schemas whose values may be edited outside the + // firmware. Unlike the legacy def() overloads, these reject duplicate keys, + // truncated strings, malformed integers, and integer overflow. `seen` must + // be a field owned by the schema object and initialized false before load. + bool defStrict(const char* key, char* value, size_t max_len, bool& seen); + bool defStrict(const char* key, int32_t& value, bool& seen); + + // Literal keys are checked where a schema defines them, so adding a field + // that an older ConfigSerializer cannot tokenize is a build failure rather + // than a silent downgrade trap. Dynamic keys retain the pointer overloads. + template static void checkKey(const char (&)[N]) { + static_assert(N <= CONFIG_MAX_KEYLEN, + "ConfigSerializer key exceeds the visible-key limit"); + } + template void def(const char (&key)[N], char* value, size_t max_len) { + checkKey(key); def(static_cast(key), value, max_len); + } + template void def(const char (&key)[N], void* value, size_t len) { + checkKey(key); def(static_cast(key), value, len); + } + template void def(const char (&key)[N], int32_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], int16_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], int8_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint32_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint16_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], uint8_t& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], float& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], double& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], bool& value) { + checkKey(key); def(static_cast(key), value); + } + template void def(const char (&key)[N], ConfigSerializer& sub_obj) { + checkKey(key); def(static_cast(key), sub_obj); + } + template + bool defStrict(const char (&key)[N], char* value, size_t max_len, bool& seen) { + checkKey(key); + return defStrict(static_cast(key), value, max_len, seen); + } + template + bool defStrict(const char (&key)[N], int32_t& value, bool& seen) { + checkKey(key); + return defStrict(static_cast(key), value, seen); + } + virtual void structure() = 0; public: diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h index 5072e422..21c53bc4 100644 --- a/src/helpers/MQTTDefaults.h +++ b/src/helpers/MQTTDefaults.h @@ -8,7 +8,7 @@ #include "MQTTPacketFilter.h" #include "MQTTPresets.h" -// Compile-time defaults for fresh /mqtt_prefs (override via platformio build_flags). +// Compile-time defaults for fresh /mqtt.json (override via platformio build_flags). // Example: // -D MQTT_DEFAULT_SLOT1_PRESET='"meshcore-ca-1"' // -D MQTT_DEFAULT_IATA='"YYZ"' diff --git a/src/helpers/MQTTObserverValidation.h b/src/helpers/MQTTObserverValidation.h index 2042f226..a8d465a7 100644 --- a/src/helpers/MQTTObserverValidation.h +++ b/src/helpers/MQTTObserverValidation.h @@ -58,3 +58,20 @@ static inline bool mqttNtpHostnameValid(const char* host) { static inline bool mqttValueFits(const char* s, size_t bufsize) { return s != NULL && bufsize > 0 && strlen(s) < bufsize; } + +// Find a preset already assigned to another slot. The caller decides which +// names are singleton presets ("none" and "custom" intentionally are not). +template +static inline int mqttAssignedPresetSlot( + const char (&presets)[SlotCount][PresetSize], + const char* preset_name, + int ignored_slot) { + if (preset_name == NULL) return -1; + for (size_t slot = 0; slot < SlotCount; ++slot) { + if (static_cast(slot) != ignored_slot && + strcmp(presets[slot], preset_name) == 0) { + return static_cast(slot); + } + } + return -1; +} diff --git a/src/helpers/MQTTPacketFilter.h b/src/helpers/MQTTPacketFilter.h index df811828..47fbfe5c 100644 --- a/src/helpers/MQTTPacketFilter.h +++ b/src/helpers/MQTTPacketFilter.h @@ -71,8 +71,8 @@ inline bool resolveToken(const char* begin, size_t len, uint8_t* type_out) { } // Parse an allowlist value. Empty input means "all" so WebConfig can clear a -// field and retain the same backwards-compatible default as an older -// /mqtt_prefs file. Keywords and type names are deliberately lowercase; "all" +// field and retain the same all-packet-types default as an older preference +// file. Keywords and type names are deliberately lowercase; "all" // and "none" cannot be mixed into a list. Entries may be decimal (0..15) or // named, may carry surrounding ASCII whitespace, and may be repeated. inline bool parse(const char* input, uint16_t* mask_out) { @@ -186,8 +186,8 @@ inline uint16_t enabledUnion(const uint16_t* masks, const bool* enabled, size_t return combined; } -// True when every slot still carries the default all-types mask, i.e. nothing -// depends on the packet-filter tail of /mqtt_prefs being written. +// True when every slot still carries the default all-types mask. The legacy +// binary encoder uses this while producing migration fixtures. inline bool allMasksDefault(const uint16_t* masks, size_t count) { if (masks == nullptr) return true; for (size_t i = 0; i < count; ++i) { diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h index be77bf99..c3b3ce20 100644 --- a/src/helpers/MQTTPrefsAtomicStore.h +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -3,12 +3,51 @@ #include #include -// Transactional writer for /mqtt_prefs. The Store interface is intentionally -// narrow so host tests can exercise every failure boundary without an Arduino -// filesystem: begin(), write(), finish(), commit(), and abort(). The caller -// supplies header and payload separately, avoiding a second full-size buffer. +// Transactional writer policy for preferences. The Store interface is +// intentionally narrow so host tests can exercise every failure boundary +// without an Arduino filesystem. namespace MQTTPrefsAtomicStore { +// Production /mqtt.json flow. finish() verifies the exact bytes written, +// verify_image reparses the finished temp through the schema, and only then is +// commit() allowed to publish it. A schema-verification failure discards the +// finished temp; a commit failure preserves it for boot recovery. +enum class VerifiedImageResult : uint8_t { + Committed, + BeginFailed, + WriteFailed, + FinishFailed, + VerifyFailed, + CommitFailed, +}; + +template +inline VerifiedImageResult writeVerifiedImage(Store& store, + ImageWriter write_image, + ImageVerifier verify_image) { + if (!store.begin()) { + store.abort(); + return VerifiedImageResult::BeginFailed; + } + if (!write_image()) { + store.abort(); + return VerifiedImageResult::WriteFailed; + } + if (!store.finish()) { + store.abort(); + return VerifiedImageResult::FinishFailed; + } + if (!verify_image()) { + store.discardFinishedTemp(); + return VerifiedImageResult::VerifyFailed; + } + if (!store.commit()) { + store.abort(); + return VerifiedImageResult::CommitFailed; + } + return VerifiedImageResult::Committed; +} + enum class Result : uint8_t { Committed, BeginFailed, @@ -59,7 +98,7 @@ inline ImageResult writeImage(Store& store, ImageWriter write_image) { } // Coordinates a two-file legacy upgrade. /com_prefs must not be compacted -// until the observer tail it carries has been published into /mqtt_prefs. +// until the observer tail it carries has been published into /mqtt.json. // Keeping this state in a tiny pure helper lets host tests cover power-cut // boundaries without an Arduino filesystem. class LegacyUpgradeGate { diff --git a/src/helpers/MQTTPrefsCodec.h b/src/helpers/MQTTPrefsCodec.h index 7bbfe30a..5ac073c0 100644 --- a/src/helpers/MQTTPrefsCodec.h +++ b/src/helpers/MQTTPrefsCodec.h @@ -53,6 +53,9 @@ static const size_t kEncodedSize = sizeof(MQTTPrefsHeader) + kV1BaselinePayloadS // refuses to overwrite the file, so the node cannot be recovered over the air. // Touching any filter opts that node into the longer payload — a deliberate, // operator-initiated trade rather than a side effect of upgrading. +// The encoder below is retained for host migration fixtures and downgrade +// compatibility tests. Production firmware reads legacy binary files but only +// writes the JSON schema. inline size_t payloadLenFor(const MQTTPrefs& prefs) { return MQTTPacketFilter::allMasksDefault(prefs.mqtt_slot_packet_filter, MQTT_PREFS_SLOT_COUNT) @@ -68,13 +71,60 @@ inline MQTTPrefsHeader makeHeader(size_t payload_len) { return header; } +inline void freezeV1(const MQTTPrefs& prefs, LegacyV1MQTTPrefs* frozen) { + if (frozen == nullptr) return; + memset(frozen, 0, sizeof(*frozen)); + memcpy(frozen->mqtt_origin, prefs.mqtt_origin, sizeof(frozen->mqtt_origin)); + memcpy(frozen->mqtt_iata, prefs.mqtt_iata, sizeof(frozen->mqtt_iata)); + frozen->mqtt_status_enabled = prefs.mqtt_status_enabled; + frozen->mqtt_packets_enabled = prefs.mqtt_packets_enabled; + frozen->mqtt_raw_enabled = prefs.mqtt_raw_enabled; + frozen->mqtt_tx_enabled = prefs.mqtt_tx_enabled; + frozen->mqtt_status_interval = prefs.mqtt_status_interval; + memcpy(frozen->wifi_ssid, prefs.wifi_ssid, sizeof(frozen->wifi_ssid)); + memcpy(frozen->wifi_password, prefs.wifi_password, sizeof(frozen->wifi_password)); + frozen->wifi_power_save = prefs.wifi_power_save; + memcpy(frozen->timezone_string, prefs.timezone_string, sizeof(frozen->timezone_string)); + frozen->timezone_offset = prefs.timezone_offset; + memcpy(frozen->mqtt_slot_preset, prefs.mqtt_slot_preset, sizeof(frozen->mqtt_slot_preset)); + memcpy(frozen->mqtt_slot_host, prefs.mqtt_slot_host, sizeof(frozen->mqtt_slot_host)); + memcpy(frozen->mqtt_slot_port, prefs.mqtt_slot_port, sizeof(frozen->mqtt_slot_port)); + memcpy(frozen->mqtt_slot_username, prefs.mqtt_slot_username, sizeof(frozen->mqtt_slot_username)); + memcpy(frozen->mqtt_slot_password, prefs.mqtt_slot_password, sizeof(frozen->mqtt_slot_password)); + memcpy(frozen->mqtt_owner_public_key, prefs.mqtt_owner_public_key, + sizeof(frozen->mqtt_owner_public_key)); + memcpy(frozen->mqtt_email, prefs.mqtt_email, sizeof(frozen->mqtt_email)); + memcpy(frozen->mqtt_slot_token, prefs.mqtt_slot_token, sizeof(frozen->mqtt_slot_token)); + memcpy(frozen->mqtt_slot_topic, prefs.mqtt_slot_topic, sizeof(frozen->mqtt_slot_topic)); + memcpy(frozen->mqtt_slot_audience, prefs.mqtt_slot_audience, + sizeof(frozen->mqtt_slot_audience)); + frozen->mqtt_rx_enabled = prefs.mqtt_rx_enabled; + memcpy(frozen->mqtt_ntp_server, prefs.mqtt_ntp_server, sizeof(frozen->mqtt_ntp_server)); + frozen->snmp_enabled = prefs.snmp_enabled; + memcpy(frozen->snmp_community, prefs.snmp_community, sizeof(frozen->snmp_community)); + frozen->radio_watchdog_minutes = prefs.radio_watchdog_minutes; + frozen->alert_enabled = prefs.alert_enabled; + memcpy(frozen->alert_psk_hex, prefs.alert_psk_hex, sizeof(frozen->alert_psk_hex)); + frozen->alert_wifi_minutes = prefs.alert_wifi_minutes; + frozen->alert_mqtt_minutes = prefs.alert_mqtt_minutes; + frozen->alert_min_interval_min = prefs.alert_min_interval_min; + memcpy(frozen->alert_hashtag, prefs.alert_hashtag, sizeof(frozen->alert_hashtag)); + memcpy(frozen->alert_region, prefs.alert_region, sizeof(frozen->alert_region)); + frozen->mqtt_neighbors_enabled = prefs.mqtt_neighbors_enabled; + frozen->mqtt_neighbors_interval = prefs.mqtt_neighbors_interval; + memcpy(frozen->mqtt_slot_packet_filter, prefs.mqtt_slot_packet_filter, + sizeof(frozen->mqtt_slot_packet_filter)); +} + inline size_t encode(const MQTTPrefs& prefs, uint8_t* output, size_t output_size) { const size_t payload_len = payloadLenFor(prefs); const size_t encoded_size = sizeof(MQTTPrefsHeader) + payload_len; if (output == nullptr || output_size < encoded_size) return 0; const MQTTPrefsHeader header = makeHeader(payload_len); + LegacyV1MQTTPrefs frozen; + freezeV1(prefs, &frozen); memcpy(output, &header, sizeof(header)); - memcpy(output + sizeof(header), &prefs, payload_len); + memcpy(output + sizeof(header), &frozen, payload_len); return encoded_size; } @@ -346,6 +396,120 @@ inline bool isPlausibleLegacy(Source source, const uint8_t* input, size_t size) } } +// A valid v1 header proves the intended layout, not that a torn or corrupted +// payload still contains bounded C strings. Validate every field present at a +// shipped v1 boundary before copying it into the runtime object; otherwise the +// later semantic validators and JSON writer could scan beyond a fixed array. +inline bool isPlausibleV1(const LegacyV1MQTTPrefs& prefs, size_t payload_len) { + if (payload_len != kV1PreObserverPayloadSize && + payload_len != kV1PreNeighborsPayloadSize && + payload_len != kV1PreFilterPayloadSize && + payload_len != kV1BaselinePayloadSize) { + return false; + } + const uint8_t* input = reinterpret_cast(&prefs); + // A v1 header already identifies the layout. At this stage only reject + // fields that could make later C-string validation/serialization read past a + // fixed array; numeric values are safely repaired by MQTTPrefsSerializer. + if (!hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_origin), sizeof(prefs.mqtt_origin)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_iata), sizeof(prefs.mqtt_iata)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, wifi_ssid), sizeof(prefs.wifi_ssid)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, wifi_password), sizeof(prefs.wifi_password)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, timezone_string), sizeof(prefs.timezone_string)) || + !hasPlausibleSharedAuth(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_owner_public_key), + offsetof(LegacyV1MQTTPrefs, mqtt_email)) || + !hasPlausibleSlotText(input, payload_len, MQTT_PREFS_SLOT_COUNT, + offsetof(LegacyV1MQTTPrefs, mqtt_slot_preset), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_host), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_username), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_password), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_token), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_topic), + offsetof(LegacyV1MQTTPrefs, mqtt_slot_audience)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, mqtt_ntp_server), + sizeof(prefs.mqtt_ntp_server))) { + return false; + } + if (payload_len >= kV1PreNeighborsPayloadSize) { + if (!hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, snmp_community), + sizeof(prefs.snmp_community)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_psk_hex), + sizeof(prefs.alert_psk_hex)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_hashtag), + sizeof(prefs.alert_hashtag)) || + !hasTerminatedText(input, payload_len, + offsetof(LegacyV1MQTTPrefs, alert_region), + sizeof(prefs.alert_region))) { + return false; + } + } + return true; +} + +inline void migrateV1(const LegacyV1MQTTPrefs& old_prefs, size_t payload_len, + MQTTPrefs* prefs) { + if (prefs == nullptr || payload_len < kV1PreObserverPayloadSize) return; + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_slot_preset, old_prefs.mqtt_slot_preset, sizeof(prefs->mqtt_slot_preset)); + memcpy(prefs->mqtt_slot_host, old_prefs.mqtt_slot_host, sizeof(prefs->mqtt_slot_host)); + memcpy(prefs->mqtt_slot_port, old_prefs.mqtt_slot_port, sizeof(prefs->mqtt_slot_port)); + memcpy(prefs->mqtt_slot_username, old_prefs.mqtt_slot_username, sizeof(prefs->mqtt_slot_username)); + memcpy(prefs->mqtt_slot_password, old_prefs.mqtt_slot_password, sizeof(prefs->mqtt_slot_password)); + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + memcpy(prefs->mqtt_slot_token, old_prefs.mqtt_slot_token, sizeof(prefs->mqtt_slot_token)); + memcpy(prefs->mqtt_slot_topic, old_prefs.mqtt_slot_topic, sizeof(prefs->mqtt_slot_topic)); + memcpy(prefs->mqtt_slot_audience, old_prefs.mqtt_slot_audience, + sizeof(prefs->mqtt_slot_audience)); + prefs->mqtt_rx_enabled = old_prefs.mqtt_rx_enabled; + memcpy(prefs->mqtt_ntp_server, old_prefs.mqtt_ntp_server, sizeof(prefs->mqtt_ntp_server)); + + if (payload_len >= kV1PreNeighborsPayloadSize) { + prefs->snmp_enabled = old_prefs.snmp_enabled; + memcpy(prefs->snmp_community, old_prefs.snmp_community, sizeof(prefs->snmp_community)); + prefs->radio_watchdog_minutes = old_prefs.radio_watchdog_minutes; + prefs->alert_enabled = old_prefs.alert_enabled; + memcpy(prefs->alert_psk_hex, old_prefs.alert_psk_hex, sizeof(prefs->alert_psk_hex)); + prefs->alert_wifi_minutes = old_prefs.alert_wifi_minutes; + prefs->alert_mqtt_minutes = old_prefs.alert_mqtt_minutes; + prefs->alert_min_interval_min = old_prefs.alert_min_interval_min; + memcpy(prefs->alert_hashtag, old_prefs.alert_hashtag, sizeof(prefs->alert_hashtag)); + memcpy(prefs->alert_region, old_prefs.alert_region, sizeof(prefs->alert_region)); + // The pre-neighbors payload includes the old zero padding byte at this + // offset; copying it preserves the deployed format's disabled default. + prefs->mqtt_neighbors_enabled = old_prefs.mqtt_neighbors_enabled; + } + if (payload_len >= kV1PreFilterPayloadSize) { + prefs->mqtt_neighbors_interval = old_prefs.mqtt_neighbors_interval; + } + if (payload_len >= kV1BaselinePayloadSize) { + memcpy(prefs->mqtt_slot_packet_filter, old_prefs.mqtt_slot_packet_filter, + sizeof(prefs->mqtt_slot_packet_filter)); + } +} + inline void migratePreSlot(const OldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); diff --git a/src/helpers/MQTTPrefsRecovery.h b/src/helpers/MQTTPrefsRecovery.h index 7534678e..37073209 100644 --- a/src/helpers/MQTTPrefsRecovery.h +++ b/src/helpers/MQTTPrefsRecovery.h @@ -2,23 +2,29 @@ #include -// Pure recovery policy for the three MQTT preference transaction files. The +// Pure recovery policy for the three MQTT preference transaction files. The // writer first moves the old primary to .bak, then moves the verified .tmp to -// the primary name. On a reset, the loader uses this policy before decoding -// /mqtt_prefs. "Preserve" is deliberately distinct from "Usable": it covers -// an unsupported newer layout, corruption, or an unreadable file and must -// never be replaced by an older image. +// the primary name. On a reset, the loader uses this policy before decoding +// the primary. FutureUsable is syntactically valid but belongs to newer +// firmware. FutureClaimed and Indeterminate cannot be safely classified by +// this firmware, so recovery may use a known-good backup but must retain the +// uncertain image and hold further writes. Preserve is definitively invalid or +// unsupported and may be discarded only where the transaction policy permits. namespace MQTTPrefsRecovery { enum class FileState : uint8_t { Missing, Usable, + FutureUsable, + FutureClaimed, + Indeterminate, Preserve, }; enum class Action : uint8_t { None, KeepPrimary, + DiscardTemp, PromoteTemp, PromoteBackup, }; @@ -30,13 +36,23 @@ inline Action select(FileState primary, FileState temp, FileState backup) { if (primary != FileState::Missing) return Action::KeepPrimary; // A completed temp is the new image and wins over the old backup. - if (temp == FileState::Usable) return Action::PromoteTemp; + if (temp == FileState::Usable || temp == FileState::FutureUsable) { + return Action::PromoteTemp; + } - // If temp is opaque but a known-good backup exists, boot from the backup. - // The caller may discard the opaque temp once that usable backup has become - // primary. Otherwise, rename the opaque temp into the empty primary name so - // the normal loader can hold it. + // Preserve is definitively invalid, so it was never a committed image. If + // no backup exists, discard the interrupted temp and leave the primary name + // absent; this is what lets a first JSON migration retry from /mqtt_prefs. + // If a backup exists, it is the prior committed image and wins regardless + // of whether this firmware understands its schema. if (temp == FileState::Preserve) { + return backup == FileState::Missing ? Action::DiscardTemp : Action::PromoteBackup; + } + + // FutureClaimed and Indeterminate may be a completed image this firmware + // cannot classify. A known-good backup can run this boot, but without one + // preserve the only candidate under the authoritative name and hold writes. + if (temp == FileState::FutureClaimed || temp == FileState::Indeterminate) { return backup == FileState::Usable ? Action::PromoteBackup : Action::PromoteTemp; } @@ -46,4 +62,8 @@ inline Action select(FileState primary, FileState temp, FileState backup) { return Action::None; } +inline bool uncertain(FileState state) { + return state == FileState::FutureClaimed || state == FileState::Indeterminate; +} + } // namespace MQTTPrefsRecovery diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h new file mode 100644 index 00000000..b4d12b18 --- /dev/null +++ b/src/helpers/MQTTPrefsSerializer.h @@ -0,0 +1,412 @@ +#pragma once + +#include +#include +#include +#include + +#ifdef WITH_MQTT_BRIDGE + +static const int32_t MQTT_PREFS_JSON_FORMAT_VERSION = 1; + +// Reads only the mandatory root version while ignoring every other schema +// field. Recovery uses this before the v1 decoder so a syntactically valid +// future file remains opaque even when that schema changes a v1 field's type. +class MQTTPrefsVersionProbe : public ConfigSerializer { + int32_t _version = 0; + bool _seen_version = false; +protected: + void structure() override { defStrict("version", _version, _seen_version); } +public: + bool hasFutureVersion() const { + return _seen_version && _version > MQTT_PREFS_JSON_FORMAT_VERSION; + } +}; + +// ConfigSerializer adapter for the observer preference POD. Keeping the +// serializer separate avoids making the runtime object layout part of the JSON +// format and adds no permanent per-slot serializer state to CommonCLI. +class MQTTPrefsSerializer : public ConfigSerializer { + + class WifiPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _power_save; + bool _seen_ssid = false, _seen_password = false, _seen_power_save = false; + protected: + void structure() override { + defStrict("ssid", _prefs->wifi_ssid, sizeof(_prefs->wifi_ssid), _seen_ssid); + defStrict("password", _prefs->wifi_password, sizeof(_prefs->wifi_password), _seen_password); + defStrict("power_save", _power_save, _seen_power_save); + } + public: + explicit WifiPrefs(MQTTPrefs* prefs) : _prefs(prefs), _power_save(prefs->wifi_power_save) {} + bool apply(bool* repaired) { + if (_power_save < 0 || _power_save > 2) { + _power_save = 1; + *repaired = true; + } + _prefs->wifi_power_save = static_cast(_power_save); + return true; + } + }; + + class TimePrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _utc_offset, _default_utc_offset; + char _default_ntp_server[sizeof(MQTTPrefs::mqtt_ntp_server)]; + bool _seen_timezone = false, _seen_utc_offset = false, _seen_ntp_server = false; + protected: + void structure() override { + defStrict("timezone", _prefs->timezone_string, sizeof(_prefs->timezone_string), _seen_timezone); + defStrict("utc_offset", _utc_offset, _seen_utc_offset); + defStrict("ntp_server", _prefs->mqtt_ntp_server, sizeof(_prefs->mqtt_ntp_server), _seen_ntp_server); + } + public: + TimePrefs(MQTTPrefs* prefs, const MQTTPrefs* defaults) + : _prefs(prefs), _utc_offset(prefs->timezone_offset), + _default_utc_offset(defaults->timezone_offset) { + if (_default_utc_offset < -12 || _default_utc_offset > 14) { + _default_utc_offset = 0; + } + memcpy(_default_ntp_server, defaults->mqtt_ntp_server, sizeof(_default_ntp_server)); + if (_default_ntp_server[0] != '\0' && + !mqttNtpHostnameValid(_default_ntp_server)) { + _default_ntp_server[0] = '\0'; + } + } + bool apply(bool* repaired) { + if (_utc_offset < -12 || _utc_offset > 14) { + _utc_offset = _default_utc_offset; + *repaired = true; + } + if (_prefs->mqtt_ntp_server[0] != '\0' && + !mqttNtpHostnameValid(_prefs->mqtt_ntp_server)) { + memcpy(_prefs->mqtt_ntp_server, _default_ntp_server, sizeof(_default_ntp_server)); + *repaired = true; + } + _prefs->timezone_offset = static_cast(_utc_offset); + return true; + } + }; + + class StatusPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _interval_ms; + bool _seen_enabled = false, _seen_interval = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("interval_ms", _interval_ms, _seen_interval); + } + public: + explicit StatusPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->mqtt_status_enabled), + _interval_ms(static_cast(prefs->mqtt_status_interval)) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 1; *repaired = true; } + if (_interval_ms < 60000 || _interval_ms > 3600000) { + _interval_ms = 300000; + *repaired = true; + } + _prefs->mqtt_status_enabled = static_cast(_enabled); + _prefs->mqtt_status_interval = static_cast(_interval_ms); + } + }; + + class NeighborPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _interval_ms; + bool _seen_enabled = false, _seen_interval = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("interval_ms", _interval_ms, _seen_interval); + } + public: + explicit NeighborPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->mqtt_neighbors_enabled), + _interval_ms(static_cast(prefs->mqtt_neighbors_interval)) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + if (_interval_ms < static_cast(MQTT_NEIGHBORS_MIN_INTERVAL_MS) || + _interval_ms > static_cast(MQTT_NEIGHBORS_MAX_INTERVAL_MS)) { + _interval_ms = static_cast(MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS); + *repaired = true; + } + _prefs->mqtt_neighbors_enabled = static_cast(_enabled); + _prefs->mqtt_neighbors_interval = static_cast(_interval_ms); + } + }; + + class OwnerPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + bool _seen_public_key = false, _seen_email = false; + protected: + void structure() override { + defStrict("public_key", _prefs->mqtt_owner_public_key, + sizeof(_prefs->mqtt_owner_public_key), _seen_public_key); + defStrict("email", _prefs->mqtt_email, sizeof(_prefs->mqtt_email), _seen_email); + } + public: + explicit OwnerPrefs(MQTTPrefs* prefs) : _prefs(prefs) {} + void apply(bool* repaired) { + if (_prefs->mqtt_owner_public_key[0] != '\0' && + !mqttOwnerKeyValid(_prefs->mqtt_owner_public_key)) { + _prefs->mqtt_owner_public_key[0] = '\0'; + *repaired = true; + } + } + }; + + class SlotPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int _index; + int32_t _port, _filter; + bool _seen_preset = false, _seen_host = false, _seen_port = false; + bool _seen_username = false, _seen_password = false, _seen_token = false; + bool _seen_topic = false, _seen_audience = false, _seen_filter = false; + protected: + void structure() override { + defStrict("preset", _prefs->mqtt_slot_preset[_index], + sizeof(_prefs->mqtt_slot_preset[_index]), _seen_preset); + defStrict("host", _prefs->mqtt_slot_host[_index], + sizeof(_prefs->mqtt_slot_host[_index]), _seen_host); + defStrict("port", _port, _seen_port); + defStrict("username", _prefs->mqtt_slot_username[_index], + sizeof(_prefs->mqtt_slot_username[_index]), _seen_username); + defStrict("password", _prefs->mqtt_slot_password[_index], + sizeof(_prefs->mqtt_slot_password[_index]), _seen_password); + defStrict("token", _prefs->mqtt_slot_token[_index], + sizeof(_prefs->mqtt_slot_token[_index]), _seen_token); + defStrict("topic", _prefs->mqtt_slot_topic[_index], + sizeof(_prefs->mqtt_slot_topic[_index]), _seen_topic); + defStrict("audience", _prefs->mqtt_slot_audience[_index], + sizeof(_prefs->mqtt_slot_audience[_index]), _seen_audience); + defStrict("packet_filter", _filter, _seen_filter); + } + public: + SlotPrefs(MQTTPrefs* prefs, int index) + : _prefs(prefs), _index(index), _port(prefs->mqtt_slot_port[index]), + _filter(prefs->mqtt_slot_packet_filter[index]) {} + void apply(bool* repaired) { + if (_port < 0 || _port > 65535) { _port = 0; *repaired = true; } + if (_filter < 0 || _filter > 65535) { _filter = 0xffff; *repaired = true; } + const char* preset = _prefs->mqtt_slot_preset[_index]; + if (strcmp(preset, MQTT_PRESET_NONE) != 0 && + strcmp(preset, MQTT_PRESET_CUSTOM) != 0 && findMQTTPreset(preset) == nullptr) { + strcpy(_prefs->mqtt_slot_preset[_index], MQTT_PRESET_NONE); + *repaired = true; + } + _prefs->mqtt_slot_port[_index] = static_cast(_port); + _prefs->mqtt_slot_packet_filter[_index] = static_cast(_filter); + } + }; + + class MqttPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _packets_enabled, _raw_enabled, _tx_enabled, _rx_enabled; + char _default_iata[sizeof(MQTTPrefs::mqtt_iata)]; + bool _seen_origin = false, _seen_iata = false, _seen_packets = false; + bool _seen_raw = false, _seen_tx = false, _seen_rx = false; + StatusPrefs _status; + NeighborPrefs _neighbors; + OwnerPrefs _owner; + static_assert(MQTT_PREFS_SLOT_COUNT == 6, + "MQTTPrefsSerializer slot members and keys must be updated"); + SlotPrefs _slot1, _slot2, _slot3, _slot4, _slot5, _slot6; + protected: + void structure() override { + defStrict("origin", _prefs->mqtt_origin, sizeof(_prefs->mqtt_origin), _seen_origin); + defStrict("iata", _prefs->mqtt_iata, sizeof(_prefs->mqtt_iata), _seen_iata); + defStrict("packets_enabled", _packets_enabled, _seen_packets); + defStrict("raw_enabled", _raw_enabled, _seen_raw); + defStrict("tx_enabled", _tx_enabled, _seen_tx); + defStrict("rx_enabled", _rx_enabled, _seen_rx); + def("status", _status); + def("neighbors", _neighbors); + def("owner", _owner); + def("slot1", _slot1); + def("slot2", _slot2); + def("slot3", _slot3); + def("slot4", _slot4); + def("slot5", _slot5); + def("slot6", _slot6); + } + public: + MqttPrefs(MQTTPrefs* prefs, const MQTTPrefs* defaults) + : _prefs(prefs), _packets_enabled(prefs->mqtt_packets_enabled), + _raw_enabled(prefs->mqtt_raw_enabled), _tx_enabled(prefs->mqtt_tx_enabled), + _rx_enabled(prefs->mqtt_rx_enabled), _status(prefs), _neighbors(prefs), + _owner(prefs), _slot1(prefs, 0), _slot2(prefs, 1), + _slot3(prefs, 2), _slot4(prefs, 3), + _slot5(prefs, 4), _slot6(prefs, 5) { + memcpy(_default_iata, defaults->mqtt_iata, sizeof(_default_iata)); + if (_default_iata[0] != '\0' && !mqttIataValid(_default_iata)) { + _default_iata[0] = '\0'; + } + for (char* p = _default_iata; *p; ++p) { + if (*p >= 'a' && *p <= 'z') *p = static_cast(*p - ('a' - 'A')); + } + } + void apply(bool* repaired) { + if (_packets_enabled < 0 || _packets_enabled > 1) { _packets_enabled = 1; *repaired = true; } + if (_raw_enabled < 0 || _raw_enabled > 1) { _raw_enabled = 0; *repaired = true; } + if (_tx_enabled < 0 || _tx_enabled > 2) { _tx_enabled = 2; *repaired = true; } + if (_rx_enabled < 0 || _rx_enabled > 1) { _rx_enabled = 1; *repaired = true; } + _prefs->mqtt_packets_enabled = static_cast(_packets_enabled); + _prefs->mqtt_raw_enabled = static_cast(_raw_enabled); + _prefs->mqtt_tx_enabled = static_cast(_tx_enabled); + _prefs->mqtt_rx_enabled = static_cast(_rx_enabled); + if (_prefs->mqtt_iata[0] != '\0') { + if (!mqttIataValid(_prefs->mqtt_iata)) { + memcpy(_prefs->mqtt_iata, _default_iata, sizeof(_default_iata)); + *repaired = true; + } else { + for (char* p = _prefs->mqtt_iata; *p; ++p) { + if (*p >= 'a' && *p <= 'z') { + *p = static_cast(*p - ('a' - 'A')); + *repaired = true; + } + } + } + } + _status.apply(repaired); + _neighbors.apply(repaired); + _owner.apply(repaired); + _slot1.apply(repaired); _slot2.apply(repaired); _slot3.apply(repaired); + _slot4.apply(repaired); _slot5.apply(repaired); _slot6.apply(repaired); + } + }; + + class SnmpPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled; + bool _seen_enabled = false, _seen_community = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("community", _prefs->snmp_community, + sizeof(_prefs->snmp_community), _seen_community); + } + public: + explicit SnmpPrefs(MQTTPrefs* prefs) : _prefs(prefs), _enabled(prefs->snmp_enabled) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + _prefs->snmp_enabled = static_cast(_enabled); + } + }; + + class RadioPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _watchdog_min; + bool _seen_watchdog = false; + protected: + void structure() override { defStrict("watchdog_min", _watchdog_min, _seen_watchdog); } + public: + explicit RadioPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _watchdog_min(prefs->radio_watchdog_minutes) {} + void apply(bool* repaired) { + if (_watchdog_min < 0 || _watchdog_min > 120) { _watchdog_min = 5; *repaired = true; } + _prefs->radio_watchdog_minutes = static_cast(_watchdog_min); + } + }; + + class AlertPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _enabled, _wifi_minutes, _mqtt_minutes, _rate_limit_min; + bool _seen_enabled = false, _seen_psk = false, _seen_wifi = false; + bool _seen_mqtt = false, _seen_rate = false, _seen_hashtag = false, _seen_region = false; + protected: + void structure() override { + defStrict("enabled", _enabled, _seen_enabled); + defStrict("psk_hex", _prefs->alert_psk_hex, sizeof(_prefs->alert_psk_hex), _seen_psk); + defStrict("wifi_minutes", _wifi_minutes, _seen_wifi); + defStrict("mqtt_minutes", _mqtt_minutes, _seen_mqtt); + defStrict("rate_limit_min", _rate_limit_min, _seen_rate); + defStrict("hashtag", _prefs->alert_hashtag, sizeof(_prefs->alert_hashtag), _seen_hashtag); + defStrict("region", _prefs->alert_region, sizeof(_prefs->alert_region), _seen_region); + } + public: + explicit AlertPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _enabled(prefs->alert_enabled), + _wifi_minutes(prefs->alert_wifi_minutes), _mqtt_minutes(prefs->alert_mqtt_minutes), + _rate_limit_min(prefs->alert_min_interval_min) {} + void apply(bool* repaired) { + if (_enabled < 0 || _enabled > 1) { _enabled = 0; *repaired = true; } + if (_wifi_minutes < 0 || _wifi_minutes > 1440) { _wifi_minutes = 30; *repaired = true; } + if (_mqtt_minutes < 0 || _mqtt_minutes > 10080) { _mqtt_minutes = 240; *repaired = true; } + if (_rate_limit_min < 60 || _rate_limit_min > 10080) { _rate_limit_min = 60; *repaired = true; } + if (_prefs->alert_psk_hex[0] != '\0') { + const size_t len = strlen(_prefs->alert_psk_hex); + bool valid_hex = len == 32; + for (size_t i = 0; valid_hex && i < len; ++i) { + const char c = _prefs->alert_psk_hex[i]; + valid_hex = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || + (c >= 'a' && c <= 'f'); + } + if (!valid_hex) { + _prefs->alert_psk_hex[0] = '\0'; + _prefs->alert_hashtag[0] = '\0'; + *repaired = true; + } + } + _prefs->alert_enabled = static_cast(_enabled); + _prefs->alert_wifi_minutes = static_cast(_wifi_minutes); + _prefs->alert_mqtt_minutes = static_cast(_mqtt_minutes); + _prefs->alert_min_interval_min = static_cast(_rate_limit_min); + } + }; + + MQTTPrefs* _prefs; + int32_t _version = MQTT_PREFS_JSON_FORMAT_VERSION; + bool _seen_version = false; + WifiPrefs _wifi; + TimePrefs _time; + MqttPrefs _mqtt; + SnmpPrefs _snmp; + RadioPrefs _radio; + AlertPrefs _alert; + +protected: + void structure() override { + defStrict("version", _version, _seen_version); + def("wifi", _wifi); + def("time", _time); + def("mqtt", _mqtt); + def("snmp", _snmp); + def("radio", _radio); + def("alert", _alert); + } + +public: + explicit MQTTPrefsSerializer(MQTTPrefs* prefs, const MQTTPrefs* repair_defaults = nullptr) + : _prefs(prefs), _wifi(prefs), + _time(prefs, repair_defaults ? repair_defaults : prefs), + _mqtt(prefs, repair_defaults ? repair_defaults : prefs), _snmp(prefs), + _radio(prefs), _alert(prefs) {} + + bool hasSupportedVersion() const { + return _seen_version && _version == MQTT_PREFS_JSON_FORMAT_VERSION; + } + bool hasFutureVersion() const { + return _seen_version && _version > MQTT_PREFS_JSON_FORMAT_VERSION; + } + + bool normalize(bool* repaired) { + if (repaired == nullptr) return false; + *repaired = false; + _wifi.apply(repaired); + _time.apply(repaired); + _mqtt.apply(repaired); + _snmp.apply(repaired); + _radio.apply(repaired); + _alert.apply(repaired); + return true; + } + + bool apply(bool* repaired) { + return hasSupportedVersion() && normalize(repaired); + } +}; + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h index e3c6a098..0bc0867e 100644 --- a/src/helpers/MQTTPrefsStorage.h +++ b/src/helpers/MQTTPrefsStorage.h @@ -62,10 +62,9 @@ struct PreWifiPowerOldMQTTPrefs { char mqtt_email[64]; }; -// MQTT preferences stored separately from NodePrefs to avoid upstream layout -// conflicts. The full layout is the frozen v1 payload baseline. The prefix -// before observer settings is also an explicitly supported v1 payload: it was -// used before the observer fields were appended. +// Runtime MQTT preferences, stored separately from NodePrefs to avoid upstream +// layout conflicts. JSON persistence is field-based; this type is deliberately +// free to evolve independently of the frozen binary migration layouts below. struct MQTTPrefs { char mqtt_origin[32]; char mqtt_iata[8]; @@ -110,17 +109,57 @@ struct MQTTPrefs { char alert_region[31]; // Neighbors publishing (PSRAM boards only). Appended at the end of the - // observer tail so a shorter (pre-neighbors) /mqtt_prefs payload from earlier - // firmware still loads with these defaulting off/24h; keeps the format at - // VERSION 1. Field order and sizes are kept byte-identical to the flex - // neighbors build so a /mqtt_prefs written by either firmware is - // interchangeable (see the offsetof static_asserts below). + // observer tail in the former binary format. Binary compatibility is now + // represented only by LegacyV1MQTTPrefs below. uint8_t mqtt_neighbors_enabled; uint32_t mqtt_neighbors_interval; // Per-slot payload-type allow masks. Bit N controls MeshCore packet type N - // for both packets and raw MQTT topics. Appended so older v1 payloads load - // with the default all-types masks intact. + // for both packets and raw MQTT topics. + uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; +}; + +// Frozen payload written by the version-1 binary format. Keep this distinct +// from the runtime MQTTPrefs type: JSON persistence must not make the runtime +// object's padding or member order an on-flash ABI again. Legacy decoding reads +// into this type and field-copies into a defaulted runtime object. +struct LegacyV1MQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; + uint8_t snmp_enabled; + char snmp_community[24]; + uint8_t radio_watchdog_minutes; + uint8_t alert_enabled; + char alert_psk_hex[33]; + uint16_t alert_wifi_minutes; + uint16_t alert_mqtt_minutes; + uint16_t alert_min_interval_min; + char alert_hashtag[24]; + char alert_region[31]; + uint8_t mqtt_neighbors_enabled; + uint32_t mqtt_neighbors_interval; uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; }; @@ -272,20 +311,20 @@ static const size_t LEGACY6_AUDIENCE_RX_SIZE = 2840; // Frozen on-flash layouts; every firmware and native fixture build checks them. static_assert(sizeof(MQTTPrefsHeader) == 8, "versioned /mqtt_prefs header must stay 8 bytes"); -static_assert(offsetof(MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, "v1 pre-observer /mqtt_prefs boundary changed"); -static_assert(sizeof(MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, +static_assert(sizeof(LegacyV1MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, "v1 /mqtt_prefs payload layout changed"); // Lock the neighbors tail to the flex neighbors build's layout so a /mqtt_prefs // written by either firmware is byte-for-byte interchangeable. The enable flag // lands in the old struct's zeroed trailing padding (offset 2857), and the // interval begins exactly at the pre-neighbors payload size (2860) so a // pre-neighbors read stops right before it and the interval keeps its default. -static_assert(offsetof(MQTTPrefs, mqtt_neighbors_enabled) == 2857, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_neighbors_enabled) == 2857, "neighbors enable flag must sit at the flex-compatible offset"); -static_assert(offsetof(MQTTPrefs, mqtt_neighbors_interval) == MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_neighbors_interval) == MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE, "neighbors interval offset must equal the pre-neighbors payload size"); -static_assert(offsetof(MQTTPrefs, mqtt_slot_packet_filter) == MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE, +static_assert(offsetof(LegacyV1MQTTPrefs, mqtt_slot_packet_filter) == MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE, "packet filters must begin at the pre-filter payload boundary"); static_assert(sizeof(OldMQTTPrefs) == 472, "frozen pre-slot /mqtt_prefs layout changed"); static_assert(sizeof(PreWifiPowerOldMQTTPrefs) == 472, "frozen pre-WiFi-power /mqtt_prefs layout changed"); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index eb7ab943..43af42b2 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -509,7 +509,7 @@ private: LifecycleOps _lifecycle_ops; MQTTLifecycle::Coordinator _lifecycle; - // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt_prefs. + // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // _prefs (held by BridgeBase) still provides upstream fields (freq/sf/node_name…). MQTTPrefs* _obs = nullptr; diff --git a/test/README.md b/test/README.md index 711e6484..80182510 100644 --- a/test/README.md +++ b/test/README.md @@ -33,7 +33,8 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_packet_filter` | `src/helpers/MQTTPacketFilter.h` | per-slot 0-15 allowlist parsing/formatting, numeric and named spellings; exact bounds; membership; candidate/eligible split and retry-completion policy; pre-queue union gate; default-mask detection | | `test_mqtt_runtime_buffer_lifecycle` | `src/helpers/MQTTRuntimeBufferLifecycle.h` | idempotent allocation/release; partial-allocation degradation; retry of only missing buffers | | `test_mqtt_prefs_codec` | `src/helpers/MQTTPrefsStorage.h`, `src/helpers/MQTTPrefsCodec.h` | binary pre-slot/3-slot/6-slot migration fixtures; v1 header integrity; downgrade preservation; shortest-payload write policy (default filters stay downgrade-readable) | -| `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h` | transactional MQTT writes and legacy `/node_prefs` handoff; exact short-write detection; begin/finish/rename failure cleanup; original-file preservation | +| `test_mqtt_prefs_serializer` | `src/helpers/MQTTPrefsSerializer.h`, `src/helpers/ConfigSerializer.*` | semantic nested `/mqtt.json` round trips; numeric slot keys; required/future version handling; strict length/overflow/duplicate rejection; safe semantic repair; scratch-before-live loading | +| `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h`, `src/helpers/MQTTPrefsRecovery.h` | production JSON begin/write/checksum-finish/schema-verify/commit orchestration; first-migration and rename-boundary recovery; legacy `/node_prefs` handoff; failure cleanup and original-file preservation | | `test_mqtt_payload_builder` | `src/helpers/MQTTPayloadBuilder.cpp` | status/packet/raw JSON contracts; optional fields; escaping; RX metrics and path; score handling; exact buffer bounds; maximum representative payloads | | `test_utils` | `src/Utils.cpp` | `Utils::toHex` (upstream) | diff --git a/test/test_config_serializer/test_config_serializer.cpp b/test/test_config_serializer/test_config_serializer.cpp index 27c3c811..90154047 100644 --- a/test/test_config_serializer/test_config_serializer.cpp +++ b/test/test_config_serializer/test_config_serializer.cpp @@ -214,6 +214,23 @@ TEST(NodePrefs, FemGainSettingsRoundTrip) { EXPECT_EQ(1, loaded.radio_fem_txgain); } +TEST(ConfigSerializer, LoadSerial_KeyDigitsAfterFirstCharacter) { + MockInputStream s("{age:1,flags:2,name:\"ok\",slot1:7}"); + TestStruct data; + data.age = data.flags = 0; + strcpy(data.name, "before"); + EXPECT_TRUE(data.loadSerial(s)); + EXPECT_EQ(1, data.age); + EXPECT_EQ(2, data.flags); + EXPECT_STREQ("ok", data.name); +} + +TEST(ConfigSerializer, LoadSerial_RejectsLeadingDigitKey) { + MockInputStream s("{1slot:7}"); + TestStruct data; + EXPECT_FALSE(data.loadSerial(s)); +} + // ── main ─────────────────────────────────────────────────────── diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp index 67880a82..4d2ae120 100644 --- a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -20,6 +20,7 @@ enum class FailurePoint { PayloadWrite, ImageWrite, Finish, + Verify, Commit, }; @@ -44,7 +45,8 @@ public: ++write_calls; if (!_open) return 0; const bool should_fail = (write_calls == 1 && _failure == FailurePoint::HeaderWrite) || - (write_calls == 2 && _failure == FailurePoint::PayloadWrite); + (write_calls == 2 && _failure == FailurePoint::PayloadWrite) || + _failure == FailurePoint::ImageWrite; const size_t written = should_fail && size > 0 ? size - 1 : size; _staging.insert(_staging.end(), bytes, bytes + written); return written; @@ -68,6 +70,18 @@ public: return true; } + bool verify() { + ++verify_calls; + return _failure != FailurePoint::Verify; + } + + void discardFinishedTemp() { + ++discard_calls; + _files.erase("/mqtt_prefs.tmp"); + _finished = false; + _owns_temp = false; + } + void abort() { ++abort_calls; _open = false; @@ -88,6 +102,8 @@ public: int finish_calls = 0; int commit_calls = 0; int abort_calls = 0; + int verify_calls = 0; + int discard_calls = 0; private: FailurePoint _failure; @@ -113,6 +129,16 @@ AtomicStore::Result runWithObserverTail(InMemoryStore* store) { return AtomicStore::write(*store, header, sizeof(header), payload, sizeof(payload)); } +AtomicStore::VerifiedImageResult runVerifiedJson(InMemoryStore* store) { + const uint8_t json[] = "{version:1,wifi:{ssid:\"mesh\"}}"; + return AtomicStore::writeVerifiedImage( + *store, + [store, &json]() { + return store->write(json, sizeof(json) - 1) == sizeof(json) - 1; + }, + [store]() { return store->verify(); }); +} + class LegacyComPrefs { public: LegacyComPrefs() : bytes({'l', 'e', 'g', 'a', 'c', 'y', '-', 'c', 'o', 'm'}) {} @@ -286,9 +312,14 @@ public: } return; } + if (action == Recovery::Action::DiscardTemp) { + _files.erase("/mqtt_prefs.tmp"); + return; + } if (action == Recovery::Action::PromoteBackup) { rename("/mqtt_prefs.bak", "/mqtt_prefs"); - if (backup == Recovery::FileState::Usable && temp != Recovery::FileState::Missing) { + if (backup == Recovery::FileState::Usable && !Recovery::uncertain(temp) && + temp != Recovery::FileState::Missing) { _files.erase("/mqtt_prefs.tmp"); } return; @@ -298,12 +329,13 @@ public: // incomplete transaction artifact. It only preserves artifacts when the // primary itself is opaque. if (had_primary && primary == Recovery::FileState::Usable) { - _files.erase("/mqtt_prefs.tmp"); - _files.erase("/mqtt_prefs.bak"); + if (!Recovery::uncertain(temp)) _files.erase("/mqtt_prefs.tmp"); + if (!Recovery::uncertain(backup)) _files.erase("/mqtt_prefs.bak"); } } bool has(const char* path) const { return _files.count(path) != 0; } + void removePrimary() { _files.erase("/mqtt_prefs"); } bool canStartSave() const { return !has("/mqtt_prefs.tmp") && !has("/mqtt_prefs.bak"); } const std::vector& primary() const { return _files.at("/mqtt_prefs"); } static std::vector oldImage() { return {'o', 'l', 'd'}; } @@ -337,6 +369,50 @@ TEST(MQTTPrefsAtomicStore, CommitPublishesExactHeaderThenPayload) { EXPECT_EQ(0, store.abort_calls); } +TEST(MQTTPrefsAtomicStore, ProductionJsonPolicyCoversEveryVerificationBoundary) { + const std::vector old_source = { + 'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + const struct { + FailurePoint point; + AtomicStore::VerifiedImageResult expected; + int writes; + int finishes; + int verifies; + int commits; + int aborts; + int discards; + bool keeps_temp; + } cases[] = { + {FailurePoint::None, AtomicStore::VerifiedImageResult::Committed, + 1, 1, 1, 1, 0, 0, false}, + {FailurePoint::Begin, AtomicStore::VerifiedImageResult::BeginFailed, + 0, 0, 0, 0, 1, 0, false}, + {FailurePoint::ImageWrite, AtomicStore::VerifiedImageResult::WriteFailed, + 1, 0, 0, 0, 1, 0, false}, + {FailurePoint::Finish, AtomicStore::VerifiedImageResult::FinishFailed, + 1, 1, 0, 0, 1, 0, false}, + {FailurePoint::Verify, AtomicStore::VerifiedImageResult::VerifyFailed, + 1, 1, 1, 0, 0, 1, false}, + {FailurePoint::Commit, AtomicStore::VerifiedImageResult::CommitFailed, + 1, 1, 1, 1, 1, 0, true}, + }; + + for (const auto& test_case : cases) { + InMemoryStore store(test_case.point); + EXPECT_EQ(test_case.expected, runVerifiedJson(&store)); + EXPECT_EQ(test_case.writes, store.write_calls); + EXPECT_EQ(test_case.finishes, store.finish_calls); + EXPECT_EQ(test_case.verifies, store.verify_calls); + EXPECT_EQ(test_case.commits, store.commit_calls); + EXPECT_EQ(test_case.aborts, store.abort_calls); + EXPECT_EQ(test_case.discards, store.discard_calls); + EXPECT_EQ(test_case.keeps_temp, store.tempExists()); + if (test_case.point != FailurePoint::None) { + EXPECT_EQ(old_source, store.source()); + } + } +} + TEST(MQTTPrefsAtomicStore, AnyFailureAbortsAndPreservesExistingSource) { const std::vector source = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; const struct { @@ -546,6 +622,23 @@ TEST(MQTTPrefsAtomicStore, PowerCutDuringTempWriteKeepsPrimaryAndAllowsNextSave) EXPECT_TRUE(store.canStartSave()); } +TEST(MQTTPrefsAtomicStore, TornFirstMigrationTempIsDiscardedSoLegacyCanRetry) { + SpiffsMqttTransaction store; + store.removePrimary(); + store.cutDuringTempWrite(); + + EXPECT_EQ(Recovery::Action::DiscardTemp, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::Preserve, + Recovery::FileState::Missing)); + store.recover(Recovery::FileState::Missing, + Recovery::FileState::Preserve, + Recovery::FileState::Missing); + EXPECT_FALSE(store.has("/mqtt_prefs")); + EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.canStartSave()); +} + TEST(MQTTPrefsAtomicStore, RecoveredUsablePrimaryClearsOpaqueTransactionArtifacts) { { SpiffsMqttTransaction store; @@ -605,8 +698,14 @@ TEST(MQTTPrefsAtomicStore, RecoveryNeverOverwritesOpaqueNewerLayout) { EXPECT_EQ(Recovery::Action::KeepPrimary, Recovery::select(Recovery::FileState::Preserve, Recovery::FileState::Usable, Recovery::FileState::Usable)); - // If there is no primary, a usable backup wins over an opaque temp. Once - // promoted, production treats the backup as authoritative and clears temp. + // A syntactically valid future temp may already have passed verification and + // reached the rename phase. It wins over the stale supported backup and is + // promoted into the authoritative name, where older firmware will hold it. + EXPECT_EQ(Recovery::Action::PromoteTemp, + Recovery::select(Recovery::FileState::Missing, Recovery::FileState::FutureUsable, + Recovery::FileState::Usable)); + // A corrupt or incomplete temp is different: the supported backup remains + // the last known committed image. EXPECT_EQ(Recovery::Action::PromoteBackup, Recovery::select(Recovery::FileState::Missing, Recovery::FileState::Preserve, Recovery::FileState::Usable)); @@ -617,6 +716,34 @@ TEST(MQTTPrefsAtomicStore, RecoveryNeverOverwritesOpaqueNewerLayout) { Recovery::FileState::Preserve)); } +TEST(MQTTPrefsAtomicStore, AmbiguousFutureOrOomTempIsNeverDeleted) { + for (const Recovery::FileState uncertain : { + Recovery::FileState::FutureClaimed, + Recovery::FileState::Indeterminate}) { + SpiffsMqttTransaction store; + store.cutAt(SpiffsMqttTransaction::Boundary::AfterBackupRename); + store.recover(Recovery::FileState::Missing, uncertain, + Recovery::FileState::Usable); + + // The supported backup can run this boot, but the candidate that this + // firmware could not classify remains available to newer firmware or a + // later boot with more heap. Its presence also blocks a new transaction. + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_FALSE(store.canStartSave()); + } +} + +TEST(MQTTPrefsAtomicStore, UsablePrimaryDoesNotCleanIndeterminateArtifact) { + SpiffsMqttTransaction store; + store.cutDuringTempWrite(); + store.recover(Recovery::FileState::Usable, + Recovery::FileState::Indeterminate); + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_FALSE(store.canStartSave()); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp index 7bf49746..73ec5ec8 100644 --- a/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp +++ b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp @@ -28,6 +28,7 @@ MQTTPrefs defaults() { prefs.alert_wifi_minutes = 30; prefs.alert_mqtt_minutes = 240; prefs.alert_min_interval_min = 60; + prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; return prefs; } @@ -59,6 +60,21 @@ Codec::DecodePlan classify(const std::vector& bytes) { return Codec::classify(bytes.data(), prefix_size, bytes.size()); } +void writeV1Payload(std::vector* bytes, const MQTTPrefs& prefs, size_t payload_len) { + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(prefs, &frozen); + ASSERT_GE(bytes->size(), sizeof(MQTTPrefsHeader) + payload_len); + memcpy(bytes->data() + sizeof(MQTTPrefsHeader), &frozen, payload_len); +} + +MQTTPrefs decodeV1(const std::vector& bytes, const Codec::DecodePlan& plan) { + LegacyV1MQTTPrefs frozen = {}; + memcpy(&frozen, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = defaults(); + Codec::migrateV1(frozen, plan.payload_len, &loaded); + return loaded; +} + void fillHighEntropy(std::vector* bytes) { uint32_t state = 0x89abcdef; for (size_t i = 0; i < bytes->size(); ++i) { @@ -244,8 +260,7 @@ TEST(MQTTPrefsCodec, CurrentVersionedPayloadRoundTripsExactly) { ASSERT_EQ(Codec::Source::Current, plan.source); ASSERT_FALSE(plan.preserve_file); ASSERT_TRUE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -277,8 +292,7 @@ TEST(MQTTPrefsCodec, DefaultFiltersKeepTheDowngradeReadablePayloadLength) { ASSERT_EQ(Codec::kV1PreFilterPayloadSize, plan.payload_len); ASSERT_FALSE(plan.preserve_file); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -299,8 +313,7 @@ TEST(MQTTPrefsCodec, AnyNonDefaultFilterOptsIntoTheLongerPayload) { const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::kV1BaselinePayloadSize, plan.payload_len); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_EQ(mask, loaded.mqtt_slot_packet_filter[slot]) << slot; EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); } @@ -338,7 +351,7 @@ TEST(MQTTPrefsCodec, PreFilterV1PayloadDefaultsEverySlotToAllTypes) { std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreFilterPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreFilterPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreFilterPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreFilterPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -346,8 +359,7 @@ TEST(MQTTPrefsCodec, PreFilterV1PayloadDefaultsEverySlotToAllTypes) { ASSERT_TRUE(plan.observer_fields_present); ASSERT_FALSE(plan.preserve_file); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("pre-filter-node", loaded.mqtt_origin); EXPECT_EQ(1u, loaded.mqtt_neighbors_enabled); EXPECT_EQ(MQTT_NEIGHBORS_MAX_INTERVAL_MS, loaded.mqtt_neighbors_interval); @@ -368,7 +380,7 @@ TEST(MQTTPrefsCodec, CompatibleShortV1PayloadPreservesDefaultsBeyondObserverBoun std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreObserverPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreObserverPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreObserverPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreObserverPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -376,8 +388,7 @@ TEST(MQTTPrefsCodec, CompatibleShortV1PayloadPreservesDefaultsBeyondObserverBoun ASSERT_FALSE(plan.preserve_file); ASSERT_FALSE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("short-v1-node", loaded.mqtt_origin); EXPECT_STREQ("ntp.short.example", loaded.mqtt_ntp_server); EXPECT_EQ(0, loaded.snmp_enabled); @@ -400,7 +411,7 @@ TEST(MQTTPrefsCodec, PreNeighborsV1PayloadLoadsObserverFieldsAndDefaultsNeighbor std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreNeighborsPayloadSize, 0); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(Codec::kV1PreNeighborsPayloadSize)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreNeighborsPayloadSize); + writeV1Payload(&bytes, source, Codec::kV1PreNeighborsPayloadSize); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -408,10 +419,7 @@ TEST(MQTTPrefsCodec, PreNeighborsV1PayloadLoadsObserverFieldsAndDefaultsNeighbor ASSERT_FALSE(plan.preserve_file); ASSERT_TRUE(plan.observer_fields_present); - MQTTPrefs loaded = defaults(); - loaded.mqtt_neighbors_enabled = 1; // pretend stale - loaded.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; // caller's defaulted tail - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + MQTTPrefs loaded = decodeV1(bytes, plan); EXPECT_STREQ("pre-neighbors-node", loaded.mqtt_origin); EXPECT_STREQ("PNW", loaded.alert_region); @@ -475,6 +483,34 @@ TEST(MQTTPrefsCodec, CorruptOrShortVersionedInputsArePreserved) { EXPECT_TRUE(plan.preserve_file); } +TEST(MQTTPrefsCodec, HeaderValidV1StillRequiresBoundedPayloadStrings) { + MQTTPrefs source = defaults(); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + ASSERT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); + + memset(frozen.mqtt_slot_topic[5], 'x', sizeof(frozen.mqtt_slot_topic[5])); + EXPECT_FALSE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); +} + +TEST(MQTTPrefsCodec, HeaderValidV1AllowsNumericValuesForSerializerRepair) { + MQTTPrefs source = defaults(); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + + // A valid v1 header fixes the layout. Numeric bytes cannot trigger an + // out-of-bounds read and are normalized after migration, so plausibility + // must not reject deployed files merely because these values need repair. + frozen.timezone_offset = 99; + frozen.mqtt_status_enabled = 0xff; + frozen.mqtt_rx_enabled = 0xff; + frozen.snmp_enabled = 0xff; + frozen.alert_enabled = 0xff; + frozen.mqtt_neighbors_enabled = 0xa5; + EXPECT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1BaselinePayloadSize)); + EXPECT_TRUE(Codec::isPlausibleV1(frozen, Codec::kV1PreNeighborsPayloadSize)); +} + TEST(MQTTPrefsCodec, LegacyPlausibilityRejectsHighEntropyBytesAtEveryWhitelistedSize) { // A headerless raw struct has no checksum, so this only reduces false // migrations; it cannot prove that a plausible-looking file is authentic. @@ -531,7 +567,9 @@ TEST(MQTTPrefsCodec, LongerSameVersionPayloadLoadsTheBaselineAndIgnoresTheTail) const size_t payload_len = Codec::kV1BaselinePayloadSize + kTail; std::vector bytes(sizeof(MQTTPrefsHeader) + payload_len, 0xA5); writeHeader(&bytes, MQTT_PREFS_VERSION, static_cast(payload_len)); - memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, sizeof(source)); + LegacyV1MQTTPrefs frozen; + Codec::freezeV1(source, &frozen); + memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &frozen, sizeof(frozen)); const Codec::DecodePlan plan = classify(bytes); ASSERT_EQ(Codec::Source::Current, plan.source); @@ -542,9 +580,11 @@ TEST(MQTTPrefsCodec, LongerSameVersionPayloadLoadsTheBaselineAndIgnoresTheTail) // Reading plan.payload_len bytes recovers this build's whole struct exactly, // and cannot run past it into the unknown tail. + LegacyV1MQTTPrefs loaded_frozen = {}; + memcpy(&loaded_frozen, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + ASSERT_TRUE(Codec::isPlausibleV1(loaded_frozen, plan.payload_len)); MQTTPrefs loaded = defaults(); - memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); - EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); + Codec::migrateV1(loaded_frozen, plan.payload_len, &loaded); EXPECT_STREQ("future-node", loaded.mqtt_origin); EXPECT_STREQ("field-ssid", loaded.wifi_ssid); EXPECT_STREQ("field-secret", loaded.wifi_password); diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp new file mode 100644 index 00000000..ec2544e7 --- /dev/null +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -0,0 +1,358 @@ +#include + +#include + +#define WITH_MQTT_BRIDGE 1 +#define PROGMEM +#include "helpers/MQTTPrefsSerializer.h" + +class InputStream : public Stream { +public: + explicit InputStream(const std::string& text) : _text(text) {} + int available() override { return static_cast(_text.size() - _pos); } + int read() override { return _pos < _text.size() ? _text[_pos++] : -1; } + int peek() override { return _pos < _text.size() ? _text[_pos] : -1; } +private: + std::string _text; + size_t _pos = 0; +}; + +class OutputStream : public Stream { +public: + size_t write(uint8_t byte) override { _text.push_back(static_cast(byte)); return 1; } + size_t print(int value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned int value, int = DEC) override { return appendNumber(value); } + size_t print(long value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned long value, int = DEC) override { return appendNumber(value); } + size_t print(long long value, int = DEC) override { return appendNumber(value); } + size_t print(unsigned long long value, int = DEC) override { return appendNumber(value); } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + const std::string& text() const { return _text; } +private: + template size_t appendNumber(T value) { + const std::string number = std::to_string(value); + _text += number; + return number.size(); + } + std::string _text; +}; + +class StickyFailingStream : public Stream { +public: + explicit StickyFailingStream(size_t limit) : _limit(limit) {} + size_t write(uint8_t) override { + if (_failed || _written >= _limit) { + _failed = true; + return 0; + } + ++_written; + return 1; + } + size_t print(int value, int = DEC) override { + const std::string number = std::to_string(value); + return Print::print(number.c_str()); + } + int available() override { return 0; } + int read() override { return -1; } + int peek() override { return -1; } + bool failed() const { return _failed; } +private: + size_t _limit; + size_t _written = 0; + bool _failed = false; +}; + +static MQTTPrefs defaults() { + MQTTPrefs prefs = {}; + prefs.mqtt_status_enabled = 1; + prefs.mqtt_packets_enabled = 1; + prefs.mqtt_tx_enabled = 2; + prefs.mqtt_rx_enabled = 1; + prefs.mqtt_status_interval = 300000; + prefs.wifi_power_save = 1; + prefs.timezone_offset = -7; + prefs.radio_watchdog_minutes = 5; + prefs.alert_wifi_minutes = 30; + prefs.alert_mqtt_minutes = 240; + prefs.alert_min_interval_min = 60; + prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; + strcpy(prefs.snmp_community, "public"); + for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { + strcpy(prefs.mqtt_slot_preset[i], "none"); + prefs.mqtt_slot_packet_filter[i] = 0xffff; + } + return prefs; +} + +TEST(MQTTPrefsSerializer, RoundTripsEveryGroupAndNumericSlotKeys) { + MQTTPrefs source = defaults(); + strcpy(source.wifi_ssid, "mesh-net"); + strcpy(source.wifi_password, "p\\\"ass\nword"); + strcpy(source.timezone_string, "MST7MDT,M3.2.0"); + strcpy(source.mqtt_ntp_server, "time.example"); + strcpy(source.mqtt_origin, "observer-one"); + strcpy(source.mqtt_iata, "SEA"); + source.mqtt_neighbors_enabled = 1; + source.mqtt_neighbors_interval = MQTT_NEIGHBORS_MAX_INTERVAL_MS; + strcpy(source.mqtt_owner_public_key, + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); + strcpy(source.mqtt_email, "owner@example.com"); + strcpy(source.mqtt_slot_preset[5], "custom"); + strcpy(source.mqtt_slot_host[5], "broker.example"); + source.mqtt_slot_port[5] = 65535; + strcpy(source.mqtt_slot_username[5], "user-six"); + strcpy(source.mqtt_slot_password[5], "secret-six"); + strcpy(source.mqtt_slot_token[5], "token-six"); + strcpy(source.mqtt_slot_topic[5], "mesh/{iata}/{type}"); + strcpy(source.mqtt_slot_audience[5], "audience-six"); + source.mqtt_slot_packet_filter[5] = 0x8001; + source.snmp_enabled = 1; + source.radio_watchdog_minutes = 120; + source.alert_enabled = 1; + strcpy(source.alert_psk_hex, "0123456789abcdef0123456789abcdef"); + strcpy(source.alert_hashtag, "#ops"); + strcpy(source.alert_region, "PNW"); + + OutputStream output; + MQTTPrefsSerializer writer(&source); + ASSERT_TRUE(writer.saveSerial(output)); + EXPECT_NE(std::string::npos, output.text().find("slot6:{")) << output.text(); + EXPECT_NE(std::string::npos, output.text().find("packet_filter:32769")) << output.text(); + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)) << output.text(); + bool repaired = true; + ASSERT_TRUE(reader.apply(&repaired)); + EXPECT_FALSE(repaired); + EXPECT_STREQ(source.wifi_password, loaded.wifi_password); + EXPECT_STREQ(source.mqtt_owner_public_key, loaded.mqtt_owner_public_key); + EXPECT_STREQ(source.mqtt_slot_host[5], loaded.mqtt_slot_host[5]); + EXPECT_EQ(65535, loaded.mqtt_slot_port[5]); + EXPECT_EQ(0x8001, loaded.mqtt_slot_packet_filter[5]); + EXPECT_EQ(MQTT_NEIGHBORS_MAX_INTERVAL_MS, loaded.mqtt_neighbors_interval); + EXPECT_STREQ("PNW", loaded.alert_region); +} + +TEST(MQTTPrefsSerializer, MissingOptionalKeysKeepDefaults) { + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,mqtt:{origin:\"changed\"}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = true; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_FALSE(repaired); + EXPECT_STREQ("changed", prefs.mqtt_origin); + EXPECT_EQ(1, prefs.mqtt_packets_enabled); + EXPECT_EQ(300000u, prefs.mqtt_status_interval); + EXPECT_EQ(0xffff, prefs.mqtt_slot_packet_filter[5]); +} + +TEST(MQTTPrefsSerializer, RequiresSupportedVersion) { + MQTTPrefs missing = defaults(); + InputStream no_version("{wifi:{ssid:\"x\"}}"); + MQTTPrefsSerializer missing_serializer(&missing); + ASSERT_TRUE(missing_serializer.loadSerial(no_version)); + bool repaired = false; + EXPECT_FALSE(missing_serializer.apply(&repaired)); + + MQTTPrefs future = defaults(); + InputStream future_input("{version:2,wifi:{ssid:\"x\"}}"); + MQTTPrefsSerializer future_serializer(&future); + ASSERT_TRUE(future_serializer.loadSerial(future_input)); + EXPECT_TRUE(future_serializer.hasFutureVersion()); + EXPECT_FALSE(future_serializer.apply(&repaired)); +} + +TEST(MQTTPrefsSerializer, FutureVersionProbeIgnoresV1FieldTypeChanges) { + InputStream probe_input("{version:2,wifi:{power_save:\"max\"}}"); + MQTTPrefsVersionProbe probe; + ASSERT_TRUE(probe.loadSerial(probe_input)); + EXPECT_TRUE(probe.hasFutureVersion()); + + // The same image is not valid under v1, demonstrating why recovery must + // probe the version before invoking the current schema. + MQTTPrefs prefs = defaults(); + InputStream v1_input("{version:2,wifi:{power_save:\"max\"}}"); + MQTTPrefsSerializer v1(&prefs); + EXPECT_FALSE(v1.loadSerial(v1_input)); + + for (const char* future_text : { + "{version:2,this_key_is_too_long:1}", + "{version:2,x:[1]}", + "{version:2,wifi:{ssid:\"torn\"}"}) { + InputStream future_input(future_text); + MQTTPrefsVersionProbe future_probe; + EXPECT_FALSE(future_probe.loadSerial(future_input)) << future_text; + EXPECT_TRUE(future_probe.hasFutureVersion()) << future_text; + } +} + +TEST(MQTTPrefsSerializer, RejectsDuplicateKnownKey) { + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,mqtt:{origin:\"one\",origin:\"two\"}}"); + MQTTPrefsSerializer serializer(&prefs); + EXPECT_FALSE(serializer.loadSerial(input)); +} + +TEST(MQTTPrefsSerializer, RejectsOverlongStringAndIntegerOverflow) { + MQTTPrefs prefs = defaults(); + InputStream long_string( + "{version:1,wifi:{ssid:\"12345678901234567890123456789012\"}}"); + MQTTPrefsSerializer string_serializer(&prefs); + EXPECT_FALSE(string_serializer.loadSerial(long_string)); + + prefs = defaults(); + InputStream overflow("{version:1,mqtt:{slot1:{port:999999999999}}}"); + MQTTPrefsSerializer number_serializer(&prefs); + EXPECT_FALSE(number_serializer.loadSerial(overflow)); + + prefs = defaults(); + InputStream quoted_number("{version:\"1\"}"); + MQTTPrefsSerializer quoted_number_serializer(&prefs); + EXPECT_FALSE(quoted_number_serializer.loadSerial(quoted_number)); + + prefs = defaults(); + InputStream bare_string("{version:1,wifi:{ssid:meshnet}}"); + MQTTPrefsSerializer bare_string_serializer(&prefs); + EXPECT_FALSE(bare_string_serializer.loadSerial(bare_string)); +} + +TEST(MQTTPrefsSerializer, RejectsScalarObjectShapeMismatches) { + MQTTPrefs prefs = defaults(); + InputStream object_version("{version:{x:1}}"); + MQTTPrefsSerializer object_version_serializer(&prefs); + EXPECT_FALSE(object_version_serializer.loadSerial(object_version)); + + prefs = defaults(); + InputStream object_port("{version:1,mqtt:{slot1:{port:{x:1883}}}}"); + MQTTPrefsSerializer object_port_serializer(&prefs); + EXPECT_FALSE(object_port_serializer.loadSerial(object_port)); + + prefs = defaults(); + InputStream scalar_mqtt("{version:1,mqtt:1}"); + MQTTPrefsSerializer scalar_mqtt_serializer(&prefs); + EXPECT_FALSE(scalar_mqtt_serializer.loadSerial(scalar_mqtt)); + + prefs = defaults(); + InputStream scalar_slot("{version:1,mqtt:{slot1:1}}"); + MQTTPrefsSerializer scalar_slot_serializer(&prefs); + EXPECT_FALSE(scalar_slot_serializer.loadSerial(scalar_slot)); +} + +TEST(MQTTPrefsSerializer, RepairsSemanticRanges) { + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,wifi:{power_save:9},time:{utc_offset:99}," + "mqtt:{tx_enabled:7,status:{enabled:3,interval_ms:10}," + "neighbors:{enabled:2,interval_ms:100},slot1:{port:-1,packet_filter:-2}}," + "radio:{watchdog_min:121},alert:{rate_limit_min:1}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(1, prefs.wifi_power_save); + EXPECT_EQ(-7, prefs.timezone_offset); + EXPECT_EQ(2, prefs.mqtt_tx_enabled); + EXPECT_EQ(300000u, prefs.mqtt_status_interval); + EXPECT_EQ(MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS, prefs.mqtt_neighbors_interval); + EXPECT_EQ(0, prefs.mqtt_slot_port[0]); + EXPECT_EQ(0xffff, prefs.mqtt_slot_packet_filter[0]); + EXPECT_EQ(5, prefs.radio_watchdog_minutes); + EXPECT_EQ(60, prefs.alert_min_interval_min); +} + +TEST(MQTTPrefsSerializer, RepairsTextValuesToSafeDefaults) { + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,time:{ntp_server:\"bad/host\"},mqtt:{iata:\"sea\"," + "owner:{public_key:\"not-a-key\"},slot1:{preset:\"not-a-preset\"}," + "slot2:{preset:\"analyzer-us\"},slot3:{preset:\"analyzer-us\"}}," + "alert:{psk_hex:\"not-hex\",hashtag:\"#stale\"}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_STREQ("SEA", prefs.mqtt_iata); + EXPECT_STREQ("", prefs.mqtt_ntp_server); + EXPECT_STREQ("", prefs.mqtt_owner_public_key); + EXPECT_STREQ("none", prefs.mqtt_slot_preset[0]); + EXPECT_STREQ("analyzer-us", prefs.mqtt_slot_preset[1]); + // Historical firmware allowed duplicate aliases. Preserve them on load; + // current setters prevent creating new duplicates without silently changing + // a deployed configuration during migration. + EXPECT_STREQ("analyzer-us", prefs.mqtt_slot_preset[2]); + EXPECT_STREQ("", prefs.alert_psk_hex); + EXPECT_STREQ("", prefs.alert_hashtag); +} + +TEST(MQTTPrefsSerializer, LateParseOrVersionFailureCannotMutateLivePrefs) { + MQTTPrefs live = defaults(); + strcpy(live.wifi_ssid, "live-network"); + + MQTTPrefs scratch = defaults(); + InputStream truncated("{wifi:{ssid:\"uncommitted\"},version:1"); + MQTTPrefsSerializer truncated_serializer(&scratch); + EXPECT_FALSE(truncated_serializer.loadSerial(truncated)); + EXPECT_STREQ("live-network", live.wifi_ssid); + + scratch = defaults(); + InputStream future("{wifi:{ssid:\"future-network\"},version:2}"); + MQTTPrefsSerializer future_serializer(&scratch); + ASSERT_TRUE(future_serializer.loadSerial(future)); + EXPECT_TRUE(future_serializer.hasFutureVersion()); + bool repaired = false; + EXPECT_FALSE(future_serializer.apply(&repaired)); + EXPECT_STREQ("live-network", live.wifi_ssid); +} + +TEST(MQTTPrefsSerializer, StickyShortWriteFailsTheCompleteSave) { + MQTTPrefs prefs = defaults(); + MQTTPrefsSerializer serializer(&prefs); + StickyFailingStream output(20); + EXPECT_FALSE(serializer.saveSerial(output)); + EXPECT_TRUE(output.failed()); +} + +TEST(MQTTPrefsSerializer, SaveNormalizationIsIdempotentAgainstKnownDefaults) { + MQTTPrefs prefs = defaults(); + prefs.timezone_offset = 99; + strcpy(prefs.mqtt_ntp_server, "bad/host"); + strcpy(prefs.mqtt_iata, "not-iata"); + strcpy(prefs.mqtt_slot_preset[0], "not-a-preset"); + + MQTTPrefs repair_defaults = defaults(); + repair_defaults.timezone_offset = -7; + strcpy(repair_defaults.mqtt_iata, "sea"); + strcpy(repair_defaults.mqtt_slot_preset[0], "analyzer-us"); + + MQTTPrefsSerializer writer(&prefs, &repair_defaults); + bool repaired = false; + ASSERT_TRUE(writer.normalize(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(-7, prefs.timezone_offset); + EXPECT_STREQ("", prefs.mqtt_ntp_server); + EXPECT_STREQ("SEA", prefs.mqtt_iata); + EXPECT_STREQ("none", prefs.mqtt_slot_preset[0]); + + OutputStream output; + ASSERT_TRUE(writer.saveSerial(output)); + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)); + repaired = true; + ASSERT_TRUE(reader.apply(&repaired)); + EXPECT_FALSE(repaired) << output.text(); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_observer_validation/test_observer_validation.cpp b/test/test_observer_validation/test_observer_validation.cpp index 3045caed..8e9913ca 100644 --- a/test/test_observer_validation/test_observer_validation.cpp +++ b/test/test_observer_validation/test_observer_validation.cpp @@ -129,6 +129,18 @@ TEST(ValueFits, RejectsNullOrZeroBuffer) { EXPECT_FALSE(mqttValueFits("abc", 0)); } +TEST(AssignedPresetSlot, FindsDuplicatesOutsideTheTargetSlot) { + const char presets[4][24] = { + "analyzer-us", "none", "analyzer-eu", "analyzer-us" + }; + + EXPECT_EQ(3, mqttAssignedPresetSlot(presets, "analyzer-us", 0)); + EXPECT_EQ(0, mqttAssignedPresetSlot(presets, "analyzer-us", 3)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, "analyzer-eu", 2)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, "missing", 0)); + EXPECT_EQ(-1, mqttAssignedPresetSlot(presets, nullptr, 0)); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_webconfig_batch/test_webconfig_batch.cpp b/test/test_webconfig_batch/test_webconfig_batch.cpp index 813412c5..670c6ead 100644 --- a/test/test_webconfig_batch/test_webconfig_batch.cpp +++ b/test/test_webconfig_batch/test_webconfig_batch.cpp @@ -230,6 +230,7 @@ TEST(WebConfigBatch, CliFailureRepliesAreRecognisedInEveryShapeCommonCLIEmits) { "Err - bad params", // MyMesh setperm "ERR: bad pubkey", // neighbor.remove "Error: IATA code must be exactly 3 letters",// observer setters + "Error: setting not persisted; change rolled back", // observer storage failure "(ERR: clock cannot go backwards)", // clock sync, parenthesised "Unknown command", // top-level fallthrough "unknown config: mqtt.nope", // set fallthrough From 4c90db2199fdc149ee9cb6293e8773edff3f57b8 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 5 Aug 2026 22:29:51 -0700 Subject: [PATCH 05/93] fix(mqtt): renew JWT credentials without stopping the esp-mqtt client The scheduled JWT bounce called PsychicMqttClient::disconnect(), which ends with esp_mqtt_client_stop(). That ends the client task and returns its 6 KiB stack to the heap at the moment the TLS teardown vacates two 16 KiB mbedTLS record buffers, so the stack lands in that hole and the next handshake cannot reuse it. On non-PSRAM boards the largest free block then ratchets down 16 KiB at a time while total free heap stays flat. Soak evidence from a Heltec V3 on 8d1a0eb3: 43 of 60 disconnects had no preceding transport error, i.e. they were this proactive bounce rather than a broker FIN, and two of the three max_alloc steps landed within 5 s of one. Losing a whole TLS session later returned exactly 16,384 bytes of contiguity. softDisconnect() closes the transport without the stop, so the task and its stack stay put across the handshake. The bounce uses it plus reconnect(), and falls back to connect() when the client really is stopped, since reconnect() is a silent no-op in that state. Also corrects a comment claiming the mbedTLS context survives a transport close: only the esp-mqtt client object does. (cherry picked from commit 10cf5cf48fb009e751e25b37fcc1f3d1256ddbbc) --- .../src/PsychicMqttClient.cpp | 39 ++++++++++++++++++- lib/PsychicMqttClient/src/PsychicMqttClient.h | 24 ++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 29 ++++++++++---- src/helpers/bridges/MQTTBridge.h | 13 +++++++ 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index abe49c88..5d037b7e 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -433,7 +433,12 @@ void PsychicMqttClient::connect() } } - ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_start(_client)); + esp_err_t start_result = esp_mqtt_client_start(_client); + ESP_ERROR_CHECK_WITHOUT_ABORT(start_result); + if (start_result == ESP_OK) + { + _started = true; + } ESP_LOGI(TAG, "MQTT client started."); } @@ -489,9 +494,40 @@ void PsychicMqttClient::disconnect() } esp_mqtt_client_stop(_client); + _started = false; ESP_LOGI(TAG, "MQTT client stopped."); } +void PsychicMqttClient::softDisconnect(unsigned long timeout_ms) +{ + if (_client == nullptr) + { + ESP_LOGW(TAG, "MQTT client not started."); + return; + } + + if (!_connected) + { + // Nothing to close; leaving the task alone is the whole point. + return; + } + + ESP_LOGI(TAG, "Disconnecting MQTT transport (client task retained)."); + _stopMqttClient = false; + esp_mqtt_client_disconnect(_client); + + unsigned long waited = 0; + while (!_stopMqttClient && waited < timeout_ms) + { + vTaskDelay(10 / portTICK_PERIOD_MS); + waited += 10; + } + if (!_stopMqttClient) + { + ESP_LOGW(TAG, "softDisconnect: no DISCONNECTED event in %lums", timeout_ms); + } +} + void PsychicMqttClient::forceStop() { if (_client == nullptr) @@ -506,6 +542,7 @@ void PsychicMqttClient::forceStop() } ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_stop(_client)); _connected = false; + _started = false; ESP_LOGI(TAG, "MQTT client forcefully stopped."); } diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index 42cf8d5c..28e53fc3 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -368,6 +368,29 @@ public: */ void disconnect(); + /** + * @brief Closes the transport but leaves the client task running. + * + * disconnect() ends with esp_mqtt_client_stop(), which ends the client task + * and returns its 6 KiB stack to the heap right as a TLS handshake vacates + * two 16 KiB mbedTLS record buffers — the stack then lands in that hole and + * the largest free block ratchets down. This variant omits the stop, so the + * task and its stack stay put. Pair it with reconnect(). + * + * @param timeout_ms how long to wait for the DISCONNECTED event before + * giving up. Bounded on purpose: disconnect()'s wait is + * unbounded and a lost event would wedge the caller. + */ + void softDisconnect(unsigned long timeout_ms = 5000); + + /** + * @brief True once esp_mqtt_client_start() has succeeded and no stop has run. + * + * reconnect() silently does nothing on a stopped client, so callers that + * want to avoid stop/start must check this and fall back to connect(). + */ + bool isStarted() const { return _started; } + /** * @brief Forcefully stops the MQTT client and disconnects from the server. * This does not trigger the onDisconnect callbacks. @@ -478,6 +501,7 @@ private: bool _connected = false; bool _stopMqttClient = false; bool _config_dirty = true; + bool _started = false; // Runtime cap on the esp-mqtt outbox for QoS 0 async publishes (bytes). // 0 = disabled. Enforced in publish(); not an esp-mqtt config field. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 735999e9..bb2ff93e 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1775,9 +1775,11 @@ bool MQTTBridge::setupSlot(int index) { } // Reconfigure path: if we're re-applying (e.g. after a preset change), stop - // the existing connection cleanly first. The client object (and its mbedTLS - // context) is reused; setCredentials / setServer below overwrite the config - // fields in place before connect() restarts the ESP-IDF client. + // the existing connection cleanly first. The client object is reused, but its + // mbedTLS context is NOT — closing the transport destroys the TLS session, + // record buffers, and peer certificate, and the next connect() reallocates + // them. setCredentials / setServer below overwrite the config fields in place + // before connect() restarts the ESP-IDF client. if (slot.initial_connect_done) { if (slot.client->connected()) { slot.client->disconnect(); @@ -2140,11 +2142,24 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // Disconnect + reconnect with fresh credentials, reusing existing client // to avoid internal heap leak/fragmentation from destroy/create cycles MQTT_DEBUG_PRINTLN("MQTT%d token renewal: reconnecting with fresh credentials", index + 1); - if (slot.client->connected()) { - slot.client->disconnect(); // stops the client internally + MQTT_TRACE_HEAP("renewal:before-bounce", index); + if (slot.client->isStarted()) { + // Keep the esp-mqtt task alive across the handshake. disconnect() + // would stop it, returning its 6 KiB stack into the hole the two + // 16 KiB mbedTLS record buffers just vacated — which is what walks + // the largest free block down 16 KiB at a time on non-PSRAM boards. + slot.client->softDisconnect(); + MQTT_TRACE_HEAP("renewal:after-disconnect", index); + slot.client->setCredentials(_jwt_username, slot.auth_token); + MQTT_TRACE_HEAP("renewal:after-credentials", index); + slot.client->reconnect(); + } else { + // Client was stopped (teardown/reconfigure). reconnect() is a no-op + // on a stopped client, so this path must start it. + slot.client->setCredentials(_jwt_username, slot.auth_token); + slot.client->connect(); } - slot.client->setCredentials(_jwt_username, slot.auth_token); - slot.client->connect(); // restart stopped client; reconnect() fails silently on a stopped client + MQTT_TRACE_HEAP("renewal:after-reconnect", index); reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1, diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index eb7ab943..6056a46a 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -36,6 +36,19 @@ class MeshSNMPAgent; // Forward declaration #define MQTT_DEBUG_PRINTLN(...) {} #endif +// Largest-free-block trace around the reconnect lifecycle. On non-PSRAM boards the +// mbedTLS record buffers are two 16 KiB internal-DRAM blocks, so what matters is the +// largest contiguous block, not the free total — a soak can show flat free heap while +// max_alloc walks down. Costs two heap_caps calls per reconnect, so it stays on. +#if defined(MQTT_DEBUG) && defined(ARDUINO) && defined(ESP32) + #define MQTT_TRACE_HEAP(point, idx) \ + MQTT_DEBUG_PRINTLN("HEAPTRACE slot=%d %s free=%u max=%u", (int)(idx) + 1, point, \ + (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), \ + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL)) +#else + #define MQTT_TRACE_HEAP(point, idx) do {} while(0) +#endif + #ifdef WITH_MQTT_BRIDGE // Periodic neighbors publication keys off the mesh neighbor cache (sized by From 8ee21d2c0fabdebe6b0fba0e49a4440eb3b0b0ec Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 10 Aug 2026 15:45:56 -0700 Subject: [PATCH 06/93] fix(mqtt): allocate the neighbors JSON buffer on first use, not at bridge start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allocateRuntimeBuffers() took NEIGHBORS_JSON_BUFFER_SIZE unconditionally on every board built WITH_MQTT_NEIGHBORS, whether or not mqtt.neighbors was ever turned on. On a non-PSRAM board that is 4 KB of internal DRAM held for the bridge's lifetime by a node that may never publish a neighbours snapshot. Gating the existing allocation on the pref would not work: mqtt.neighbors is read live by the mesh loop with no bridge restart, so enabling it at runtime would find no buffer and silently publish nothing. Allocate on first use instead, in requestPublishNeighbors(), which is reached only when something actually wants to publish — periodic or a manual discovery. Publishing the pointer across cores is safe with the existing handshake: the allocation precedes the release store on _neighbors_publish_pending, and the task loop reads the pointer only after its matching acquire load, so the pointer cannot be observed half-published. A failed allocation drops that one snapshot and retries on the next, rather than disabling neighbours for the bridge's lifetime as the eager path did. (cherry picked from commit e6da052a93f8765824d0fb4bd0c704ca3ed3d294) --- src/helpers/bridges/MQTTBridge.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index bb2ff93e..da201596 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -768,17 +768,10 @@ void MQTTBridge::allocateRuntimeBuffers() { _json_scratch_buffer ? "PSRAM" : "stack fallback"); #endif -#if defined(WITH_MQTT_NEIGHBORS) - // Persistent neighbors JSON buffer, heap-allocated on every board: too large to - // keep inline in the bridge object the way the non-PSRAM status/packet buffers - // are. psram_malloc() falls back to internal DRAM, so this works without PSRAM. - // Unlike status/packet there is no stack fallback — a nullptr simply disables - // publishing (requestPublishNeighbors/publishNeighbors both no-op on nullptr). - _neighbors_json_buffer = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( - _neighbors_json_buffer, NEIGHBORS_JSON_BUFFER_SIZE, psram_malloc)); - MQTT_DEBUG_PRINTLN("Neighbors buffer: %s", - _neighbors_json_buffer ? "ready" : "unavailable"); -#endif + // The neighbors JSON buffer is NOT allocated here — requestPublishNeighbors() + // allocates it on first use, so a node with mqtt.neighbors off never pays its + // 4 KB. mqtt.neighbors is read live with no bridge restart, so gating on the + // pref here would leave a runtime enable with no buffer. } void MQTTBridge::releaseRuntimeBuffers() { @@ -795,7 +788,7 @@ void MQTTBridge::releaseRuntimeBuffers() { _json_scratch_doc.clear(); #if defined(WITH_MQTT_NEIGHBORS) - // Paired with the unconditional allocation in allocateRuntimeBuffers(). + // Paired with the lazy allocation in requestPublishNeighbors(); no-op if never used. _neighbors_json_buffer = static_cast(MQTTRuntimeBufferLifecycle::release( _neighbors_json_buffer, psram_free)); _neighbors_publish_len = 0; @@ -3640,10 +3633,19 @@ void MQTTBridge::setNeighborsSchedule(NeighborsPhase phase, uint32_t secs_until_ } void MQTTBridge::requestPublishNeighbors(const char* json, size_t len) { - if (!_neighbors_json_buffer || !json || len == 0) return; + if (!json || len == 0) return; // Drop a new snapshot while one is still being published (Core 0 clears the // flag when done). Acquire pairs with the task loop's release store. if (_neighbors_publish_pending.load(std::memory_order_acquire)) return; + // Allocated on first use so a node with neighbors off never pays the 4 KB. + // Cross-core safe: the release store below publishes this pointer, and the task + // loop only reads it after the matching acquire load. + _neighbors_json_buffer = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( + _neighbors_json_buffer, NEIGHBORS_JSON_BUFFER_SIZE, psram_malloc)); + if (!_neighbors_json_buffer) { + MQTT_DEBUG_PRINTLN("Neighbors buffer unavailable, dropping snapshot"); + return; + } if (len >= NEIGHBORS_JSON_BUFFER_SIZE) { len = NEIGHBORS_JSON_BUFFER_SIZE - 1; } From 8275512964063902067c26a3215af7984fde8c2e Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 10 Aug 2026 15:57:55 -0700 Subject: [PATCH 07/93] build(mqtt): make the reduced-TLS mbedTLS archives shippable, opt-in and verified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reduced-TLS work was validated on hardware but only reachable through PLATFORMIO_BUILD_FLAGS pointing at an absolute path in a developer's home directory, so nothing outside that machine could reproduce it. Distribute the archives as a release asset instead of committing them: ~6 MB per architecture, and they must be rebuilt for every espressif32 bump, so committing would grow history permanently and go stale without any signal. scripts/mbedtls_4k_manifest.txt per-arch sha256 of each archive scripts/fetch_mbedtls_4k.sh fetch into .mbedtls-4k//, verify scripts/mbedtls_4k.py pre-build wiring and post-link proof Off by default. The script is attached to esp32_base but returns immediately unless MESHCORE_REDUCED_TLS=1, so ordinary builds need no artifact and are byte-for-byte unaffected — confirmed by building with it absent. Both ways this can fail silently produce a firmware that looks fine and lacks the change, so the opt-in path refuses to guess: - a -L at a missing or partial directory: the linker ignores an unusable search path and resolves mbedTLS from the framework. Now a hard error. - archives left over from an earlier platform version: now a sha256 mismatch against the manifest, naming both hashes. - a -L that is present but outranked, leaving the flag inert: after the link, firmware.map must resolve every libmbed*.a into .mbedtls-4k/, or the build fails and prints the offending paths. That last check earned its place immediately — it caught its own first implementation comparing a relative map path against an absolute one, and an earlier build flag in this investigation was accepted by the compiler while no source read it. A flag reaching the compiler proves nothing about the link. Verified all four paths on Heltec_v3_repeater_observer_mqtt: default build unaffected; opted in with archives present links all four from .mbedtls-4k/ and says so; archives absent fails with a fetch hint; a single appended byte fails on sha256. Also removes platformio.local.ini.hold, which held the superseded approach of pointing platform_packages at a whole custom framework. That installs over the shared framework package and changes mbedTLS for every other ESP32 project on the machine; the -L path keeps the change scoped to one env. Note the inbound record buffer stays at 16 KiB, so this lowers per-connection footprint by ~12 KiB but does not move the contiguous allocation a handshake needs. It buys headroom, not a lower floor. (cherry picked from commit a87faff6ff170c328fdd0550f4b4dd9089aa2ea0) --- .gitignore | 4 + docs/mbedtls-tls-footprint.md | 30 +++++++ platformio.ini | 2 + platformio.local.ini.hold | 15 ---- scripts/fetch_mbedtls_4k.sh | 72 ++++++++++++++++ scripts/mbedtls_4k.py | 144 ++++++++++++++++++++++++++++++++ scripts/mbedtls_4k_manifest.txt | 12 +++ 7 files changed, 264 insertions(+), 15 deletions(-) delete mode 100644 platformio.local.ini.hold create mode 100755 scripts/fetch_mbedtls_4k.sh create mode 100644 scripts/mbedtls_4k.py create mode 100644 scripts/mbedtls_4k_manifest.txt diff --git a/.gitignore b/.gitignore index 699b2824..40f04f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ platformio.local.ini .build-wt-*/ .wt-*/ scripts/__pycache__/* + +# Reduced-TLS mbedTLS archives, ~6 MB per arch, fetched by scripts/fetch_mbedtls_4k.sh. +# Not committed because they must be rebuilt for every espressif32 platform bump. +.mbedtls-4k/ diff --git a/docs/mbedtls-tls-footprint.md b/docs/mbedtls-tls-footprint.md index 0985c4f6..70ea6db1 100644 --- a/docs/mbedtls-tls-footprint.md +++ b/docs/mbedtls-tls-footprint.md @@ -136,6 +136,36 @@ fix was `rm -rf` the package and `pio pkg install` to re-download stock. A prepe search path keeps the change scoped to one env, because the linker takes each archive member from the first archive that satisfies an undefined symbol. +### How the archives are distributed + +They are not committed: ~6 MB per architecture, and they have to be rebuilt for every +`platformio/espressif32` bump, so committing them would grow history permanently and go +stale silently. Instead they are published as a release asset and fetched on demand: + +``` +scripts/fetch_mbedtls_4k.sh esp32s3 # download + verify against the manifest +MESHCORE_REDUCED_TLS=1 pio run -e Heltec_v3_repeater_observer_mqtt +``` + +- `scripts/mbedtls_4k_manifest.txt` — per-arch sha256 of each archive. **Update it on every + platform bump**, together with the published asset. +- `scripts/fetch_mbedtls_4k.sh` — downloads into `.mbedtls-4k//` (gitignored) and + verifies. `MBEDTLS_4K_LOCAL=` copies from a local build tree instead of downloading. +- `scripts/mbedtls_4k.py` — wired into `esp32_base.extra_scripts`, but a **no-op unless + `MESHCORE_REDUCED_TLS=1`**, so ordinary builds need no artifact and behave as before. + +The opt-in path is deliberately loud, because both ways this can go wrong produce a +firmware that looks correct and silently lacks the change: + +| failure | what happens without a guard | guard | +|---|---|---| +| directory missing or partial | linker ignores an unusable `-L` and resolves mbedTLS from the framework | pre-build: hard error | +| archives stale after a platform bump | links the wrong build | pre-build: sha256 vs manifest | +| `-L` present but outranked | framework archives win, flag is inert | post-link: `firmware.map` must resolve every `libmbed*.a` to `.mbedtls-4k/` | + +That last one is the reason the post-link check exists rather than trusting the flag: a +build flag reaching the compiler proves nothing about what got linked. + ## How to verify it worked 1. `strings`/`grep` the new `sdkconfig.h` for the four settings. diff --git a/platformio.ini b/platformio.ini index 6c012f1e..73ce373b 100644 --- a/platformio.ini +++ b/platformio.ini @@ -64,6 +64,8 @@ platform = platformio/espressif32@6.11.0 monitor_filters = esp32_exception_decoder extra_scripts = pre:scripts/generate_webconfig_html.py +; No-op unless MESHCORE_REDUCED_TLS=1; see docs/mbedtls-tls-footprint.md. + pre:scripts/mbedtls_4k.py merge-bin.py build_flags = ${arduino_base.build_flags} -D ESP32_PLATFORM diff --git a/platformio.local.ini.hold b/platformio.local.ini.hold deleted file mode 100644 index e0d89578..00000000 --- a/platformio.local.ini.hold +++ /dev/null @@ -1,15 +0,0 @@ -; Local-only override (gitignored) pointing the Heltec V3 observer env at a custom -; framework whose mbedTLS archives were rebuilt with an asymmetric TLS record buffer: -; CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y -; CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN=16384 (unchanged — a peer may send a 16 KiB record) -; CONFIG_MBEDTLS_SSL_OUT_CONTENT_LEN=4096 (was 16384) -; -; Expected: ~12 KiB less internal DRAM per TLS connection, ~24 KiB across two broker slots. -; Built from the shipped sdkconfig verbatim plus those three lines, so the archives differ -; only by this change. See docs/mbedtls-tls-footprint.md. -; -; Absolute path, hence local-only: not committable. - -[env:Heltec_v3_repeater_observer_mqtt] -platform_packages = - framework-arduinoespressif32 @ file:///Users/adam/framework-arduinoespressif32-tlsfix diff --git a/scripts/fetch_mbedtls_4k.sh b/scripts/fetch_mbedtls_4k.sh new file mode 100755 index 00000000..f1e43c77 --- /dev/null +++ b/scripts/fetch_mbedtls_4k.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Fetch the reduced-TLS mbedTLS archives into .mbedtls-4k//. +# +# These archives are built from the shipped sdkconfig plus three lines +# (CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y, IN_CONTENT_LEN 16384, +# OUT_CONTENT_LEN 4096) and save ~12 KiB of internal DRAM per TLS connection. +# See docs/mbedtls-tls-footprint.md for the rationale and the build recipe. +# +# They are not committed: ~6 MB per architecture, and they must be rebuilt for +# every platform bump, so they are published as a release asset keyed on the +# espressif32 platform version instead. +# +# scripts/fetch_mbedtls_4k.sh [arch] # default: esp32s3 +# +# Set MBEDTLS_4K_LOCAL to skip the download and copy from a local build tree: +# MBEDTLS_4K_LOCAL=~/mbedtls-4k-esp32s3/staged scripts/fetch_mbedtls_4k.sh +set -euo pipefail + +ARCH="${1:-esp32s3}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEST="$REPO_ROOT/.mbedtls-4k/$ARCH" +MANIFEST="$REPO_ROOT/scripts/mbedtls_4k_manifest.txt" +BASE_URL="${MBEDTLS_4K_BASE_URL:-https://github.com/agessaman/MeshCore/releases/download/mbedtls-4k}" + +if [ ! -f "$MANIFEST" ]; then + echo "error: missing $MANIFEST" >&2 + exit 1 +fi + +# Manifest lines: . Blank lines and # comments ignored. +expected="$(awk -v a="$ARCH" '$1 == a && $0 !~ /^#/ {print $2" "$3}' "$MANIFEST")" +if [ -z "$expected" ]; then + echo "error: no manifest entries for arch '$ARCH'" >&2 + echo "known arches: $(awk '$0 !~ /^#/ && NF {print $1}' "$MANIFEST" | sort -u | tr '\n' ' ')" >&2 + exit 1 +fi + +mkdir -p "$DEST" + +if [ -n "${MBEDTLS_4K_LOCAL:-}" ]; then + echo "copying from $MBEDTLS_4K_LOCAL" + while read -r _sha name; do + cp "$MBEDTLS_4K_LOCAL/$name" "$DEST/$name" + done <<< "$expected" +else + TARBALL="mbedtls-4k-$ARCH.tar.gz" + echo "downloading $BASE_URL/$TARBALL" + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + curl -fsSL "$BASE_URL/$TARBALL" -o "$tmp/$TARBALL" + tar -xzf "$tmp/$TARBALL" -C "$tmp" + while read -r _sha name; do + # Accept the archive whether or not the tarball has a leading directory. + found="$(find "$tmp" -name "$name" -type f | head -1)" + if [ -z "$found" ]; then + echo "error: $name missing from $TARBALL" >&2 + exit 1 + fi + cp "$found" "$DEST/$name" + done <<< "$expected" +fi + +# Verify every archive against the manifest. A wrong or truncated archive would +# otherwise link silently and produce a firmware without the reduced buffers. +cd "$DEST" +if command -v shasum >/dev/null 2>&1; then + echo "$expected" | shasum -a 256 -c - +else + echo "$expected" | sha256sum -c - +fi + +echo "ok: $ARCH archives verified in $DEST" diff --git a/scripts/mbedtls_4k.py b/scripts/mbedtls_4k.py new file mode 100644 index 00000000..9ea01d3b --- /dev/null +++ b/scripts/mbedtls_4k.py @@ -0,0 +1,144 @@ +"""Link the reduced-TLS mbedTLS archives, and prove they were actually linked. + +Opt in per build with MESHCORE_REDUCED_TLS=1. Off by default, so an ordinary build +needs no 6 MB artifact and behaves exactly as before. + + MESHCORE_REDUCED_TLS=1 pio run -e Heltec_v3_repeater_observer_mqtt + +The archives lower the mbedTLS outbound record buffer from 16 KiB to 4 KiB, saving +~12 KiB of internal DRAM per TLS connection on non-PSRAM observers. The inbound +buffer stays at 16 KiB, so the contiguous allocation a handshake needs is unchanged +— this buys headroom, it does not move that floor. See docs/mbedtls-tls-footprint.md. + +Two failure modes this guards against, both of which produce a firmware that looks +fine and silently lacks the change: + + - a -L pointing at a missing or partial directory. The linker ignores an + unusable search path and quietly resolves mbedTLS from the framework instead. + - archives that do not match the manifest, e.g. left over from an earlier + platform version. + +So the opt-in path verifies every archive by sha256 before the build, and after the +link re-reads firmware.map to confirm every libmbed*.a came from our directory. +""" +Import("env") + +import hashlib +import os +import sys + +REQUIRED = ("libmbedcrypto.a", "libmbedtls_2.a", "libmbedtls.a", "libmbedx509.a") + + +def _fail(msg): + print("\n*** reduced-TLS build failed ***", file=sys.stderr) + print(msg, file=sys.stderr) + print( + "\nFetch the archives with: scripts/fetch_mbedtls_4k.sh " + "\nOr build without them by unsetting MESHCORE_REDUCED_TLS.", + file=sys.stderr, + ) + env.Exit(1) + + +def _sha256(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _manifest(project_dir, arch): + path = os.path.join(project_dir, "scripts", "mbedtls_4k_manifest.txt") + if not os.path.isfile(path): + _fail("missing scripts/mbedtls_4k_manifest.txt") + wanted = {} + with open(path) as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) == 3 and parts[0] == arch: + wanted[parts[2]] = parts[1] + return wanted + + +if os.environ.get("MESHCORE_REDUCED_TLS", "") not in ("1", "true", "yes"): + Return() + +project_dir = env.subst("$PROJECT_DIR") +arch = env.BoardConfig().get("build.mcu", "") +if not arch: + _fail("could not determine board MCU, so cannot pick an archive set") + +staged = os.path.join(project_dir, ".mbedtls-4k", arch) +if not os.path.isdir(staged): + _fail("no archives for %s at %s" % (arch, staged)) + +wanted = _manifest(project_dir, arch) +if not wanted: + _fail("manifest has no entries for arch '%s'" % arch) + +for name in REQUIRED: + archive = os.path.join(staged, name) + if not os.path.isfile(archive): + _fail("missing %s" % archive) + if name not in wanted: + _fail("%s is not in the manifest for %s" % (name, arch)) + actual = _sha256(archive) + if actual != wanted[name]: + _fail( + "%s does not match the manifest\n expected %s\n actual %s\n" + "Rebuild it for this platform version, or re-run the fetch script." + % (archive, wanted[name], actual) + ) + +# Prepend so these satisfy mbedTLS symbols ahead of the framework's own copies: +# the linker takes each archive member from the first archive that resolves it. +env.Prepend(LIBPATH=[staged]) +print("reduced-TLS: linking mbedTLS from %s (verified)" % staged) + + +def _verify_map(source, target, env): + """Confirm every mbedTLS archive in the link came from our directory.""" + map_path = os.path.join(env.subst("$BUILD_DIR"), "firmware.map") + if not os.path.isfile(map_path): + print("reduced-TLS: WARNING no firmware.map, cannot confirm the link", + file=sys.stderr) + return + # The map records whatever the linker was given, which for a -L hit is a path + # relative to the linker's cwd (the project dir). Resolve before comparing, or + # every one of our own archives reads as stray. + staged_real = os.path.realpath(staged) + stray = set() + seen = set() + with open(map_path, errors="replace") as fh: + for line in fh: + for token in line.split(): + if "libmbed" not in token or ".a" not in token: + continue + path = token.split("(")[0] + base = os.path.basename(path) + if not base.startswith("libmbed") or not base.endswith(".a"): + continue + seen.add(base) + resolved = os.path.realpath(os.path.join(project_dir, path)) + if os.path.dirname(resolved) != staged_real: + stray.add(path) + if stray: + print("\n*** reduced-TLS: archives linked from the WRONG place ***", + file=sys.stderr) + for path in sorted(stray): + print(" " + path, file=sys.stderr) + env.Exit(1) + if not seen: + print("reduced-TLS: WARNING firmware.map names no mbedTLS archive", + file=sys.stderr) + return + print("reduced-TLS: confirmed %d archives linked from %s" + % (len(seen), staged)) + + +env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", _verify_map) diff --git a/scripts/mbedtls_4k_manifest.txt b/scripts/mbedtls_4k_manifest.txt new file mode 100644 index 00000000..06c0f385 --- /dev/null +++ b/scripts/mbedtls_4k_manifest.txt @@ -0,0 +1,12 @@ +# Reduced-TLS mbedTLS archives (OUT_CONTENT_LEN 4096, IN_CONTENT_LEN 16384). +# Format: +# +# Rebuild these for every espressif32 platform bump — the archives must match the +# rest of the framework they link against. Built as of platformio/espressif32@6.11.0 +# from the recipe in docs/mbedtls-tls-footprint.md. +# +# Fetch with: scripts/fetch_mbedtls_4k.sh esp32s3 +esp32s3 01629f635b33ffa2c1fdfcd8ac52327cd3a92e8d4e7c9a32d92974e0cfdfe398 libmbedcrypto.a +esp32s3 07e7a09847589fefc35bc8d2f739d556535d535acd7185a9824b2af8edd5b05a libmbedtls_2.a +esp32s3 e12bcc8d76a368e819987f266e73c265178d6b6675f05d0e29014bb543c402af libmbedtls.a +esp32s3 c1e10324e19f6d7763737f72d9032cec832bdfce60c7a7d381a5fcfc7dcad573 libmbedx509.a From daec2e4edd38c573cca109bb8b8b8f73cc0522dd Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 10 Aug 2026 16:13:15 -0700 Subject: [PATCH 08/93] =?UTF-8?q?fix(mqtt):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20stopped=20clients,=20late=20allocation,=20fail-open=20map=20?= =?UTF-8?q?check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects found reviewing the preceding commits. 1. reconnectSlotClient() stranded a STOPPED client, reintroducing the very bug this branch fixes. It only rebuilt when isStarted() was true and otherwise fell through to reconnect(), which is a documented no-op on a stopped client — so nothing restarted it, at any rung, including the breaker probe. The WiFi-transition teardown reaches exactly this state: it calls the hard disconnect(), clearing _started while initial_connect_done stays set, so after WiFi returned the slot could never come back. Now a stopped client is started with connect() before the rebuild/reuse decision is considered. The post-NTP credential refresh had the same exposure — it called reconnect() directly — so it now goes through the helper too, still reusing the transport since its fault is stale credentials, not the transport. 2. Allocating the neighbors buffer on first use let a stopped bridge allocate. A neighbour discovery started before a stop can complete after it, and neither caller rechecks bridge state, so requestPublishNeighbors() would allocate 4 KB after releaseRuntimeBuffers() had already run and strand _neighbors_publish_pending with no task to consume it. end() then returns early on !_initialized, retaining the buffer until a later begin/end or a reboot. Guarded on isRunning(), the same flag end() checks. The release/acquire handoff itself was confirmed sound: the allocation and copy precede the release store, and the task loop reads the pointer only after its acquire load, so a half-published pointer is not observable. 3. The post-link map check failed open, contradicting the fail-closed claim in its own commit message. A missing map, an unrecognised map format, or a partial archive list each warned and passed; and it hardcoded firmware.map while the post-action target used ${PROGNAME}, so a renamed program could inspect a stale or absent file and still succeed. All four now fail the build, and it requires every one of the four archives to appear rather than at least one. Rebuilt Heltec_v3_repeater_observer_mqtt, Heltec_v3_repeater and heltec_v4_repeater_observer_mqtt; the opt-in path still reports all 4 archives linked from .mbedtls-4k/. (cherry picked from commit 5b5f076e5e165997e8050f2be061c7c67340fcf7) --- scripts/mbedtls_4k.py | 41 ++++++++++++++++++++++++------ src/helpers/bridges/MQTTBridge.cpp | 26 ++++++++++++++++++- src/helpers/bridges/MQTTBridge.h | 3 +++ 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/scripts/mbedtls_4k.py b/scripts/mbedtls_4k.py index 9ea01d3b..6718cae4 100644 --- a/scripts/mbedtls_4k.py +++ b/scripts/mbedtls_4k.py @@ -102,11 +102,27 @@ print("reduced-TLS: linking mbedTLS from %s (verified)" % staged) def _verify_map(source, target, env): - """Confirm every mbedTLS archive in the link came from our directory.""" - map_path = os.path.join(env.subst("$BUILD_DIR"), "firmware.map") + """Confirm every mbedTLS archive in the link came from our directory. + + Fails closed. Anything that stops this from *proving* the link — no map, an + unparsable map, a short archive list — is a failure, not a warning. A warning + here would leave exactly the hole the check exists to close: an opt-in build + that succeeds while silently linking the framework's 16 KiB buffers. + """ + # Derive the map name from PROGNAME rather than hardcoding firmware.map, so a + # renamed program cannot leave us inspecting a stale or absent file. + map_path = os.path.join(env.subst("$BUILD_DIR"), + env.subst("${PROGNAME}") + ".map") if not os.path.isfile(map_path): - print("reduced-TLS: WARNING no firmware.map, cannot confirm the link", + legacy = os.path.join(env.subst("$BUILD_DIR"), "firmware.map") + map_path = legacy if os.path.isfile(legacy) else map_path + if not os.path.isfile(map_path): + print("\n*** reduced-TLS: no linker map at %s ***" % map_path, file=sys.stderr) + print("Cannot prove the reduced-TLS archives were linked. Ensure the env " + "emits a map (-Wl,-Map), or unset MESHCORE_REDUCED_TLS.", + file=sys.stderr) + env.Exit(1) return # The map records whatever the linker was given, which for a -L hit is a path # relative to the linker's cwd (the project dir). Resolve before comparing, or @@ -133,12 +149,21 @@ def _verify_map(source, target, env): for path in sorted(stray): print(" " + path, file=sys.stderr) env.Exit(1) - if not seen: - print("reduced-TLS: WARNING firmware.map names no mbedTLS archive", - file=sys.stderr) return - print("reduced-TLS: confirmed %d archives linked from %s" - % (len(seen), staged)) + # Every required archive must appear. Seeing only some of them means the rest + # resolved somewhere this parse did not recognise, which is not proof of anything. + missing = [name for name in REQUIRED if name not in seen] + if missing: + print("\n*** reduced-TLS: %s names only %d of %d archives ***" + % (os.path.basename(map_path), len(seen), len(REQUIRED)), + file=sys.stderr) + print(" missing: " + ", ".join(missing), file=sys.stderr) + print("Either the map format changed or mbedTLS was resolved elsewhere; " + "the reduced buffers cannot be assumed.", file=sys.stderr) + env.Exit(1) + return + print("reduced-TLS: confirmed all %d archives linked from %s" + % (len(REQUIRED), staged)) env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", _verify_map) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index da201596..d6dcc2b4 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1992,6 +1992,23 @@ void MQTTBridge::teardownSlot(int index) { slot.last_deferred_log_ms = 0; } +// A stopped client needs connect(): reconnect() is a documented no-op on one, so reaching +// it here would strand the slot. The WiFi-transition teardown stops a client while leaving +// initial_connect_done set, so the ladder does see this state. +void MQTTBridge::reconnectSlotClient(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[index]; + if (slot.client == nullptr) return; + + if (!slot.client->isStarted()) { + MQTT_DEBUG_PRINTLN("MQTT%d start (client was stopped)", index + 1); + slot.client->connect(); + return; + } + slot.client->reconnect(); +} + + void MQTTBridge::maintainSlotConnections() { if (!_identity) return; @@ -3637,6 +3654,11 @@ void MQTTBridge::requestPublishNeighbors(const char* json, size_t len) { // Drop a new snapshot while one is still being published (Core 0 clears the // flag when done). Acquire pairs with the task loop's release store. if (_neighbors_publish_pending.load(std::memory_order_acquire)) return; + // Allocating here means a stopped bridge must not: a discovery started before the + // stop can finish after it, and releaseRuntimeBuffers() has already run, so the + // allocation would be retained with no task left to consume it. isRunning() is the + // same flag end() guards on. + if (!isRunning()) return; // Allocated on first use so a node with neighbors off never pays the 4 KB. // Cross-core safe: the release store below publishes this pointer, and the task // loop only reads it after the matching acquire load. @@ -3959,7 +3981,9 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { if (createSlotAuthToken(i)) { _slots[i].client->setCredentials(_jwt_username, _slots[i].auth_token); } - _slots[i].client->reconnect(); + // Reuse the transport — the fault is stale credentials, not the transport — + // but via the helper, so a stopped client is started rather than no-opped. + reconnectSlotClient(i); } } } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 6056a46a..cbc82482 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -447,6 +447,9 @@ private: int activatedSlotCount() const; bool canActivateSlot(int index) const; void teardownSlot(int index); // Disconnect the slot's client (keeps the object alive) + // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a + // no-op on a stopped client). See the definition. + void reconnectSlotClient(int index); void maintainSlotConnections(); // Maintain all slot connections (token renewal, reconnect) void maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted); bool createSlotAuthToken(int index); // Create/renew JWT token for a slot From f4ba55be7a97fc8f61523b5ec8ea964d268e3e4b Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 11 Aug 2026 08:14:39 -0700 Subject: [PATCH 09/93] fix(mqtt): stop bouncing a live waev session to renew its token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waev's operator confirmed on 2026-08-11 that their servers do not disconnect a client when its JWT passes exp — a 60-minute token can hold a session open for hours. The renewal path assumed the opposite, in as many words: the comment at the bounce called the renewal buffer "the ONLY margin between 'device re-authenticates' and 'broker enforces exp and FIN-closes the session mid-stream' — observed on the waev preset". That premise made waev expensive, because waev is the only preset with a short token_lifetime (3300 s; every other is 0, meaning the 24 h default). It was therefore the only slot bouncing often: measured every ~47 minutes, about 30 times a day per device. And the bounce's re-handshake is where contiguous internal DRAM goes — one renewal traced on hardware took the largest free block from 27,124 to 16,372 B, below the 16,384 B mbedTLS inbound record buffer, after which that slot could not re-handshake at all. The teardown and the credential update cost nothing; the handshake costs everything. So for a broker that leaves live sessions alone, refresh the credentials in place and let the next genuine reconnect use them. That path already existed for the "token renewed but old one still valid" case; this just stops treating imminent expiry as a reason to tear down a healthy connection. mqttPresetEnforcesTokenExp() defaults to true and is keyed by preset name rather than a new struct field: adding a field would mean re-ordering a dozen positional initialisers, where a mistake is silent, and the wrong default costs an outage rather than a re-handshake. Custom and audience-only slots have no preset and are treated as enforcing. Our own logs already argued against the premise and we had not noticed: across 14 multi-device outages (10 hitting all four devices) the drops landed within ~3 s of each other, on devices whose independent boot times gave them independent token issue times. Independent expiries cannot align that tightly, so exp enforcement was never a good explanation for them. Unverified on hardware yet — the operator's statement is second-hand. Next: apply to one board only and confirm the session survives past exp, that a later reconnect still authenticates, and that the ~47-minute 27,124<->16,372 oscillation stops. (cherry picked from commit 27bd05a17b9303b158feec7dab60af2fe128f5ce) --- src/helpers/MQTTPresets.h | 18 ++++++++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 11 ++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index b5f342c8..1083026d 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -47,6 +47,24 @@ struct MQTTPresetDef { // Braces match topic placeholders ({device}/{iata}); never send this string to the broker. static const char MQTT_USERPASS_USERNAME_PUBKEY[] = "{pubkey}"; +// True when the broker tears down a live session once its JWT passes exp, so the +// renewal must proactively bounce the connection to present a fresh token. +// +// Default true, because getting this wrong the safe way costs a re-handshake and +// getting it wrong the unsafe way costs an outage. waev is the exception: its +// operator confirmed (2026-08-11) that their servers do not disconnect on expiry, +// so a live session there needs only its credentials refreshed for the next +// reconnect. waev is also the only preset with a short token_lifetime, so it was +// the only one bouncing often — every ~47 min, and each bounce's re-handshake can +// cost ~10 KB of contiguous internal DRAM on a non-PSRAM board. +// +// Keyed by name rather than a struct field on purpose: adding a field would mean +// re-ordering a dozen positional initialisers below, where a mistake is silent. +static inline bool mqttPresetEnforcesTokenExp(const MQTTPresetDef* preset) { + if (!preset || !preset->name) return true; // custom/audience slots: assume enforced + return strcmp(preset->name, "waev") != 0; +} + static inline bool mqttPresetUsesDevicePubkeyUsername(const MQTTPresetDef* preset) { return preset && preset->auth_type == MQTT_AUTH_USERPASS && preset->userpass_username && diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index d6dcc2b4..906f525a 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2148,7 +2148,16 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns (time_synced && old_token_expires_at >= 1000000000 && current_time >= (old_token_expires_at - renewal_buffer)); - if (old_token_expired_or_imminent || !slot.client->connected()) { + // Only bounce for exp if this broker actually enforces it. A broker that + // leaves live sessions alone past expiry needs the fresh token at the next + // reconnect, not now, and the bounce's re-handshake is where contiguity goes. + const bool exp_forces_bounce = + old_token_expired_or_imminent && mqttPresetEnforcesTokenExp(slot.preset); + if (!exp_forces_bounce && old_token_expired_or_imminent && slot.client->connected()) { + MQTT_DEBUG_PRINTLN("MQTT%d token renewed, no bounce (broker does not enforce exp)", + index + 1); + } + if (exp_forces_bounce || !slot.client->connected()) { // Disconnect + reconnect with fresh credentials, reusing existing client // to avoid internal heap leak/fragmentation from destroy/create cycles MQTT_DEBUG_PRINTLN("MQTT%d token renewal: reconnecting with fresh credentials", index + 1); From 88c824c1755f36386ef0d107dae0a1772699f288 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 11 Aug 2026 18:39:58 -0700 Subject: [PATCH 10/93] docs(mqtt): correct what the WS buffer padding actually fixes The comment claimed the (WS_BUFFER_SIZE + 1) padding makes an oversized upgrade response "fail cleanly (Upgrade header not found)". Reading release/v4.4 transport_ws.c against release/v5.3 shows it does not. v4.4's response loop is } while (NULL == strstr(ws->buffer, "\r\n\r\n") && header_len < WS_BUFFER_SIZE); so it also exits when the buffer fills without the terminator, and the code then looks for "Sec-WebSocket-Accept:" and returns 0 if it is present. That header appears early in a response, so an oversized header block yields a BOGUS SUCCESS rather than a clean failure: the unread remainder stays queued on the socket and is delivered as the first post-upgrade read, where the deframer parses HTTP bytes as a WebSocket frame header. Observed on hardware 2026-08-12 on a Heltec V4: a Cloudflare Page Shield CSP report-uri header pushed the 101 response past the buffer, and the tail of that header ("csp-reporting.cloudflare.com/cdn-cgi/script_monitor/report?") reached the MQTT layer as payload, surfacing as "Invalid MSG_TYPE response: 3" (0x35 = '5', high nibble 3). The padding's real and only value is preventing the one-byte overflow of the heap canary, which is still worth having. Narrow the comment to that claim and record where the parser fix has to come from: IDF 5.2+ requires the "\r\n\r\n" delimiter, memmoves the bytes following it, and fails cleanly when the buffer fills. It cannot be patched here, since transport_ws.c ships precompiled in libtcp_transport.a on Arduino 2.x. No functional change. (cherry picked from commit 9894e65e8ad961704d430a12a065baffda353f50) --- src/helpers/ESP32WsTransportFix.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/helpers/ESP32WsTransportFix.cpp b/src/helpers/ESP32WsTransportFix.cpp index 2e14bb33..a69e0b08 100644 --- a/src/helpers/ESP32WsTransportFix.cpp +++ b/src/helpers/ESP32WsTransportFix.cpp @@ -33,13 +33,24 @@ // Fix: [esp32_base] adds `-Wl,--wrap=esp_transport_ws_init`, so every // creation of a WS transport (esp-mqtt does one per wss slot) is routed // through __wrap_esp_transport_ws_init below, which replaces the freshly -// allocated 1024-byte buffer with a (WS_BUFFER_SIZE + 1)-byte one. The -// out-of-bounds index WS_BUFFER_SIZE then lands on our extra byte and the -// handshake fails cleanly ("Upgrade" header not found) instead of corrupting -// the heap. Upstream fixed this in ESP-IDF 5.x, so this file compiles to a +// allocated 1024-byte buffer with a (WS_BUFFER_SIZE + 1)-byte one, so the +// out-of-bounds index WS_BUFFER_SIZE lands on our extra byte instead of the +// heap canary. Upstream fixed this in ESP-IDF 5.x, so this file compiles to a // pass-through there and can be deleted (together with the --wrap flag) when // the fork moves to Arduino core 3.x. // +// Scope: this stops the heap corruption and NOTHING else. It does not make an +// oversized response fail the handshake — v4.4's read loop also exits on +// `header_len < WS_BUFFER_SIZE` going false, and then still accepts the +// connection if "Sec-WebSocket-Accept:" was inside those first bytes (it comes +// early, so it usually is). ws_connect() therefore returns success on a partial +// header block and the unread remainder arrives as the first "payload" read, +// where the deframer parses HTTP bytes as a frame header. Observed on hardware +// 2026-08-12: a large Cloudflare CSP header produced exactly that, surfacing as +// `Invalid MSG_TYPE response: 3`. Only IDF 5.2+ fixes it (it requires the +// "\r\n\r\n" delimiter, preserves the bytes after it, and fails cleanly when the +// buffer fills); it cannot be patched here because transport_ws.c is precompiled. +// // transport_ws_t below is copied verbatim from ESP-IDF release/v4.4 // transport_ws.c (the struct is file-private, so it is not in any shipped // header). Source fidelity was verified against the shipped binary: addr2line From c0c823b6b004c1d3591376f535764c3576cfea56 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 12 Aug 2026 10:03:59 -0700 Subject: [PATCH 11/93] fix(mqtt): reuse a still-valid JWT on ordinary reconnects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ordinary backoff reconnect and every circuit-breaker probe minted a fresh JWT and re-applied credentials, with no check of whether the existing token was still valid. setCredentials() always dirties the esp-mqtt config, so reconnect() then called esp_mqtt_set_config() as well. On a flapping broker that is a signing plus a configuration-copy cycle on every retry, and these observers see ~38 genuine reconnects/day per slot. The no-bounce renewal change (27bd05a1) only stopped the proactive renewal from tearing down a live session; it left this retry path untouched, which is why a soak shows renewals neither firing nor failing for hours while drops continue — each reconnect silently re-mints and pushes the expiry out. Reuse the credentials when their validity is provable and refresh them otherwise. canReuseJwtForReconnect() lives with the other policy predicates so it is host-testable, and it establishes current_time < token_expires_at before subtracting: token_expires_at is unsigned, so an already-expired token would otherwise wrap to ~4e9 seconds and read as valid for decades. The >= kMinimumValidEpoch term also rejects the 0 that a failed renewal writes. Minting stays the default for every uncertain case — unsynced clock, missing or insane expiry, empty token, or an expiry inside kJwtReconnectSafetyMarginSecs (60 s), which covers the handshake itself. Two paths still always mint, deliberately: - The circuit-breaker probe. It is the recovery of last resort for a slot that has already failed repeatedly, quite possibly on auth, and it runs once per 30 minutes — so a fresh token there costs nothing worth counting against keeping that path guaranteed-clean. - Any slot whose last error was a broker refusal. Before this change, minting on every retry accidentally recovered from server-side credential invalidation: key rotation, revocation, broker clock skew, or an audience change after a reconfigure. Reuse would have retried a rejected credential until it neared expiry — up to 24 h for every preset that leaves token_lifetime at the default. onError already detects MQTT_ERROR_TYPE_CONNECTION_REFUSED and only logged it; it now also sets a per-slot force-mint flag, cleared on a successful connect and wherever the credentials it referred to are blanked. The flag is volatile because the esp-mqtt callback sets it and the bridge loop consumes it. The reconnect log line reports the decision and its outcome — REUSE, MINT with a reason, and OK/FAILED for the mint — because a silently failed mint is the case most likely to end in an auth refusal. It never prints the token. Host tests cover the reuse boundary: exact margin, already-expired, expiry 0, sub-epoch expiry, empty token, unsynced clock, and the force-mint override. --- src/helpers/MQTTConnectionPolicy.h | 15 ++++ src/helpers/bridges/MQTTBridge.cpp | 79 +++++++++++++++---- src/helpers/bridges/MQTTBridge.h | 4 + .../test_mqtt_connection_policy.cpp | 16 ++++ 4 files changed, 97 insertions(+), 17 deletions(-) diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index 0ab4be12..ebbf4975 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -17,6 +17,7 @@ static const uint8_t kMaxFailuresAtMaxBackoff = 3; static const uint32_t kDefaultJwtLifetimeSecs = 86400UL; static const uint32_t kMaxJwtStaggerSecs = 300UL; static const uint32_t kMinimumValidEpoch = 1000000000UL; +static const uint32_t kJwtReconnectSafetyMarginSecs = 60UL; static const uint32_t kJwtClockThreshold = 1735689600UL; // 2025-01-01 UTC // A wall clock at or past this instant was set from a real source (NTP or an // admin); anything earlier is the firmware's unset-clock default (1715770351, @@ -157,6 +158,20 @@ static inline bool tokenNeedsRenewal(bool time_synced, uint32_t current_time, return current_time >= token_expires_at - renewal_buffer_secs; } +// A reconnect may keep its credentials only when their validity is known to +// outlast the next handshake; uncertainty refreshes them before reconnecting. +static inline bool canReuseJwtForReconnect(bool time_synced, bool has_token, + bool force_mint, + uint32_t current_time, + uint32_t token_expires_at) { + return time_synced && + has_token && + !force_mint && + token_expires_at >= kMinimumValidEpoch && + current_time < token_expires_at && + (token_expires_at - current_time) > kJwtReconnectSafetyMarginSecs; +} + static inline bool renewalAttemptAllowed(uint32_t now, uint32_t last_attempt) { return elapsedMs(now, last_attempt) >= kRenewalThrottleMs; } diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 906f525a..5a4cc917 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -704,6 +704,7 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg _slots[i].last_log_time = 0; _slots[i].port = 1883; _slot_reconfigure_pending[i] = false; + _slot_force_jwt_mint[i] = false; _status_publish_pending[i] = false; } @@ -1601,6 +1602,7 @@ bool MQTTBridge::ensureSlotClient(int index) { slot.client->onConnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); _slots[index].connected = true; + _slot_force_jwt_mint[index] = false; // NOTE: reconnect_backoff / max_backoff_failures are NOT reset here. // A CONNACK alone doesn't prove the link is healthy — a broker that // accepts and then drops within seconds would reset the ladder every @@ -1647,6 +1649,7 @@ bool MQTTBridge::ensureSlotClient(int index) { _slots[index].last_sock_errno = error.esp_transport_sock_errno; _slots[index].last_error_time = millis(); if (error.error_type == MQTT_ERROR_TYPE_CONNECTION_REFUSED) { + _slot_force_jwt_mint[index] = true; // Broker rejected the MQTT CONNECT itself — not a transport failure. // return code: 1=protocol, 2=client-id rejected, 3=server unavailable, // 4=bad username/password, 5=not authorized. Codes 3/4/5 point at a @@ -1801,6 +1804,8 @@ bool MQTTBridge::setupSlot(int index) { slot.max_backoff_failures = 0; slot.circuit_breaker_tripped = false; slot.last_reconnect_attempt = 0; + // The refusal that set this belonged to the credentials being cleared here. + _slot_force_jwt_mint[index] = false; } bool uses_jwt = (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) || slot.audience[0] != '\0'; @@ -1990,6 +1995,8 @@ void MQTTBridge::teardownSlot(int index) { slot.last_reconnect_attempt = 0; slot.last_log_time = 0; slot.last_deferred_log_ms = 0; + // The refusal that set this belonged to the credentials being cleared here. + _slot_force_jwt_mint[index] = false; } // A stopped client needs connect(): reconnect() is a documented no-op on one, so reaching @@ -2203,6 +2210,59 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // persistent clients (Phase 1), the mbedTLS context is allocated once at // startup and the preflight is no longer necessary. + const auto prepareJwtReconnect = [&](bool force_mint, int backoff_level) { + const bool has_token = slot.auth_token && slot.auth_token[0] != '\0'; + const unsigned long expires_at = slot.token_expires_at; + const bool remaining_known = time_synced && + expires_at >= MQTTConnectionPolicy::kMinimumValidEpoch; + const unsigned long remaining_secs = current_time < expires_at + ? expires_at - current_time + : 0; + const bool force_mint_after_refusal = _slot_force_jwt_mint[index]; + force_mint = force_mint || force_mint_after_refusal; + const bool reuse_token = MQTTConnectionPolicy::canReuseJwtForReconnect( + time_synced, has_token, force_mint, static_cast(current_time), + static_cast(expires_at)); + const char* mint_reason = "none"; + if (!reuse_token) { + if (backoff_level < 0) { + mint_reason = "circuit-breaker-probe"; + } else if (force_mint_after_refusal) { + mint_reason = "connection-refused"; + } else if (!time_synced) { + mint_reason = "clock-unsynced"; + } else if (!has_token) { + mint_reason = "empty-token"; + } else if (expires_at < MQTTConnectionPolicy::kMinimumValidEpoch) { + mint_reason = "invalid-expiry"; + } else if (current_time >= expires_at) { + mint_reason = "expired"; + } else { + mint_reason = "safety-margin"; + } + } + const char* mint_result = reuse_token ? "REUSED" : "FAILED"; + if (!reuse_token && createSlotAuthToken(index)) { + slot.client->setCredentials(_jwt_username, slot.auth_token); + mint_result = "OK"; + } + char remaining_text[24]; + if (remaining_known) { + snprintf(remaining_text, sizeof(remaining_text), "%lus", remaining_secs); + } else { + strncpy(remaining_text, "unknown", sizeof(remaining_text)); + } + if (backoff_level >= 0) { + MQTT_DEBUG_PRINTLN("MQTT%d JWT reconnect backoff=%d token=%s mint_reason=%s result=%s remaining=%s", + index + 1, backoff_level, reuse_token ? "REUSE" : "MINT", mint_reason, + mint_result, remaining_text); + } else { + MQTT_DEBUG_PRINTLN("MQTT%d JWT circuit-breaker probe token=%s mint_reason=%s result=%s remaining=%s", + index + 1, reuse_token ? "REUSE" : "MINT", mint_reason, mint_result, + remaining_text); + } + }; + // Periodic probe for circuit-breaker-tripped slots (recovery from transient outages) // Attempts a single reconnect every 30 minutes to see if the server has come back if (slot.circuit_breaker_tripped && !reconnect_attempted) { @@ -2219,13 +2279,7 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _radio ? _radio->getRadioState() : -1, (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); if (slot_uses_jwt) { - // Regenerate or refresh token, then reconnect the persistent client. - // Reaching the ladder at all means setupSlot() ran, so the client object - // and its mbedTLS context are live and no full setup is needed here. - if (createSlotAuthToken(index)) { - slot.client->setCredentials(_jwt_username, slot.auth_token); - MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (fresh token)", index + 1); - } + prepareJwtReconnect(true, -1); slot.client->reconnect(); } else { slot.client->reconnect(); @@ -2259,16 +2313,7 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; if (slot_uses_jwt) { - // Always lightweight reconnect on the persistent client. A stale/expired - // token is handled by regenerating it in place and updating credentials - // — no teardown is needed because the client and its mbedTLS context - // persist for the bridge lifetime. - if (createSlotAuthToken(index)) { - slot.client->setCredentials(_jwt_username, slot.auth_token); - MQTT_DEBUG_PRINTLN("MQTT%d reconnect (fresh token, backoff %d)", index + 1, slot.reconnect_backoff); - } else { - MQTT_DEBUG_PRINTLN("MQTT%d reconnect (token refresh failed, backoff %d)", index + 1, slot.reconnect_backoff); - } + prepareJwtReconnect(false, slot.reconnect_backoff); slot.client->reconnect(); } else { // Non-JWT slots — lightweight reconnect on existing client. diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index cbc82482..bda752a6 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -228,6 +228,10 @@ private: // Pending slot reconfigure: set from CLI (Core 1), processed by MQTT task (Core 0) volatile bool _slot_reconfigure_pending[RUNTIME_MQTT_SLOTS]; + // A broker refusal can invalidate an otherwise clock-valid JWT. The esp-mqtt + // callback sets this and the bridge loop consumes it; byte access is atomic. + volatile bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS]; + // Pending on-connect status publish: set from the onConnect callback (which // runs on the esp-mqtt event task, NOT this bridge task), consumed by the MQTT // task (Core 0). publishStatusToSlot() touches the shared status doc/buffer/ diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index 496a1c43..55c58793 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -137,6 +137,22 @@ TEST(MQTTConnectionPolicy, SyncedClockRenewsInvalidExpiredOrImminentTokens) { EXPECT_TRUE(Policy::tokenNeedsRenewal(true, expires + 1U, expires, 300U)); } +TEST(MQTTConnectionPolicy, JwtReconnectReusesOnlyProvenValidCredentials) { + const uint32_t now = 1735689600U; + const uint32_t usable_expiry = now + Policy::kJwtReconnectSafetyMarginSecs + 1U; + + EXPECT_TRUE(Policy::canReuseJwtForReconnect(true, true, false, now, usable_expiry)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect( + true, true, false, now, Policy::kMinimumValidEpoch - 1U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect( + true, true, false, now, now + Policy::kJwtReconnectSafetyMarginSecs)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, now - 1U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, false, false, now, usable_expiry)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(false, true, false, now, usable_expiry)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, true, now, usable_expiry)); +} + TEST(MQTTConnectionPolicy, RenewalThrottleHasExactBoundaryAndHandlesRollover) { EXPECT_FALSE(Policy::renewalAttemptAllowed(59999U, 0U)); EXPECT_TRUE(Policy::renewalAttemptAllowed(60000U, 0U)); From 6ffadd6e727794e5024d98147136e8b22068359f Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 09:49:16 -0700 Subject: [PATCH 12/93] fix(mqtt): route both reconnect ladders through the stopped-client guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reconnectSlotClient() checks isStarted() and calls connect() instead of reconnect() when the client was stopped, but only the post-NTP stale-token path used it. The ordinary backoff ladder and the circuit-breaker probe called slot.client->reconnect() directly, and esp_mqtt_client_reconnect() is a no-op on a client that is not started. Two ways in. connect() sets _started only when esp_mqtt_client_start() returns ESP_OK while setupSlot() sets initial_connect_done unconditionally, so a start failure under heap pressure stranded the slot. More routinely, the WiFi-drop handler calls disconnect() on every connected slot, which clears _started — after that the ladder issued no-ops forever and the slot never came back. Not caught by the soaks: the log line the guard prints can only come from the NTP path, so a stranded slot and a slot that never entered the state produce identical logs. Observed reconnects were broker-side drops with WiFi up, which leave the client started. The renewal-bounce path keeps its own isStarted() branch — it needs softDisconnect(), which the helper does not do. --- src/helpers/bridges/MQTTBridge.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 5a4cc917..7983f161 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2280,10 +2280,10 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); if (slot_uses_jwt) { prepareJwtReconnect(true, -1); - slot.client->reconnect(); - } else { - slot.client->reconnect(); } + // Via the helper: reconnect() is a no-op on a client the WiFi-drop path + // stopped, which would probe forever without ever starting it. + reconnectSlotClient(index); // If the connect callback fires and sets slot.connected = true, // it will clear circuit_breaker_tripped via the onConnect handler } @@ -2314,12 +2314,13 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _last_slot_reconnect_ms = now_millis; if (slot_uses_jwt) { prepareJwtReconnect(false, slot.reconnect_backoff); - slot.client->reconnect(); } else { // Non-JWT slots — lightweight reconnect on existing client. MQTT_DEBUG_PRINTLN("MQTT%d reconnect (non-JWT, backoff %d)", index + 1, slot.reconnect_backoff); - slot.client->reconnect(); } + // Via the helper: reconnect() is a no-op on a client the WiFi-drop path + // stopped, which would back off forever without ever starting it. + reconnectSlotClient(index); } } } From f64852e223296b36cb75c825af6d1aecdc1636e2 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 09:53:38 -0700 Subject: [PATCH 13/93] fix(mqtt): log a failed client start instead of reporting success connect() logged "MQTT client started." unconditionally, so a failing esp_mqtt_client_start() looked identical to a successful one. That is the one state a later reconnect() cannot recover from, which made it the worst possible line to be wrong. --- lib/PsychicMqttClient/src/PsychicMqttClient.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index 5d037b7e..ba1dc5f1 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -438,8 +438,13 @@ void PsychicMqttClient::connect() if (start_result == ESP_OK) { _started = true; + ESP_LOGI(TAG, "MQTT client started."); + } + else + { + // Reporting success here hides the one state reconnect() cannot recover from. + ESP_LOGE(TAG, "MQTT client failed to start: %s", esp_err_to_name(start_result)); } - ESP_LOGI(TAG, "MQTT client started."); } void PsychicMqttClient::reconnect() From 2173794966f02977b62d4f85c71cef3a9c7931a3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 18:41:20 -0700 Subject: [PATCH 14/93] fix(mqtt): reconnect the NTP-corrected slot instead of no-opping on a live client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit esp_mqtt_client_reconnect() is honoured only from MQTT_STATE_WAIT_RECONNECT, so the post-correction path minted a fresh token, staged it, and then asked a connected client to reconnect — a request esp-mqtt refuses. The slot kept running on the token the clock correction had just proven stale, and recovery became broker-driven rather than the clean reconnect this code intends. Split the three states the path can find: a stopped or waiting client goes through reconnectSlotClient() as before, and a live one has its transport closed first. Only where the broker enforces exp, though — waev leaves live sessions alone past expiry, so bouncing it would spend the 16 KiB contiguous handshake that the rest of this branch exists to avoid. Not a regression: the base branch called client->reconnect() directly at the same site. On ESP32 the block is reachable from the WiFi-reconnect resync and the CLI forced sync; the hourly refresh uses refreshNTP(), which does not carry it. --- src/helpers/bridges/MQTTBridge.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 7983f161..a325d0ab 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4015,8 +4015,8 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { MQTT_DEBUG_PRINTLN("Time synced: %lu (via %s)", epochTime, ntp_server_used); // If slots are already set up and the time jumped significantly (e.g., SNTP - // initially returned stale RTC time, then a later sync corrected it), tear down - // and re-setup all JWT-authenticated slots so they get fresh tokens. + // initially returned stale RTC time, then a later sync corrected it), re-issue + // credentials for every JWT slot the correction left holding an expired token. if (_slots_setup_done && was_ntp_synced) { unsigned long current_time = (unsigned long)time(nullptr); // Every slot, not _max_active_slots: that is a count of positions, never an @@ -4033,12 +4033,27 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { // in place and reconnect the persistent client. No teardown needed. if (_slots[i].token_expires_at > 0 && current_time > _slots[i].token_expires_at) { MQTT_DEBUG_PRINTLN("MQTT%d token stale after time correction, re-creating", i + 1); - if (createSlotAuthToken(i)) { + const bool minted = createSlotAuthToken(i); + if (minted) { + // Staged only; the config is applied by connect()/reconnect() below. _slots[i].client->setCredentials(_jwt_username, _slots[i].auth_token); } - // Reuse the transport — the fault is stale credentials, not the transport — - // but via the helper, so a stopped client is started rather than no-opped. - reconnectSlotClient(i); + if (!_slots[i].client->connected()) { + // Reuse the transport — the fault is stale credentials, not the transport — + // but via the helper, so a stopped client is started rather than no-opped. + reconnectSlotClient(i); + } else if (minted && mqttPresetEnforcesTokenExp(_slots[i].preset)) { + // esp-mqtt honours reconnect() only from WAIT_RECONNECT, so on a live + // session it is refused and the slot keeps running on the stale token. + // Close the transport first — and only here, where the broker enforces + // exp: elsewhere that handshake buys nothing. + MQTT_DEBUG_PRINTLN("MQTT%d bouncing for the corrected-clock token", i + 1); + _slots[i].client->softDisconnect(); + _slots[i].client->reconnect(); + } else if (minted) { + MQTT_DEBUG_PRINTLN("MQTT%d token re-created, no bounce (broker does not enforce exp)", + i + 1); + } } } } From 74a3df320627d3def10d2913309d5e006484d8a1 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 18:41:29 -0700 Subject: [PATCH 15/93] build(tls): bind the reduced-TLS archives to the framework they were built against MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest said "rebuild these for every espressif32 platform bump" and nothing enforced it. mbedtls_4k.py verified the staged archives against the manifest's own hashes, which proves the pair agrees with itself and nothing more: bump the platform without rebuilding and every check still passes while the link takes mbedTLS built against a different IDF. That fails at runtime on struct-layout drift, not at the link, which is the failure the mechanism claimed to prevent. Fingerprint the framework's own mbedTLS archives — the ones ours displace — as stock: lines in the manifest and check them before the build. If the framework moves, the staged pair is stale by construction and the build stops with the replacement hashes printed ready to paste. Stronger than comparing a version string: framework-arduinoespressif32 versions independently of the platform, and its archives are what actually has to match. The lib directory is resolved by trying the layouts espressif32 has used rather than hardcoding one, and failing closed if none holds all four archives. The fetch script ignores the new lines; its known-arches hint skips them so they cannot be reported as architectures. --- scripts/fetch_mbedtls_4k.sh | 5 +- scripts/mbedtls_4k.py | 89 ++++++++++++++++++++++++++++++--- scripts/mbedtls_4k_manifest.txt | 11 ++++ 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/scripts/fetch_mbedtls_4k.sh b/scripts/fetch_mbedtls_4k.sh index f1e43c77..462773e6 100755 --- a/scripts/fetch_mbedtls_4k.sh +++ b/scripts/fetch_mbedtls_4k.sh @@ -28,10 +28,13 @@ if [ ! -f "$MANIFEST" ]; then fi # Manifest lines: . Blank lines and # comments ignored. +# The "platform" and "stock:" lines bind the archives to a framework version; +# they are the build check's business, not ours, and are not architectures. expected="$(awk -v a="$ARCH" '$1 == a && $0 !~ /^#/ {print $2" "$3}' "$MANIFEST")" if [ -z "$expected" ]; then echo "error: no manifest entries for arch '$ARCH'" >&2 - echo "known arches: $(awk '$0 !~ /^#/ && NF {print $1}' "$MANIFEST" | sort -u | tr '\n' ' ')" >&2 + echo "known arches: $(awk '$0 !~ /^#/ && NF && $1 != "platform" && $1 !~ /^stock:/ {print $1}' \ + "$MANIFEST" | sort -u | tr '\n' ' ')" >&2 exit 1 fi diff --git a/scripts/mbedtls_4k.py b/scripts/mbedtls_4k.py index 6718cae4..29cf0e3b 100644 --- a/scripts/mbedtls_4k.py +++ b/scripts/mbedtls_4k.py @@ -10,16 +10,21 @@ The archives lower the mbedTLS outbound record buffer from 16 KiB to 4 KiB, savi buffer stays at 16 KiB, so the contiguous allocation a handshake needs is unchanged — this buys headroom, it does not move that floor. See docs/mbedtls-tls-footprint.md. -Two failure modes this guards against, both of which produce a firmware that looks -fine and silently lacks the change: +Three failure modes this guards against, each of which produces a firmware that looks +fine and is silently wrong: - a -L pointing at a missing or partial directory. The linker ignores an unusable search path and quietly resolves mbedTLS from the framework instead. - archives that do not match the manifest, e.g. left over from an earlier platform version. + - a manifest and archives that agree with each other but not with the installed + framework, which is what a platform bump without a rebuild leaves behind. That + one links cleanly and drifts on struct layout at runtime. -So the opt-in path verifies every archive by sha256 before the build, and after the -link re-reads firmware.map to confirm every libmbed*.a came from our directory. +So the opt-in path verifies every archive by sha256 before the build, checks the +framework's own mbedTLS archives still fingerprint as the ones these were built +against, and after the link re-reads firmware.map to confirm every libmbed*.a came +from our directory. """ Import("env") @@ -54,15 +59,44 @@ def _manifest(project_dir, arch): if not os.path.isfile(path): _fail("missing scripts/mbedtls_4k_manifest.txt") wanted = {} + stock = {} + platform_id = "" with open(path) as fh: for line in fh: line = line.strip() if not line or line.startswith("#"): continue parts = line.split() - if len(parts) == 3 and parts[0] == arch: + if len(parts) == 2 and parts[0] == "platform": + platform_id = parts[1] + elif len(parts) == 3 and parts[0] == "stock:" + arch: + stock[parts[2]] = parts[1] + elif len(parts) == 3 and parts[0] == arch: wanted[parts[2]] = parts[1] - return wanted + return wanted, stock, platform_id + + +# Where each espressif32 generation keeps the archives we displace. First directory +# holding all four wins, so this resolves without knowing which platform is in play. +FRAMEWORK_LIB_DIRS = ( + ("framework-arduinoespressif32", "tools/sdk/%s/lib"), + ("framework-arduinoespressif32-libs", "%s/lib"), + ("framework-arduinoespressif32", "tools/esp32-arduino-libs/%s/lib"), +) + + +def _framework_lib_dir(platform, arch): + for package, layout in FRAMEWORK_LIB_DIRS: + try: + base = platform.get_package_dir(package) + except Exception: + base = None + if not base: + continue + path = os.path.join(base, *(layout % arch).split("/")) + if all(os.path.isfile(os.path.join(path, name)) for name in REQUIRED): + return path + return None if os.environ.get("MESHCORE_REDUCED_TLS", "") not in ("1", "true", "yes"): @@ -77,7 +111,7 @@ staged = os.path.join(project_dir, ".mbedtls-4k", arch) if not os.path.isdir(staged): _fail("no archives for %s at %s" % (arch, staged)) -wanted = _manifest(project_dir, arch) +wanted, stock, manifest_platform = _manifest(project_dir, arch) if not wanted: _fail("manifest has no entries for arch '%s'" % arch) @@ -95,6 +129,47 @@ for name in REQUIRED: % (archive, wanted[name], actual) ) +# Bind the staged archives to the framework they were built against. The hashes above +# only prove the staged files are the ones the manifest names; they say nothing about +# whether the manifest is still current. Bump the platform without rebuilding, and +# every check above still passes while the link takes mbedTLS built against a +# different IDF — a struct-layout drift that corrupts silently at runtime. So +# fingerprint the framework's own archives, the ones being displaced: if those moved, +# the staged pair is stale by construction. +platform = env.PioPlatform() +platform_id = "%s@%s" % (platform.name, platform.version) +stock_dir = _framework_lib_dir(platform, arch) +if stock_dir is None: + _fail("cannot locate the framework's own mbedTLS archives for %s, so the staged " + "ones cannot be tied to a framework version" % arch) +if not stock: + _fail("manifest has no stock:%s fingerprints — it predates the framework binding.\n" + "Add these lines for the framework now installed (%s):\n%s" + % (arch, platform_id, + "\n".join("stock:%s %s %s" % (arch, _sha256(os.path.join(stock_dir, n)), n) + for n in REQUIRED))) + +for name in REQUIRED: + actual = _sha256(os.path.join(stock_dir, name)) + if name not in stock: + _fail("manifest has no stock:%s entry for %s" % (arch, name)) + if actual != stock[name]: + _fail( + "the framework's mbedTLS archives are not the ones these were built against.\n" + " %s\n manifest %s\n installed %s\n" + "Manifest records %s; installed is %s.\n" + "Rebuild the reduced-TLS archives against this framework " + "(docs/mbedtls-tls-footprint.md), then update every hash in the manifest." + % (os.path.join(stock_dir, name), stock[name], actual, + manifest_platform or "no platform", platform_id) + ) + +if manifest_platform and manifest_platform != platform_id: + # Hashes are the check; the version string is orientation. Identical archives + # under a renamed platform are not a compatibility problem. + print("reduced-TLS: manifest says %s, installed is %s — archives match, so this is " + "only a stale label" % (manifest_platform, platform_id)) + # Prepend so these satisfy mbedTLS symbols ahead of the framework's own copies: # the linker takes each archive member from the first archive that resolves it. env.Prepend(LIBPATH=[staged]) diff --git a/scripts/mbedtls_4k_manifest.txt b/scripts/mbedtls_4k_manifest.txt index 06c0f385..5ac608a6 100644 --- a/scripts/mbedtls_4k_manifest.txt +++ b/scripts/mbedtls_4k_manifest.txt @@ -10,3 +10,14 @@ esp32s3 01629f635b33ffa2c1fdfcd8ac52327cd3a92e8d4e7c9a32d92974e0cfdfe398 libmbed esp32s3 07e7a09847589fefc35bc8d2f739d556535d535acd7185a9824b2af8edd5b05a libmbedtls_2.a esp32s3 e12bcc8d76a368e819987f266e73c265178d6b6675f05d0e29014bb543c402af libmbedtls.a esp32s3 c1e10324e19f6d7763737f72d9032cec832bdfce60c7a7d381a5fcfc7dcad573 libmbedx509.a + +# Framework binding, read by scripts/mbedtls_4k.py and ignored by the fetch script. +# "stock:" lines fingerprint the framework's own archives — the ones the entries above +# displace. They are what makes "rebuild on every platform bump" enforceable instead of +# advisory: if the framework's copies move, the entries above are stale by construction +# and the build stops. Regenerate both sets together, never one alone. +platform espressif32@6.11.0 +stock:esp32s3 abdaf759ee17aa697468427b55a86c6c0082ac4cdeb643a63d8b3ac2df324633 libmbedcrypto.a +stock:esp32s3 9a580d2ff7c885e1bf59479a14422e3d10a485ae9083acb5db943df714c0ac35 libmbedtls_2.a +stock:esp32s3 a0f331245feb5cf4fe8d9f99b03d34d64f10e38d8a8d680e35b34ec82aa3cf9e libmbedtls.a +stock:esp32s3 3d56277224b066118b6c48a973ae18fdf8078448bf54b482ac5557699e52f223 libmbedx509.a From b1ceaf01a82ac7884200bb1bb8393d789b400694 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:40:48 -0700 Subject: [PATCH 16/93] fix(mqtt): require real SNTP completion before crediting a fallback server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback configured a server, waited 500 ms, and accepted any plausible system clock as proof that server had answered. It usually has not answered that fast — and the device usually already holds valid time, from an earlier sync or the RTC — so the first server in the list was credited unconditionally, the walk stopped there, _last_ntp_sync was refreshed, and an unreachable host was logged as the source. On the `set mqtt.ntp` validation path, where the single-server walk exists so a typo fails fast, that reported a bad server as OK. Poll sntp_get_sync_status() for SNTP_SYNC_STATUS_COMPLETED instead, which is the layer's own statement that a packet arrived. The status is one-shot — reading COMPLETED clears it — so a result left by an earlier sync would latch on the first poll; clear it before the loop. An implausible epoch after a completed sync now moves to the next server rather than spinning out the remaining attempts against a server that has answered. --- src/helpers/bridges/MQTTBridge.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index a325d0ab..b73ab34b 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -21,6 +21,7 @@ #ifdef ESP_PLATFORM #include +#include #include #include #include @@ -3986,15 +3987,26 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { const char* server = servers[s]; MQTT_DEBUG_PRINTLN("SNTP fallback trying %s...", server); configTime(0, 0, server); + // A plausible clock is not evidence this server answered. The device usually + // already holds valid time here — from an earlier sync, or the RTC — so polling + // time(nullptr) declared the very first server successful without a packet ever + // arriving, stopped the fallback walk there, and refreshed _last_ntp_sync. Worse + // on the `set mqtt.ntp` validation path, where a typo is supposed to fail fast. + // Wait for SNTP itself to report completion. The status is one-shot — reading + // COMPLETED clears it — so drop any result an earlier sync left behind. + sntp_set_sync_status(SNTP_SYNC_STATUS_RESET); for (int i = 0; i < 20; i++) { delay(500); + if (sntp_get_sync_status() != SNTP_SYNC_STATUS_COMPLETED) continue; epochTime = (unsigned long)time(nullptr); if (epochTime >= kMinValidEpoch) { ntp_ok = true; ntp_server_used = server; MQTT_DEBUG_PRINTLN("SNTP fallback succeeded on %s: %lu", server, epochTime); - break; + } else { + MQTT_DEBUG_PRINTLN("SNTP fallback: %s synced an implausible epoch %lu", server, epochTime); } + break; } } } From 168d4a0a8a779c2a5099236331d8df352f5c6443 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:41:04 -0700 Subject: [PATCH 17/93] fix(mqtt): make the accepted NTP epoch authoritative before any JWT work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit syncTimeWithNTP() read an epoch over UDP, called configTime(), set _ntp_synced, and then had the stale-token test and createSlotAuthToken() read time(nullptr) — without anything having put the accepted epoch there. configTime() restarts SNTP and returns; the clock lands whenever a packet does. _rtc->setCurrentTime() looks like it covers this and does not. AutoDiscoverRTCClock::setCurrentTime() writes a detected DS3231/RV3028/PCF8563/ RX8130CE *instead of* delegating to its fallback, and only that fallback (ESP32RTCClock) calls settimeofday(). So on every board carrying an RTC chip — T-Beam Supreme and Station G3 both compile this bridge and both instantiate AutoDiscoverRTCClock — libc kept the pre-correction time, and the correction path tested staleness and minted iat claims against exactly the clock it had just proven wrong. Boards without a chip take the fallback and were unaffected, which is why the soak rig (Heltec V3/V4, no RTC) never showed it. settimeofday() with the accepted epoch first, so the invariant downstream code already assumes actually holds: once _ntp_synced is true, time(nullptr) returns the epoch we accepted. configTime() still follows, to keep future syncs running. --- src/helpers/bridges/MQTTBridge.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index b73ab34b..adbe0cb2 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4013,6 +4013,18 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { #endif if (ntp_ok && ntp_server_used) { + // Take ownership of the system clock here, before anything reads it. configTime() + // only restarts SNTP and returns, and _rtc reaches settimeofday() on exactly one + // path: AutoDiscoverRTCClock writes a detected DS3231/RV3028/PCF8563/RX8130CE chip + // *instead of* its fallback, so on any board carrying one, libc keeps the pre-sync + // time. Everything downstream reads time(nullptr) — the stale-token test below, and + // the iat of every JWT minted from here on — so once _ntp_synced is true that call + // has to already return the epoch we accepted. + struct timeval accepted; + accepted.tv_sec = (time_t)epochTime; + accepted.tv_usec = 0; + settimeofday(&accepted, nullptr); + configTime(0, 0, ntp_server_used); if (_rtc) { From 0d12ec7d796289835090843ce485b9dc7dfedb74 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:41:14 -0700 Subject: [PATCH 18/93] fix(mqtt): defer the stale-token reconnect when the mint fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The corrected-clock path reconnected a disconnected slot whether or not createSlotAuthToken() had produced anything, which re-presented the credentials the correction had just invalidated. Minting fails for recoverable reasons — allocation pressure is treated as recoverable elsewhere in this file — so the path is reachable, and the reconnect it spends is one that cannot succeed. Move the decision into MQTTConnectionPolicy as classifyStaleToken(), where the four outcomes are named and host-tested rather than spelled out in nested conditions: Defer on a failed mint, Reconnect a client that is down, Bounce a live session whose broker enforces exp, KeepAlive one whose broker does not. Deferring leaves the slot to the backoff ladder, which mints again on its next attempt. Covers the reviewer's first four cases. The other two — that a completed SNTP sync is required, and that time(nullptr) reflects the accepted epoch before _ntp_synced flips — are inside MQTTBridge.cpp, which the native env does not compile; locking those down needs a seam around the IDF calls that does not exist yet. --- src/helpers/MQTTConnectionPolicy.h | 23 +++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 24 +++++++------- .../test_mqtt_connection_policy.cpp | 32 +++++++++++++++++++ 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index ebbf4975..b2c57690 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -211,4 +211,27 @@ static inline SlotActivation classifySlotActivation(int slot, const bool* enable : SlotActivation::OverActiveCap; } +// What to do with a slot whose token a clock correction just proved stale. Four +// outcomes, because esp-mqtt accepts a reconnect request only from +// MQTT_STATE_WAIT_RECONNECT: asking a live client to reconnect is refused and +// leaves it running on the stale token, so a live session has to have its +// transport closed first — and that costs a handshake, which is only worth +// spending where the broker actually enforces exp. +enum class StaleTokenAction : uint8_t { + Defer, // no fresh credentials — leave the slot to the backoff ladder + Reconnect, // client is down: start it, or wake one that is waiting + Bounce, // live session the broker will reject: close the transport, then reconnect + KeepAlive, // live session the broker tolerates: stage credentials, keep the handshake +}; + +// A failed mint yields Defer even when the slot is down: reconnecting then would +// re-present the credentials the correction just invalidated. Minting fails for +// recoverable reasons (allocation pressure), and the ladder retries. +static inline StaleTokenAction classifyStaleToken(bool minted, bool connected, + bool broker_enforces_exp) { + if (!minted) return StaleTokenAction::Defer; + if (!connected) return StaleTokenAction::Reconnect; + return broker_enforces_exp ? StaleTokenAction::Bounce : StaleTokenAction::KeepAlive; +} + } // namespace MQTTConnectionPolicy diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index adbe0cb2..24553a5d 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4057,24 +4057,26 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { // in place and reconnect the persistent client. No teardown needed. if (_slots[i].token_expires_at > 0 && current_time > _slots[i].token_expires_at) { MQTT_DEBUG_PRINTLN("MQTT%d token stale after time correction, re-creating", i + 1); - const bool minted = createSlotAuthToken(i); - if (minted) { - // Staged only; the config is applied by connect()/reconnect() below. - _slots[i].client->setCredentials(_jwt_username, _slots[i].auth_token); + const MQTTConnectionPolicy::StaleTokenAction action = + MQTTConnectionPolicy::classifyStaleToken( + createSlotAuthToken(i), _slots[i].client->connected(), + mqttPresetEnforcesTokenExp(_slots[i].preset)); + if (action == MQTTConnectionPolicy::StaleTokenAction::Defer) { + MQTT_DEBUG_PRINTLN("MQTT%d token refresh failed after time correction, " + "deferring to the reconnect ladder", i + 1); + continue; } - if (!_slots[i].client->connected()) { + // Staged only; the config is applied by connect()/reconnect() below. + _slots[i].client->setCredentials(_jwt_username, _slots[i].auth_token); + if (action == MQTTConnectionPolicy::StaleTokenAction::Reconnect) { // Reuse the transport — the fault is stale credentials, not the transport — // but via the helper, so a stopped client is started rather than no-opped. reconnectSlotClient(i); - } else if (minted && mqttPresetEnforcesTokenExp(_slots[i].preset)) { - // esp-mqtt honours reconnect() only from WAIT_RECONNECT, so on a live - // session it is refused and the slot keeps running on the stale token. - // Close the transport first — and only here, where the broker enforces - // exp: elsewhere that handshake buys nothing. + } else if (action == MQTTConnectionPolicy::StaleTokenAction::Bounce) { MQTT_DEBUG_PRINTLN("MQTT%d bouncing for the corrected-clock token", i + 1); _slots[i].client->softDisconnect(); _slots[i].client->reconnect(); - } else if (minted) { + } else { MQTT_DEBUG_PRINTLN("MQTT%d token re-created, no bounce (broker does not enforce exp)", i + 1); } diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index 55c58793..7c7bd7de 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -258,6 +258,38 @@ TEST(SlotActivation, DisabledAndOutOfRangeSlots) { EXPECT_EQ(SlotActivation::Disabled, Policy::classifySlotActivation(0, nullptr, 3, 2)); } +using Policy::StaleTokenAction; + +TEST(StaleToken, ConnectedSlotOnAnExpEnforcingBrokerBounces) { + // The broker will reject the stale token, and esp-mqtt refuses reconnect() from + // CONNECTED, so the transport has to close first. + EXPECT_EQ(StaleTokenAction::Bounce, + Policy::classifyStaleToken(/*minted=*/true, /*connected=*/true, + /*broker_enforces_exp=*/true)); +} + +TEST(StaleToken, ConnectedSlotOnATolerantBrokerKeepsItsSession) { + // waev leaves live sessions alone past exp. Bouncing would spend a 16 KiB + // contiguous handshake to replace a session the broker was not going to drop. + EXPECT_EQ(StaleTokenAction::KeepAlive, + Policy::classifyStaleToken(true, true, /*broker_enforces_exp=*/false)); +} + +TEST(StaleToken, DisconnectedSlotReconnectsRegardlessOfBrokerPolicy) { + EXPECT_EQ(StaleTokenAction::Reconnect, Policy::classifyStaleToken(true, false, true)); + EXPECT_EQ(StaleTokenAction::Reconnect, Policy::classifyStaleToken(true, false, false)); +} + +TEST(StaleToken, FailedMintNeverReconnects) { + // Reconnecting here would re-present the credentials the correction invalidated. + // Every combination defers — including the disconnected one, which is the case + // that previously reconnected on the stale token. + EXPECT_EQ(StaleTokenAction::Defer, Policy::classifyStaleToken(false, false, true)); + EXPECT_EQ(StaleTokenAction::Defer, Policy::classifyStaleToken(false, false, false)); + EXPECT_EQ(StaleTokenAction::Defer, Policy::classifyStaleToken(false, true, true)); + EXPECT_EQ(StaleTokenAction::Defer, Policy::classifyStaleToken(false, true, false)); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 1a01344e717b79b90c278b316616fb9c67d32f24 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:43:18 -0700 Subject: [PATCH 19/93] fix(mqtt): keep the usable-clock fallback the SNTP strictness removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requiring a real SNTP completion took away something the plausible-clock test was doing by accident. _ntp_synced gates slot setup outright (:1386, :2894), so a device that cannot reach NTP now brings up no slots at all — and a network that blocks UDP/123 while allowing 443 is an ordinary firewall configuration, not a corner case. An RTC-backed observer there used to stay synced and keep minting JWTs against a perfectly good clock. Accept the existing clock explicitly when every server has failed, logged as what it is rather than as a claim about a server that never replied. Excluded from the `set mqtt.ntp` validation path, where the question is whether that server works and the clock cannot answer it. configTime() is now called only when a server did answer, since otherwise there is nothing new to point SNTP at. --- src/helpers/bridges/MQTTBridge.cpp | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 24553a5d..1a3c2d15 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4012,7 +4012,24 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { } #endif - if (ntp_ok && ntp_server_used) { + // No server answered, but the clock itself may still be usable. Requiring a real + // SNTP completion above removed something the plausible-clock test was doing by + // accident: an RTC-backed device on a network that blocks NTP (UDP/123) while + // allowing the broker (443) stayed synced and kept minting JWTs. _ntp_synced gates + // slot setup outright, so losing that strands those deployments with no slots at + // all. Keep the behaviour, but as its own decision rather than as a claim about a + // server that never replied. Not on the validation path — `set mqtt.ntp` asks + // whether that server works, and the clock cannot answer for it. + if (!ntp_ok && !primary_only) { + unsigned long existing = (unsigned long)time(nullptr); + if (existing >= kMinValidEpoch) { + epochTime = existing; + ntp_ok = true; + MQTT_DEBUG_PRINTLN("No NTP server answered; continuing on the existing clock: %lu", existing); + } + } + + if (ntp_ok) { // Take ownership of the system clock here, before anything reads it. configTime() // only restarts SNTP and returns, and _rtc reaches settimeofday() on exactly one // path: AutoDiscoverRTCClock writes a detected DS3231/RV3028/PCF8563/RX8130CE chip @@ -4025,7 +4042,11 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { accepted.tv_usec = 0; settimeofday(&accepted, nullptr); - configTime(0, 0, ntp_server_used); + // Only when a server actually answered: there is nothing to point SNTP at + // otherwise, and the existing configuration is the best guess available. + if (ntp_server_used) { + configTime(0, 0, ntp_server_used); + } if (_rtc) { _rtc->setCurrentTime(epochTime); @@ -4036,7 +4057,8 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { _last_ntp_sync = millis(); sync_in_progress = false; - MQTT_DEBUG_PRINTLN("Time synced: %lu (via %s)", epochTime, ntp_server_used); + MQTT_DEBUG_PRINTLN("Time synced: %lu (via %s)", epochTime, + ntp_server_used ? ntp_server_used : "existing clock"); // If slots are already set up and the time jumped significantly (e.g., SNTP // initially returned stale RTC time, then a later sync corrected it), re-issue From ee2f866b104bcf19d61facd4a36c3f9fa3f7ceb6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:58:39 -0700 Subject: [PATCH 20/93] fix(mqtt): clear the stale SNTP status before starting the new request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reset was on the wrong side of configTime(). configTime() configures the server, calls sntp_init(), and returns — the new request is live before it comes back — so a fast reply could set SNTP_SYNC_STATUS_COMPLETED inside that call, and the reset immediately after would erase it. The following ten seconds of polling would then see nothing and reject a server that had in fact answered. On the `set mqtt.ntp` path that surfaces as a good server failing validation. Stop any running session first, discard its status, then start the new one, so the only completion observable is the one being waited for. --- src/helpers/bridges/MQTTBridge.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1a3c2d15..4da87149 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -3986,15 +3986,21 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { for (int s = 0; s < server_count && !ntp_ok; s++) { const char* server = servers[s]; MQTT_DEBUG_PRINTLN("SNTP fallback trying %s...", server); - configTime(0, 0, server); // A plausible clock is not evidence this server answered. The device usually // already holds valid time here — from an earlier sync, or the RTC — so polling // time(nullptr) declared the very first server successful without a packet ever // arriving, stopped the fallback walk there, and refreshed _last_ntp_sync. Worse // on the `set mqtt.ntp` validation path, where a typo is supposed to fail fast. // Wait for SNTP itself to report completion. The status is one-shot — reading - // COMPLETED clears it — so drop any result an earlier sync left behind. + // COMPLETED clears it — so drop any result an earlier sync left behind, and do + // that *before* starting this one: configTime() returns after sntp_init(), so a + // fast reply can complete inside it, and clearing afterwards would erase the + // very result being waited for. + if (sntp_enabled()) { + sntp_stop(); + } sntp_set_sync_status(SNTP_SYNC_STATUS_RESET); + configTime(0, 0, server); for (int i = 0; i < 20; i++) { delay(500); if (sntp_get_sync_status() != SNTP_SYNC_STATUS_COMPLETED) continue; From 436bb65ae611a5fb62ba14412dfd5e6afa978591 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 19:58:50 -0700 Subject: [PATCH 21/93] fix(mqtt): consult the RTC when libc cannot vouch for the fallback clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The usable-clock fallback asked libc only, which does not answer for the case it was written to cover. On a cold boot ESP32RTCClock::begin() stamps libc with a 2024 placeholder on power-on; AutoDiscoverRTCClock::begin() probes the chip but never copies its time across, and getCurrentTime() reads the chip directly. So a Station G3 or T-Beam Supreme that knows exactly what time it is, on a network with UDP/123 blocked, still failed the plausibility test, left _ntp_synced false, and brought up no slots — precisely the deployment the fallback exists for. Ask the RTC when libc is below the floor. libc still wins when it is usable: a clock SNTP set recently outranks a chip that may have drifted. Accepting the RTC value then flows through the same block, so settimeofday() repairs libc and the epoch is written back to the chip. The choice is chooseFallbackClock() in MQTTConnectionPolicy, host-tested across the four states including the power-on placeholder and the exact floor. Also corrects the previous commit's claim that configTime() is called only when a server replied — the fallback necessarily points it at each server before knowing that; it is the post-acceptance call that is now conditional. --- src/helpers/MQTTConnectionPolicy.h | 21 ++++++++++ src/helpers/bridges/MQTTBridge.cpp | 25 +++++++---- .../test_mqtt_connection_policy.cpp | 42 +++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index b2c57690..096c5d38 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -234,4 +234,25 @@ static inline StaleTokenAction classifyStaleToken(bool minted, bool connected, return broker_enforces_exp ? StaleTokenAction::Bounce : StaleTokenAction::KeepAlive; } +// Which clock to fall back on when no NTP server answered. +enum class ClockSource : uint8_t { + None, // nothing plausible to work from — stay unsynced + System, // libc already holds a usable time + Rtc, // libc does not, but the RTC does +}; + +// System first: a clock SNTP set recently outranks an RTC that may have drifted. +// The RTC matters on a cold boot, where ESP32RTCClock::begin() seeds libc with a 2024 +// placeholder on power-on while a detected chip already holds real time and +// AutoDiscoverRTCClock::begin() never copies one into the other. Never while +// validating a server: that asks whether a specific host answers, and no clock can +// answer it. Pass rtc_time 0 when the board has no clock to consult. +static inline ClockSource chooseFallbackClock(bool validating_server, uint32_t system_time, + uint32_t rtc_time, uint32_t min_valid_epoch) { + if (validating_server) return ClockSource::None; + if (system_time >= min_valid_epoch) return ClockSource::System; + if (rtc_time >= min_valid_epoch) return ClockSource::Rtc; + return ClockSource::None; +} + } // namespace MQTTConnectionPolicy diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 4da87149..7afb2e28 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4026,12 +4026,21 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { // all. Keep the behaviour, but as its own decision rather than as a claim about a // server that never replied. Not on the validation path — `set mqtt.ntp` asks // whether that server works, and the clock cannot answer for it. - if (!ntp_ok && !primary_only) { - unsigned long existing = (unsigned long)time(nullptr); - if (existing >= kMinValidEpoch) { - epochTime = existing; + if (!ntp_ok) { + const unsigned long system_time = (unsigned long)time(nullptr); + // On a cold boot with a detected RTC chip these disagree: ESP32RTCClock::begin() + // stamps libc with a 2024 placeholder on power-on, AutoDiscoverRTCClock::begin() + // never copies the chip into it, and getCurrentTime() reads the chip. Asking libc + // alone would reject a board that knows exactly what time it is. + const unsigned long rtc_time = _rtc ? (unsigned long)_rtc->getCurrentTime() : 0; + const MQTTConnectionPolicy::ClockSource source = MQTTConnectionPolicy::chooseFallbackClock( + primary_only, (uint32_t)system_time, (uint32_t)rtc_time, (uint32_t)kMinValidEpoch); + if (source != MQTTConnectionPolicy::ClockSource::None) { + const bool from_rtc = (source == MQTTConnectionPolicy::ClockSource::Rtc); + epochTime = from_rtc ? rtc_time : system_time; ntp_ok = true; - MQTT_DEBUG_PRINTLN("No NTP server answered; continuing on the existing clock: %lu", existing); + MQTT_DEBUG_PRINTLN("No NTP server answered; continuing on the existing %s: %lu", + from_rtc ? "RTC" : "system clock", epochTime); } } @@ -4048,8 +4057,10 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { accepted.tv_usec = 0; settimeofday(&accepted, nullptr); - // Only when a server actually answered: there is nothing to point SNTP at - // otherwise, and the existing configuration is the best guess available. + // Only when a server supplied the accepted epoch. The fallback above necessarily + // points configTime() at each server before knowing whether it replies; this is + // the post-acceptance call, and there is nothing to re-point it at when the epoch + // came from a local clock. if (ntp_server_used) { configTime(0, 0, ntp_server_used); } diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index 7c7bd7de..c3cd469e 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -290,6 +290,48 @@ TEST(StaleToken, FailedMintNeverReconnects) { EXPECT_EQ(StaleTokenAction::Defer, Policy::classifyStaleToken(false, true, false)); } +using Policy::ClockSource; + +// 2026-01-01, the bridge's plausibility floor, and the 2024 placeholder +// ESP32RTCClock::begin() stamps into libc on a power-on reset. +static const uint32_t kFloor = 1767225600; +static const uint32_t kPowerOnPlaceholder = 1715770351; +static const uint32_t kPlausibleNow = 1786000000; + +TEST(FallbackClock, PrefersTheSystemClockWhenItIsUsable) { + // A clock SNTP set recently outranks an RTC that may have drifted. + EXPECT_EQ(ClockSource::System, + Policy::chooseFallbackClock(false, kPlausibleNow, kPlausibleNow - 900, kFloor)); +} + +TEST(FallbackClock, FallsBackToTheRtcOnAColdBoot) { + // The case the system-clock-only check missed: libc holds the power-on + // placeholder while a detected chip holds real time. + EXPECT_EQ(ClockSource::Rtc, + Policy::chooseFallbackClock(false, kPowerOnPlaceholder, kPlausibleNow, kFloor)); +} + +TEST(FallbackClock, NothingUsableStaysUnsynced) { + EXPECT_EQ(ClockSource::None, + Policy::chooseFallbackClock(false, kPowerOnPlaceholder, kPowerOnPlaceholder, kFloor)); + // rtc_time 0 is how a board with no clock to consult is passed in. + EXPECT_EQ(ClockSource::None, + Policy::chooseFallbackClock(false, kPowerOnPlaceholder, 0, kFloor)); +} + +TEST(FallbackClock, ServerValidationNeverAcceptsALocalClock) { + // `set mqtt.ntp` asks whether that host answers. No clock can answer for it, + // however plausible — this is the path where a typo has to fail. + EXPECT_EQ(ClockSource::None, + Policy::chooseFallbackClock(true, kPlausibleNow, kPlausibleNow, kFloor)); +} + +TEST(FallbackClock, TheFloorItselfIsAccepted) { + EXPECT_EQ(ClockSource::System, Policy::chooseFallbackClock(false, kFloor, 0, kFloor)); + EXPECT_EQ(ClockSource::Rtc, Policy::chooseFallbackClock(false, kFloor - 1, kFloor, kFloor)); + EXPECT_EQ(ClockSource::None, Policy::chooseFallbackClock(false, kFloor - 1, kFloor - 1, kFloor)); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 9597d9ca5253c444518f726762fda0ce57ea4c9d Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 14 Aug 2026 20:55:23 -0700 Subject: [PATCH 22/93] fix(mqtt): do not send an NTP request to a name that failed to resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found on hardware while validating the SNTP fix. `set mqtt.ntp bogus.invalid` reported SUCCESS with a correct epoch, in 4 s, with no retry and without ever reaching the SNTP fallback: [E] hostByName(): DNS Failed for bogus.invalid [E] beginPacket(): could not get host from dns: 11 MQTT: Time synced: 1786764354 (via bogus.invalid) Three pieces compose it. WiFiUDP::beginPacket(const char*, port) returns 0 on a DNS failure and leaves remote_ip/remote_port at their previous values. NTPClient::sendNTPPacket() discards that return and calls endPacket() regardless. endPacket() sends to whatever remote_ip still holds. So the request went to the pool address resolved at boot, that server answered with a genuine timestamp, and the loop recorded ntp_server_used as the name that had never been contacted. This sits one layer above the fallback that b1ceaf01 made honest — control never reaches it — so `set mqtt.ntp `, whose whole purpose is to fail fast, still reported OK and the fleet kept a server name it had never spoken to. The DNS pre-check was already here and only logged a warning. Make it decide: skip a name that does not resolve rather than attempt a send that cannot go where it claims. IP literals are unaffected — hostByName() returns them via fromString() without a lookup — and the lookup already ran, so no latency is added. Moved setPoolServerName() below it so the client is never pointed at a server being skipped. Residual, narrower window: our lookup succeeds and NTPClient's own gethostbyname() then fails, which needs the entry to leave the lwIP cache between two calls microseconds apart. Closing it properly needs the resolved IP handed to NTPClient, and this version exposes no setPoolServerIP(); the constructor is the only way in. Not host-testable — NTPClient and WiFiUDP both. Verified by inspection of both library sources plus the captured hardware trace above. --- src/helpers/bridges/MQTTBridge.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 7afb2e28..1ce0f1bb 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -3954,15 +3954,25 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { const int kMaxNtpRetriesPerServer = 2; for (int s = 0; s < server_count && !ntp_ok; s++) { const char* server = servers[s]; - _ntp_client.setPoolServerName(server); #ifdef ESP_PLATFORM + // Authoritative, not advisory. NTPClient::sendNTPPacket() ignores what + // beginPacket() returns, and WiFiUDP leaves remote_ip/remote_port at the previous + // destination when a name fails to resolve — so asking an unresolvable host sends + // the request to whichever server resolved last, and that server's genuine reply + // gets credited to this name. Observed on d4: `set mqtt.ntp bogus.invalid` reported + // success with a correct epoch, answered by the pool address left over from boot. + // Skipping is what keeps the credit honest; the name that answered is the name + // recorded. IPAddress resolved_ip; if (!WiFi.hostByName(server, resolved_ip)) { - MQTT_DEBUG_PRINTLN("WARNING: DNS resolution failed for %s - NTP sync may fail", server); + MQTT_DEBUG_PRINTLN("NTP: %s does not resolve — skipping, not attempting a send", server); + continue; } #endif + _ntp_client.setPoolServerName(server); + for (int attempt = 1; attempt <= kMaxNtpRetriesPerServer && !ntp_ok; attempt++) { if (attempt > 1) { MQTT_DEBUG_PRINTLN("NTP retry %d/%d on %s...", attempt, kMaxNtpRetriesPerServer, server); From cba8074bdb10f28b75be321b12d0e866b0f18e9d Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 15 Aug 2026 15:09:55 -0700 Subject: [PATCH 23/93] fix(mqtt): undo a failed /mqtt.json publish instead of claiming a rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing moves the old primary to .bak before the verified temp takes that name, so a failed second rename left the new image exactly where boot recovery promotes it — while the observer setter told the operator the change had been rolled back. The refused value came back at the next reset. Restore the backup and discard the temp on that path, and distinguish CommitIndeterminate from CommitFailed when the filesystem cannot be put back, so the CLI reply says the flash state is unresolved rather than claiming the change is gone. The indeterminate condition latches for the boot: the artifact left behind also makes every later transaction fail to begin, so it cannot clear itself. Also state the version-first rule the future-version probe depends on. The probe reads the root version with this firmware's grammar, so a newer file that introduces unknown syntax ahead of that field reads as corrupt rather than future and loses its preservation guarantee. Tests: publish-failure rollback and the indeterminate outcome against a SPIFFS-shaped store fake; the version-first writer invariant and the cost of violating it; /prefs.json coverage for the strict shape checks (deployed-shape file, unknown nested groups, torn files, mismatches). --- MQTT_INTERNALS.md | 26 +- src/helpers/CommonCLI.cpp | 66 ++++- src/helpers/CommonCLI.h | 5 + src/helpers/CommonCLI_Observer.cpp | 8 +- src/helpers/MQTTPrefsAtomicStore.h | 17 +- src/helpers/MQTTPrefsSerializer.h | 11 +- .../test_config_serializer.cpp | 95 +++++++ .../test_mqtt_prefs_atomic_store.cpp | 267 ++++++++++++++++-- .../test_mqtt_prefs_serializer.cpp | 23 ++ 9 files changed, 469 insertions(+), 49 deletions(-) diff --git a/MQTT_INTERNALS.md b/MQTT_INTERNALS.md index 005de530..eb5c7944 100644 --- a/MQTT_INTERNALS.md +++ b/MQTT_INTERNALS.md @@ -170,7 +170,8 @@ favour of independent `mqtt.rx` / `mqtt.tx` controls. Everything MQTT-specific l Observer preferences use the same `ConfigSerializer` object notation as upstream `/prefs.json`: semantic unquoted keys, quoted strings, decimal numbers, and nested -objects. Schema version 1 requires the root `version:1` field. The main shape is: +objects. Schema version 1 requires the root `version:1` field, which must be the +first property of the root object. The main shape is: ```text {version:1, @@ -210,7 +211,20 @@ though the setting were durable. Saves stream to `/mqtt.json.tmp` through a sticky short-write detector while computing size and checksum. The firmware closes and rereads the temp, verifies size/checksum, parses it into another scratch object, then publishes with -`/mqtt.json` -> `/mqtt.json.bak` and temp -> primary. Boot recovery selects the usable +`/mqtt.json` -> `/mqtt.json.bak` and temp -> primary. Failure safety is paid for in +transient heap: a CLI setter holds one `MQTTPrefs` rollback snapshot (2876 bytes) for the +whole command, and the save allocates one more for normalization defaults (released +before file I/O) and then one for verification — about 5.6 KiB peak above baseline. +Each allocation is checked, so exhaustion refuses the setting rather than crashing. + +If publishing fails partway, the transaction is rolled back rather than left for boot +recovery: the backup is renamed back over the empty primary and the verified temp is +discarded. This is what makes the setter's "change rolled back" reply true — without it +the next boot would promote that temp and activate the value the CLI just refused. When +the rollback itself cannot complete, the save reports an indeterminate outcome instead, +the artifacts stay for recovery, and the CLI says the flash state is unresolved. Further +saves are refused until then, because a transaction cannot start while a temp or backup +exists. Boot recovery selects the usable primary/temp/backup without overwriting an opaque future or corrupt primary. A valid future-version temp that reached the rename phase wins over the stale backup and is held for newer firmware. If a temp claims a future version but uses grammar this firmware @@ -233,6 +247,14 @@ loading its known-looking fields. Literal schema keys are compile-time checked a the 15-character visible-key limit so a new version-1 field cannot accidentally violate that downgrade contract. +Every future version must also keep `version` as the first root property. Older firmware +detects a future file by probing that field with its own grammar, and the probe stops at +the first construct it cannot tokenize (an array, an overlong key, a value type it does +not expect). Syntax a newer schema introduces *before* the version field therefore makes +its files look corrupt rather than future, which costs them the preservation guarantee +above: as the temp of an interrupted commit such a file is discardable instead of +retained. `test_mqtt_prefs_serializer` pins both the ordering and that consequence. + #### Downgrade and rollback The old `/mqtt_prefs` binary is read only for one-time migration and is deliberately diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index cf2e52eb..930676a5 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -608,12 +608,14 @@ public: _finished = false; _open = false; _owns_temp = false; + _owns_backup = false; _bytes_written = 0; _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; - // Recovery owns stale artifacts. Do not delete them here: a failed commit - // may have moved the old primary to .bak and left a verified temp that the - // next boot must choose between. Refusing the save is safer than erasing an - // image this firmware cannot decode. + // Recovery owns stale artifacts. Do not delete them here: a power cut, or a + // failed commit that could not be rolled back, may have moved the old + // primary to .bak and left a verified temp that the next boot must choose + // between. Refusing the save is safer than erasing an image this firmware + // cannot decode. if (_fs->exists("/mqtt.json.tmp") || _fs->exists("/mqtt.json.bak")) return false; #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) _file = _fs->open("/mqtt.json.tmp", FILE_O_WRITE); @@ -679,11 +681,12 @@ public: if (!_finished) return false; // SPIFFS refuses rename(tmp, existing_dest). Move the existing image to a // recoverable backup first, then publish temp into the now-empty primary. - // Never remove either image after a failed boundary; boot recovery selects - // the completed temp or restores the backup. + // A power cut at either boundary is resolved by boot recovery; a failure + // that returns here is undone by rollbackFailedCommit(). if (_fs->exists("/mqtt.json.bak")) return false; - if (_fs->exists("/mqtt.json") && !_fs->rename("/mqtt.json", "/mqtt.json.bak")) { - return false; + if (_fs->exists("/mqtt.json")) { + if (!_fs->rename("/mqtt.json", "/mqtt.json.bak")) return false; + _owns_backup = true; } if (!_fs->rename("/mqtt.json.tmp", "/mqtt.json")) return false; // Cleanup failure is non-fatal: the new primary is published and recovery @@ -692,6 +695,32 @@ public: return true; } + // Undo a commit that failed midway. Without this, a failed publish can leave + // the old primary parked in .bak and the verified temp still holding the new + // image, which boot recovery then promotes — so a setter that told the + // operator the change was rolled back would be wrong after the next reset. + // + // Republishing the backup is the operation that matters: once the primary + // name is occupied, recovery keeps it and treats the temp as stale, so the + // temp removal below is only housekeeping. Returns false when the + // pre-transaction state could not be restored and the outcome of the next + // boot is therefore uncertain. + bool rollbackFailedCommit() { + // Only ever restore the backup this transaction made. Anything else under + // that name predates the transaction and is recovery's to resolve. + if (_owns_backup && !_fs->exists("/mqtt.json") && + !_fs->rename("/mqtt.json.bak", "/mqtt.json")) { + return false; + } + _owns_backup = false; + if (_owns_temp && _fs->exists("/mqtt.json.tmp") && !_fs->remove("/mqtt.json.tmp")) { + // A verified temp still outranks a missing primary during recovery, so + // this is only harmless if the primary name is occupied again. + return _fs->exists("/mqtt.json"); + } + return true; + } + void discardFinishedTemp() { if (_open) _file.close(); _open = false; @@ -703,14 +732,15 @@ public: void abort() { if (_open) _file.close(); _open = false; - // Once finish() has verified the temp, commit may already have moved the - // primary to .bak. Keep the temp on a commit failure so recovery can - // publish it (or fall back to .bak) after reset. + // Only unfinished staging is disposable here. Once finish() has verified + // the temp, rollbackFailedCommit() has already decided its fate, and a temp + // that survived that is one recovery must resolve after reset. if (_owns_temp && !_finished && _fs->exists("/mqtt.json.tmp")) { _fs->remove("/mqtt.json.tmp"); } _finished = false; _owns_temp = false; + _owns_backup = false; } private: @@ -719,6 +749,9 @@ private: bool _open = false; bool _finished = false; bool _owns_temp = false; + // This transaction moved the old primary to /mqtt.json.bak, so a failed + // publish may put it back. Never true for a backup it did not create. + bool _owns_backup = false; size_t _bytes_written = 0; uint32_t _expected_crc = MQTT_JSON_FNV1A_OFFSET_BASIS; }; @@ -1061,7 +1094,16 @@ bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { } break; case MQTTPrefsAtomicStore::VerifiedImageResult::CommitFailed: - MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename; recovery files preserved"); + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename; transaction rolled back"); + break; + case MQTTPrefsAtomicStore::VerifiedImageResult::CommitIndeterminate: + // The failed publish could not be undone, so a recovery file this + // firmware may still promote is holding the new image. No further save + // can start until it is resolved: begin() refuses while either the temp + // or the backup exists, which is exactly the state that gets us here. + _observer_save_indeterminate = true; + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename and could not be rolled back; " + "recovery files preserved"); break; } return false; diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 2e2f02ef..42287ffb 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -373,6 +373,11 @@ class CommonCLI { // /mqtt.json is newer, corrupt, or temporarily unreadable. The in-memory prefs // run on defaults and saveMQTTPrefs() must not overwrite the source file. bool _mqtt_prefs_hold = false; + // A failed publish could not be undone, so the next boot may still come up + // with the value that was refused. persistObserverPrefs() must not call that + // a rollback. Latched for the boot: the artifact left behind also makes every + // later transaction fail to begin, so the condition cannot clear itself. + bool _observer_save_indeterminate = false; #endif bool _com_prefs_needs_upgrade = false; // old-format legacy prefs detected; rewrite once after load diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 93c25776..c6fcb606 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -208,7 +208,13 @@ bool CommonCLI::persistObserverPrefs(char* reply) { if (_observer_prefs_rollback != nullptr) { memcpy(&_mqtt_prefs, _observer_prefs_rollback, sizeof(_mqtt_prefs)); } - strcpy(reply, "Error: setting not persisted; change rolled back"); + // The running node is back on the old value either way, but only claim the + // change is gone when the write really was undone on flash. + if (_observer_save_indeterminate) { + strcpy(reply, "Error: setting not persisted; flash state unresolved, recheck after reboot"); + } else { + strcpy(reply, "Error: setting not persisted; change rolled back"); + } return false; #else (void)reply; diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h index c3b3ce20..91b7ae39 100644 --- a/src/helpers/MQTTPrefsAtomicStore.h +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -11,7 +11,14 @@ namespace MQTTPrefsAtomicStore { // Production /mqtt.json flow. finish() verifies the exact bytes written, // verify_image reparses the finished temp through the schema, and only then is // commit() allowed to publish it. A schema-verification failure discards the -// finished temp; a commit failure preserves it for boot recovery. +// finished temp; a commit failure is undone by rollbackFailedCommit(). +// +// CommitFailed and CommitIndeterminate are distinct outcomes, not shades of the +// same one. Publishing moves the old primary aside before the verified temp +// takes its name, so a half-done commit leaves an image that boot recovery +// would promote. CommitFailed means that was undone and the change is really +// gone; CommitIndeterminate means it could not be, and the next boot may still +// come up with the new value. Callers must not report the two the same way. enum class VerifiedImageResult : uint8_t { Committed, BeginFailed, @@ -19,6 +26,7 @@ enum class VerifiedImageResult : uint8_t { FinishFailed, VerifyFailed, CommitFailed, + CommitIndeterminate, }; template @@ -42,8 +50,13 @@ inline VerifiedImageResult writeVerifiedImage(Store& store, return VerifiedImageResult::VerifyFailed; } if (!store.commit()) { + // Undo the partial publish before answering. Only the store knows whether + // the pre-transaction state was actually restored, so take its word for + // which failure this was. + const bool rolled_back = store.rollbackFailedCommit(); store.abort(); - return VerifiedImageResult::CommitFailed; + return rolled_back ? VerifiedImageResult::CommitFailed + : VerifiedImageResult::CommitIndeterminate; } return VerifiedImageResult::Committed; } diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h index b4d12b18..4d182c44 100644 --- a/src/helpers/MQTTPrefsSerializer.h +++ b/src/helpers/MQTTPrefsSerializer.h @@ -9,6 +9,15 @@ static const int32_t MQTT_PREFS_JSON_FORMAT_VERSION = 1; +// Format invariant, binding on every future schema version: `version` is the +// FIRST property of the root object. The probe below reads it with this +// firmware's grammar and stops at the first construct that grammar cannot +// tokenize, so a newer file that introduces unknown syntax ahead of its version +// field is indistinguishable from a corrupt one here. That costs it the +// opaque-future preservation guarantee: as a temp left by an interrupted commit +// it would be classified as discardable rather than retained. Keep the version +// first when adding fields or bumping the version; a host test pins it. +// // Reads only the mandatory root version while ignoring every other schema // field. Recovery uses this before the v1 decoder so a syntactically valid // future file remains opaque even when that schema changes a v1 field's type. @@ -369,7 +378,7 @@ class MQTTPrefsSerializer : public ConfigSerializer { protected: void structure() override { - defStrict("version", _version, _seen_version); + defStrict("version", _version, _seen_version); // must stay first; see above def("wifi", _wifi); def("time", _time); def("mqtt", _mqtt); diff --git a/test/test_config_serializer/test_config_serializer.cpp b/test/test_config_serializer/test_config_serializer.cpp index 90154047..ae63f682 100644 --- a/test/test_config_serializer/test_config_serializer.cpp +++ b/test/test_config_serializer/test_config_serializer.cpp @@ -231,6 +231,101 @@ TEST(ConfigSerializer, LoadSerial_RejectsLeadingDigitKey) { EXPECT_FALSE(data.loadSerial(s)); } +// ── /prefs.json compatibility under the strict shape checks ───────────────── +// +// The scalar-vs-object rejection added for /mqtt.json also runs for the plain +// def() overloads that NodePrefs uses, and loadPrefsInt() applies /prefs.json +// straight onto the live object without consulting the return value. These +// pin what that combination does to files already on deployed devices. + +// A settings file in the exact shape the firmware writes, transcribed rather +// than produced by saveSerial() so a change to the writer cannot quietly move +// the fixture with it. +static const char* DEPLOYED_PREFS_JSON = + "{name:\"Repeater-1\",pass:\"hunter2\",guest:\"\",owner:\"ops@example.com\"," + "adv_int:4,f_adv_int:12,lat:47.612345,lon:-122.334567,disc_mod:1719791234," + "radio:{freq:910.5250,bw:250.0000,sf:10,cr:5,cad:0,int_thr:0,rxgain:1," + "fem_rxgain:0,fem_txgain:1,tx:22,af:1.0000,rxdelay:1000.0000," + "f_txdelay:0.5000,d_txdelay:0.2000,agc_int:0,hash_mode:0,multi_ack:0}," + "bridge:{en:0,delay:500,src:1,baud:115200,ch:1,secret:\"\"}," + "gps:{en:0,int:60,adv_loc:0}," + "repeat:{disable:0,f_max:64,f_max_uns:32,f_max_adv:16,loop:1}," + "room:{rd_only:0},power:{adc_mult:1.0000,pwr_sav_en:0}}"; + +TEST(NodePrefs, DeployedPrefsJsonStillLoadsCompletely) { + MockInputStream s(DEPLOYED_PREFS_JSON); + NodePrefs prefs; + + ASSERT_TRUE(prefs.loadSerial(s)); + EXPECT_STREQ("Repeater-1", prefs.node_name); + EXPECT_STREQ("ops@example.com", prefs.owner_info); + EXPECT_EQ(4, prefs.advert_interval); + EXPECT_DOUBLE_EQ(47.612345, prefs.node_lat); + EXPECT_DOUBLE_EQ(-122.334567, prefs.node_lon); + EXPECT_EQ(1719791234u, prefs.discovery_mod_timestamp); + EXPECT_FLOAT_EQ(910.525f, prefs.freq); + EXPECT_EQ(10, prefs.sf); + EXPECT_EQ(22, prefs.tx_power_dbm); + EXPECT_EQ(1, prefs.radio_fem_txgain); + EXPECT_EQ(1, prefs.bridge_pkt_src); + EXPECT_EQ(115200u, prefs.bridge_baud); + EXPECT_EQ(60u, prefs.gps_interval); + EXPECT_EQ(64, prefs.flood_max); + EXPECT_EQ(1, prefs.loop_detect); +} + +TEST(NodePrefs, UnknownNestedObjectCannotBeMistakenForARootScalar) { + // Depth is what keeps an unknown group's inner keys from matching a root + // field of the same name, so an unrecognized section must stay ignorable. + MockInputStream s("{name:\"Repeater-1\",mqtt:{name:\"other\",owner:\"nobody\"},adv_int:4}"); + NodePrefs prefs; + strcpy(prefs.owner_info, "ops@example.com"); + + EXPECT_TRUE(prefs.loadSerial(s)); + EXPECT_STREQ("Repeater-1", prefs.node_name); + EXPECT_STREQ("ops@example.com", prefs.owner_info); + EXPECT_EQ(4, prefs.advert_interval); +} + +TEST(NodePrefs, TornPrefsJsonAppliesOnlyTheFieldsBeforeTheTear) { + // A power cut during the old non-transactional /prefs.json write leaves a + // file like this. loadPrefsInt() ignores the failed return and keeps the + // partial result, which is the pre-existing behavior; the strict checks + // must not turn it into something worse than a partial load. + MockInputStream s("{name:\"Repeater-1\",adv_int:4,radio:{freq:910.5250,sf:1"); + NodePrefs prefs; + prefs.sf = 9; + + EXPECT_FALSE(prefs.loadSerial(s)); + EXPECT_STREQ("Repeater-1", prefs.node_name); + EXPECT_EQ(4, prefs.advert_interval); + EXPECT_FLOAT_EQ(910.525f, prefs.freq); + EXPECT_EQ(9, prefs.sf); // the torn value never completed a token +} + +TEST(NodePrefs, ShapeMismatchIsRejectedAndStopsFurtherApplication) { + // Hand-edited or corrupted files are the regression surface for the strict + // checks: a known scalar holding an object now fails the load and stops + // parsing, so nothing after the mismatch reaches the live object. + MockInputStream scalar_as_object("{name:{x:1},adv_int:4}"); + NodePrefs prefs; + prefs.advert_interval = 7; + EXPECT_FALSE(prefs.loadSerial(scalar_as_object)); + EXPECT_EQ(7, prefs.advert_interval); + + // The reverse mismatch is rejected too: a known group given a scalar. + MockInputStream object_as_scalar("{name:\"Repeater-1\",radio:1}"); + NodePrefs scalar_group; + EXPECT_FALSE(scalar_group.loadSerial(object_as_scalar)); + + // And a known scalar nested one level deeper than the schema places it. + MockInputStream over_nested("{radio:{sf:{value:10}}}"); + NodePrefs nested; + nested.sf = 9; + EXPECT_FALSE(nested.loadSerial(over_nested)); + EXPECT_EQ(9, nested.sf); +} + // ── main ─────────────────────────────────────────────────────── diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp index 4d2ae120..ddd6e184 100644 --- a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -70,18 +70,6 @@ public: return true; } - bool verify() { - ++verify_calls; - return _failure != FailurePoint::Verify; - } - - void discardFinishedTemp() { - ++discard_calls; - _files.erase("/mqtt_prefs.tmp"); - _finished = false; - _owns_temp = false; - } - void abort() { ++abort_calls; _open = false; @@ -102,8 +90,6 @@ public: int finish_calls = 0; int commit_calls = 0; int abort_calls = 0; - int verify_calls = 0; - int discard_calls = 0; private: FailurePoint _failure; @@ -129,12 +115,167 @@ AtomicStore::Result runWithObserverTail(InMemoryStore* store) { return AtomicStore::write(*store, header, sizeof(header), payload, sizeof(payload)); } -AtomicStore::VerifiedImageResult runVerifiedJson(InMemoryStore* store) { - const uint8_t json[] = "{version:1,wifi:{ssid:\"mesh\"}}"; +// Models MQTTPrefsJsonFileStore under the SPIFFS rule it was written for: +// rename() refuses an existing destination, so publishing must move the old +// primary to .bak before the verified temp can take its name. Both halves of +// that publish, and the rollback that undoes a half-done one, are injectable. +class InMemoryJsonStore { +public: + struct Options { + FailurePoint failure = FailurePoint::None; + bool rollback_rename_fails = false; + bool rollback_remove_fails = false; + bool has_primary = true; + }; + + explicit InMemoryJsonStore(FailurePoint failure) : _opts{failure, false, false, true} { + _files["/mqtt.json"] = oldImage(); + } + explicit InMemoryJsonStore(Options opts) : _opts(opts) { + if (_opts.has_primary) _files["/mqtt.json"] = oldImage(); + } + + bool begin() { + ++begin_calls; + // Production refuses to open a new transaction while recovery still owns + // an artifact, so a rollback that left one behind blocks the retry. + if (has("/mqtt.json.tmp") || has("/mqtt.json.bak")) return false; + _staging.clear(); + _open = _opts.failure != FailurePoint::Begin; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + ++write_calls; + if (!_open) return 0; + const size_t written = + _opts.failure == FailurePoint::ImageWrite && size > 0 ? size - 1 : size; + _staging.insert(_staging.end(), bytes, bytes + written); + return written; + } + + bool finish() { + ++finish_calls; + _open = false; + if (_opts.failure == FailurePoint::Finish) return false; + _files["/mqtt.json.tmp"] = _staging; + _finished = true; + return true; + } + + bool verify() { + ++verify_calls; + return _opts.failure != FailurePoint::Verify; + } + + bool commit() { + ++commit_calls; + if (!_finished) return false; + if (has("/mqtt.json.bak")) return false; + if (has("/mqtt.json") && !rename("/mqtt.json", "/mqtt.json.bak")) return false; + // Fail the publish rename, the boundary that leaves the old primary parked + // in .bak with the verified new image still sitting in the temp. + if (_opts.failure == FailurePoint::Commit) return false; + if (!rename("/mqtt.json.tmp", "/mqtt.json")) return false; + _files.erase("/mqtt.json.bak"); + _finished = false; + return true; + } + + bool rollbackFailedCommit() { + ++rollback_calls; + if (!has("/mqtt.json") && has("/mqtt.json.bak")) { + if (_opts.rollback_rename_fails || !rename("/mqtt.json.bak", "/mqtt.json")) { + return false; + } + } + if (has("/mqtt.json.tmp")) { + if (_opts.rollback_remove_fails) return has("/mqtt.json"); + _files.erase("/mqtt.json.tmp"); + } + return true; + } + + void discardFinishedTemp() { + ++discard_calls; + _files.erase("/mqtt.json.tmp"); + _finished = false; + } + + void abort() { + ++abort_calls; + _open = false; + _staging.clear(); + if (!_finished) _files.erase("/mqtt.json.tmp"); + _finished = false; + } + + // Apply the boot-recovery policy to whatever the transaction left behind and + // report the image the node would come up on. Every file present in these + // scenarios is a complete image, so each maps to Usable. + std::vector imageAfterReboot() { + const auto state = [this](const char* path) { + return has(path) ? Recovery::FileState::Usable : Recovery::FileState::Missing; + }; + switch (Recovery::select(state("/mqtt.json"), state("/mqtt.json.tmp"), + state("/mqtt.json.bak"))) { + case Recovery::Action::PromoteTemp: + rename("/mqtt.json.tmp", "/mqtt.json"); + break; + case Recovery::Action::PromoteBackup: + rename("/mqtt.json.bak", "/mqtt.json"); + break; + case Recovery::Action::DiscardTemp: + _files.erase("/mqtt.json.tmp"); + break; + default: + break; + } + return has("/mqtt.json") ? _files.at("/mqtt.json") : std::vector(); + } + + bool has(const char* path) const { return _files.count(path) != 0; } + bool canStartSave() const { return !has("/mqtt.json.tmp") && !has("/mqtt.json.bak"); } + const std::vector& source() const { return _files.at("/mqtt.json"); } + static std::vector oldImage() { + const char* text = "{version:1,wifi:{ssid:\"old\"}}"; + return std::vector(text, text + strlen(text)); + } + static std::vector newImage() { + const char* text = "{version:1,wifi:{ssid:\"mesh\"}}"; + return std::vector(text, text + strlen(text)); + } + + int begin_calls = 0; + int write_calls = 0; + int finish_calls = 0; + int commit_calls = 0; + int abort_calls = 0; + int verify_calls = 0; + int discard_calls = 0; + int rollback_calls = 0; + +private: + bool rename(const char* from, const char* to) { + if (_files.count(from) == 0 || _files.count(to) != 0) return false; + _files[to] = _files[from]; + _files.erase(from); + return true; + } + + Options _opts; + bool _open = false; + bool _finished = false; + std::vector _staging; + std::map> _files; +}; + +AtomicStore::VerifiedImageResult runVerifiedJson(InMemoryJsonStore* store) { + const std::vector json = InMemoryJsonStore::newImage(); return AtomicStore::writeVerifiedImage( *store, [store, &json]() { - return store->write(json, sizeof(json) - 1) == sizeof(json) - 1; + return store->write(json.data(), json.size()) == json.size(); }, [store]() { return store->verify(); }); } @@ -284,7 +425,8 @@ public: } // Inject ordinary operation failures (as distinct from a power cut). A - // failed temp rename leaves both the verified temp and old backup intact. + // failed temp rename leaves both the verified temp and old backup intact, + // which is the state rollbackFailedPublish() then undoes. bool publish(bool fail_backup_rename, bool fail_temp_rename, bool fail_cleanup) { writeVerifiedTemp(); if (fail_backup_rename || !rename("/mqtt_prefs", "/mqtt_prefs.bak")) return false; @@ -293,6 +435,15 @@ public: return true; // backup cleanup is intentionally non-fatal after publish } + bool rollbackFailedPublish() { + if (!has("/mqtt_prefs") && has("/mqtt_prefs.bak") && + !rename("/mqtt_prefs.bak", "/mqtt_prefs")) { + return false; + } + _files.erase("/mqtt_prefs.tmp"); + return true; + } + void recover(Recovery::FileState primary = Recovery::FileState::Usable, Recovery::FileState temp = Recovery::FileState::Usable, Recovery::FileState backup = Recovery::FileState::Usable) { @@ -370,8 +521,6 @@ TEST(MQTTPrefsAtomicStore, CommitPublishesExactHeaderThenPayload) { } TEST(MQTTPrefsAtomicStore, ProductionJsonPolicyCoversEveryVerificationBoundary) { - const std::vector old_source = { - 'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; const struct { FailurePoint point; AtomicStore::VerifiedImageResult expected; @@ -381,24 +530,24 @@ TEST(MQTTPrefsAtomicStore, ProductionJsonPolicyCoversEveryVerificationBoundary) int commits; int aborts; int discards; - bool keeps_temp; + int rollbacks; } cases[] = { {FailurePoint::None, AtomicStore::VerifiedImageResult::Committed, - 1, 1, 1, 1, 0, 0, false}, + 1, 1, 1, 1, 0, 0, 0}, {FailurePoint::Begin, AtomicStore::VerifiedImageResult::BeginFailed, - 0, 0, 0, 0, 1, 0, false}, + 0, 0, 0, 0, 1, 0, 0}, {FailurePoint::ImageWrite, AtomicStore::VerifiedImageResult::WriteFailed, - 1, 0, 0, 0, 1, 0, false}, + 1, 0, 0, 0, 1, 0, 0}, {FailurePoint::Finish, AtomicStore::VerifiedImageResult::FinishFailed, - 1, 1, 0, 0, 1, 0, false}, + 1, 1, 0, 0, 1, 0, 0}, {FailurePoint::Verify, AtomicStore::VerifiedImageResult::VerifyFailed, - 1, 1, 1, 0, 0, 1, false}, + 1, 1, 1, 0, 0, 1, 0}, {FailurePoint::Commit, AtomicStore::VerifiedImageResult::CommitFailed, - 1, 1, 1, 1, 1, 0, true}, + 1, 1, 1, 1, 1, 0, 1}, }; for (const auto& test_case : cases) { - InMemoryStore store(test_case.point); + InMemoryJsonStore store(test_case.point); EXPECT_EQ(test_case.expected, runVerifiedJson(&store)); EXPECT_EQ(test_case.writes, store.write_calls); EXPECT_EQ(test_case.finishes, store.finish_calls); @@ -406,13 +555,66 @@ TEST(MQTTPrefsAtomicStore, ProductionJsonPolicyCoversEveryVerificationBoundary) EXPECT_EQ(test_case.commits, store.commit_calls); EXPECT_EQ(test_case.aborts, store.abort_calls); EXPECT_EQ(test_case.discards, store.discard_calls); - EXPECT_EQ(test_case.keeps_temp, store.tempExists()); + EXPECT_EQ(test_case.rollbacks, store.rollback_calls); + // Every failure leaves the previous image published and no artifact behind, + // so the next save can start immediately. if (test_case.point != FailurePoint::None) { - EXPECT_EQ(old_source, store.source()); + EXPECT_EQ(InMemoryJsonStore::oldImage(), store.source()); + EXPECT_TRUE(store.canStartSave()); } } } +TEST(MQTTPrefsAtomicStore, FailedPublishIsUndoneSoTheRefusedValueCannotReturnAtBoot) { + InMemoryJsonStore store(FailurePoint::Commit); + + // The publish moved the old primary to .bak before failing to rename the + // verified temp into place. Reporting the change as rolled back is only + // truthful because that half-done transaction is undone here. + ASSERT_EQ(AtomicStore::VerifiedImageResult::CommitFailed, runVerifiedJson(&store)); + EXPECT_EQ(InMemoryJsonStore::oldImage(), store.source()); + EXPECT_FALSE(store.has("/mqtt.json.tmp")); + EXPECT_FALSE(store.has("/mqtt.json.bak")); + EXPECT_EQ(InMemoryJsonStore::oldImage(), store.imageAfterReboot()); +} + +TEST(MQTTPrefsAtomicStore, UnrestorablePublishFailureIsReportedAsIndeterminate) { + // Rollback cannot republish the backup: the primary name stays empty and the + // verified temp still wins recovery, so the refused value does come back at + // the next boot. The caller must say so rather than claim a rollback. + InMemoryJsonStore backup_stuck(InMemoryJsonStore::Options{ + FailurePoint::Commit, /*rollback_rename_fails=*/true, false, true}); + ASSERT_EQ(AtomicStore::VerifiedImageResult::CommitIndeterminate, + runVerifiedJson(&backup_stuck)); + EXPECT_TRUE(backup_stuck.has("/mqtt.json.tmp")); + EXPECT_TRUE(backup_stuck.has("/mqtt.json.bak")); + EXPECT_FALSE(backup_stuck.canStartSave()); // retries fail until this resolves + EXPECT_EQ(InMemoryJsonStore::newImage(), backup_stuck.imageAfterReboot()); + + // Same verdict for the first-ever save, where there is no backup to restore + // and the undeletable temp is the only image the next boot can find. + InMemoryJsonStore first_save(InMemoryJsonStore::Options{ + FailurePoint::Commit, false, /*rollback_remove_fails=*/true, + /*has_primary=*/false}); + ASSERT_EQ(AtomicStore::VerifiedImageResult::CommitIndeterminate, + runVerifiedJson(&first_save)); + EXPECT_FALSE(first_save.has("/mqtt.json")); + EXPECT_EQ(InMemoryJsonStore::newImage(), first_save.imageAfterReboot()); +} + +TEST(MQTTPrefsAtomicStore, FirstSavePublishFailureLeavesNoImageToPromote) { + InMemoryJsonStore store(InMemoryJsonStore::Options{ + FailurePoint::Commit, false, false, /*has_primary=*/false}); + + // No prior /mqtt.json exists, so rollback only has to discard the temp. The + // next boot re-runs migration from the legacy source instead of adopting the + // value the CLI just refused. + ASSERT_EQ(AtomicStore::VerifiedImageResult::CommitFailed, runVerifiedJson(&store)); + EXPECT_FALSE(store.has("/mqtt.json.tmp")); + EXPECT_TRUE(store.imageAfterReboot().empty()); + EXPECT_TRUE(store.canStartSave()); +} + TEST(MQTTPrefsAtomicStore, AnyFailureAbortsAndPreservesExistingSource) { const std::vector source = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; const struct { @@ -677,8 +879,11 @@ TEST(MQTTPrefsAtomicStore, SpiffsRenameAndCleanupFailuresRemainRecoverable) { EXPECT_FALSE(store.has("/mqtt_prefs")); EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); EXPECT_TRUE(store.has("/mqtt_prefs.bak")); + // Unlike a power cut at this boundary, a returned failure is answered to + // the caller, so the transaction is undone before recovery ever sees it. + EXPECT_TRUE(store.rollbackFailedPublish()); store.recover(); - EXPECT_EQ(SpiffsMqttTransaction::newImage(), store.primary()); + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); EXPECT_FALSE(store.has("/mqtt_prefs.bak")); } diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp index ec2544e7..ce91b0c1 100644 --- a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -191,6 +191,29 @@ TEST(MQTTPrefsSerializer, FutureVersionProbeIgnoresV1FieldTypeChanges) { } } +TEST(MQTTPrefsSerializer, VersionIsWrittenAsTheFirstRootProperty) { + MQTTPrefs prefs = defaults(); + MQTTPrefsSerializer writer(&prefs); + OutputStream output; + ASSERT_TRUE(writer.saveSerial(output)); + EXPECT_EQ(0u, output.text().find("{version:1,")) << output.text(); +} + +TEST(MQTTPrefsSerializer, FutureGrammarAheadOfVersionCannotBeRecognizedAsFuture) { + // Why the version-first invariant is part of the format rather than a style + // preference. These two files differ only in key order, and only the + // compliant one keeps its preservation guarantee on this firmware. + InputStream compliant("{version:2,x:[1]}"); + MQTTPrefsVersionProbe compliant_probe; + EXPECT_FALSE(compliant_probe.loadSerial(compliant)); + EXPECT_TRUE(compliant_probe.hasFutureVersion()); + + InputStream violating("{x:[1],version:2}"); + MQTTPrefsVersionProbe violating_probe; + EXPECT_FALSE(violating_probe.loadSerial(violating)); + EXPECT_FALSE(violating_probe.hasFutureVersion()); +} + TEST(MQTTPrefsSerializer, RejectsDuplicateKnownKey) { MQTTPrefs prefs = defaults(); InputStream input("{version:1,mqtt:{origin:\"one\",origin:\"two\"}}"); From dad6d39c458323b9f60df60fdb566ebcdd449cb3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 15 Aug 2026 15:22:22 -0700 Subject: [PATCH 24/93] fix(mqtt): keep an unclassifiable /mqtt.json candidate across boots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recovery preserved a FutureClaimed or Indeterminate temp, but published the backup into the primary name to run that boot. That spent the one piece of state saying the candidate had already passed the backup rename: the next boot saw an ordinary usable primary beside a stray temp, and deleted the temp precisely when more heap or newer firmware finally made it readable. The OOM path needed no future firmware to hit it — power cut after the backup rename, one boot short of classification scratch, and a verified new image was gone. Answer an uncertain temp with UseBackupHeld instead: rename nothing, read the last committed image straight out of /mqtt.json.bak, and hold writes. The filenames then still describe the interrupted transaction, so a later boot promotes the candidate through the ordinary temp rule, or falls back to the backup once the candidate proves definitively corrupt. Tests: two-boot sequences for Indeterminate and FutureClaimed candidates that later classify as Usable or FutureUsable, the invalid-candidate fallback, and the no-usable-backup case where the candidate still takes the authoritative name. --- MQTT_INTERNALS.md | 10 +- src/helpers/CommonCLI.cpp | 112 +++++++++++++----- src/helpers/MQTTPrefsRecovery.h | 21 +++- .../test_mqtt_prefs_atomic_store.cpp | 79 +++++++++++- 4 files changed, 184 insertions(+), 38 deletions(-) diff --git a/MQTT_INTERNALS.md b/MQTT_INTERNALS.md index eb5c7944..bc1e13b2 100644 --- a/MQTT_INTERNALS.md +++ b/MQTT_INTERNALS.md @@ -228,8 +228,14 @@ exists. Boot recovery selects the usable primary/temp/backup without overwriting an opaque future or corrupt primary. A valid future-version temp that reached the rename phase wins over the stale backup and is held for newer firmware. If a temp claims a future version but uses grammar this firmware -cannot parse, or cannot be classified because scratch allocation fails, recovery may run -the last usable backup but retains the uncertain temp and holds all observer writes. A +cannot parse, or cannot be classified because scratch allocation fails, recovery renames +nothing at all: it reads the last usable backup straight out of `/mqtt.json.bak`, leaves +the primary name empty, and holds all observer writes. The empty primary name is the only +record that the candidate had already passed the backup rename, so publishing the backup +would make the candidate indistinguishable from a stale artifact — and the next boot, the +one with enough heap or new enough firmware to finally read it, would delete it. Leaving +the names alone means that boot promotes the candidate through the ordinary temp rule, or +falls back to the backup if it turns out to be definitively corrupt. A definitively corrupt/incomplete current-version temp may be discarded for that backup. If power fails during the very first migration, there is no JSON primary or backup yet; recovery discards a definitively invalid temp so the intact `/mqtt_prefs` source can be diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 930676a5..c901fb45 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -409,8 +409,14 @@ enum class JsonPrefsLoadResult : uint8_t { UnsupportedVersion, FutureClaimed, Invalid, + NoMemory, }; +static bool jsonPrefsLoaded(JsonPrefsLoadResult result) { + return result == JsonPrefsLoadResult::Loaded || + result == JsonPrefsLoadResult::LoadedWithRepairs; +} + static JsonPrefsLoadResult loadMqttJsonFile(FILESYSTEM* fs, const char* path, MQTTPrefs* output) { if (output == nullptr) return JsonPrefsLoadResult::Invalid; @@ -441,6 +447,19 @@ static JsonPrefsLoadResult loadMqttJsonFile(FILESYSTEM* fs, const char* path, return repaired ? JsonPrefsLoadResult::LoadedWithRepairs : JsonPrefsLoadResult::Loaded; } +// Parse `path` through a heap scratch object and publish it over `dest` only +// once the whole file is known good, so a late failure cannot leave the live +// preferences half-loaded. +static JsonPrefsLoadResult adoptMqttJsonFile(FILESYSTEM* fs, const char* path, + MQTTPrefs* dest) { + MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; + if (scratch == nullptr) return JsonPrefsLoadResult::NoMemory; + const JsonPrefsLoadResult result = loadMqttJsonFile(fs, path, scratch); + if (jsonPrefsLoaded(result)) memcpy(dest, scratch, sizeof(*dest)); + delete scratch; + return result; +} + static MQTTPrefsRecovery::FileState mqttJsonFileState(FILESYSTEM* fs, const char* path) { if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; @@ -462,7 +481,15 @@ static MQTTPrefsRecovery::FileState mqttJsonFileState(FILESYSTEM* fs, const char : MQTTPrefsRecovery::FileState::Preserve; } -static bool recoverMqttJsonFiles(FILESYSTEM* fs) { +// hold: observer writes must not replace whatever was left on disk. +// run_from_backup: nothing was renamed and /mqtt.json does not exist; this boot +// reads the last committed image straight out of /mqtt.json.bak. +struct MqttJsonRecovery { + bool hold; + bool run_from_backup; +}; + +static MqttJsonRecovery recoverMqttJsonFiles(FILESYSTEM* fs) { const MQTTPrefsRecovery::FileState primary = mqttJsonFileState(fs, "/mqtt.json"); const MQTTPrefsRecovery::FileState temp = mqttJsonFileState(fs, "/mqtt.json.tmp"); const MQTTPrefsRecovery::FileState backup = mqttJsonFileState(fs, "/mqtt.json.bak"); @@ -479,17 +506,18 @@ static bool recoverMqttJsonFiles(FILESYSTEM* fs) { fs->remove("/mqtt.json.bak"); } } - return MQTTPrefsRecovery::uncertain(primary) || - MQTTPrefsRecovery::uncertain(temp) || - MQTTPrefsRecovery::uncertain(backup); + return {MQTTPrefsRecovery::uncertain(primary) || + MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup), + false}; } if (action == MQTTPrefsRecovery::Action::DiscardTemp) { if (fs->remove("/mqtt.json.tmp")) { MESH_DEBUG_PRINTLN("MQTT: discarded incomplete first-migration JSON temp"); - return false; + return {false, false}; } MESH_DEBUG_PRINTLN("MQTT: could not discard incomplete /mqtt.json temp; source held"); - return true; + return {true, false}; } if (action == MQTTPrefsRecovery::Action::PromoteTemp) { if (fs->rename("/mqtt.json.tmp", "/mqtt.json")) { @@ -498,11 +526,21 @@ static bool recoverMqttJsonFiles(FILESYSTEM* fs) { fs->remove("/mqtt.json.bak"); } MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction temp"); - return MQTTPrefsRecovery::uncertain(temp) || - MQTTPrefsRecovery::uncertain(backup); + return {MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup), + false}; } MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json temp; files preserved"); - return true; + return {true, false}; + } + if (action == MQTTPrefsRecovery::Action::UseBackupHeld) { + // Deliberately rename nothing. The empty primary name is what records that + // the interrupted commit had already moved the old image aside, and a boot + // that cannot classify the candidate must not spend that record: a later + // boot with more heap, or firmware that understands the candidate, promotes + // it through the ordinary rule instead of deleting it as a stale artifact. + MESH_DEBUG_PRINTLN("MQTT: unresolved /mqtt.json candidate; running the backup in place and holding writes"); + return {true, true}; } if (action == MQTTPrefsRecovery::Action::PromoteBackup) { if (fs->rename("/mqtt.json.bak", "/mqtt.json")) { @@ -512,13 +550,14 @@ static bool recoverMqttJsonFiles(FILESYSTEM* fs) { fs->remove("/mqtt.json.tmp"); } MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt.json from transaction backup"); - return MQTTPrefsRecovery::uncertain(temp) || - MQTTPrefsRecovery::uncertain(backup); + return {MQTTPrefsRecovery::uncertain(temp) || + MQTTPrefsRecovery::uncertain(backup), + false}; } MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt.json backup; files preserved"); - return true; + return {true, false}; } - return false; + return {false, false}; } static MQTTPrefsRecovery::FileState mqttPrefsFileState(FILESYSTEM* fs, const char* path) { @@ -579,6 +618,12 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt_prefs temp; files preserved"); return true; } + if (action == MQTTPrefsRecovery::Action::UseBackupHeld) { + // Unreachable for the binary format, whose classifier never reports an + // uncertain state. Hold rather than fall through to "nothing to do" if a + // later classifier change makes it reachable. + return true; + } if (action == MQTTPrefsRecovery::Action::PromoteBackup) { if (fs->rename("/mqtt_prefs.bak", "/mqtt_prefs")) { // Symmetric case: a usable backup is now primary, so any interrupted @@ -784,7 +829,26 @@ private: void CommonCLI::loadMQTTPrefs( FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade) { setMQTTPrefsDefaults(&_mqtt_prefs); - _mqtt_prefs_hold = recoverMqttJsonFiles(fs); + const MqttJsonRecovery recovery = recoverMqttJsonFiles(fs); + _mqtt_prefs_hold = recovery.hold; + + // An interrupted commit left a candidate this boot cannot classify, so + // recovery renamed nothing. Read the last committed image out of the backup + // rather than publishing it: the transaction filenames must survive this boot + // intact for a later one to promote the candidate. + if (recovery.run_from_backup) { + _legacy_tail.valid = false; + const JsonPrefsLoadResult backup_result = + adoptMqttJsonFile(fs, "/mqtt.json.bak", &_mqtt_prefs); + if (jsonPrefsLoaded(backup_result)) { + MESH_DEBUG_PRINTLN("MQTT: running the /mqtt.json backup; candidate preserved and writes held"); + } else { + setMQTTPrefsDefaults(&_mqtt_prefs); + MESH_DEBUG_PRINTLN("MQTT: /mqtt.json backup became unreadable; using defaults (files preserved)"); + } + return; + } + if (_mqtt_prefs_hold && !fs->exists("/mqtt.json") && (fs->exists("/mqtt.json.tmp") || fs->exists("/mqtt.json.bak"))) { _legacy_tail.valid = false; @@ -796,18 +860,9 @@ void CommonCLI::loadMQTTPrefs( // stale binary snapshot when JSON is corrupt, unreadable, or from a future // schema: preserve it and run defaults until an operator resolves it. if (fs->exists("/mqtt.json")) { - MQTTPrefs* scratch = new (std::nothrow) MQTTPrefs; - if (scratch == nullptr) { - _mqtt_prefs_hold = true; - _legacy_tail.valid = false; - MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json; source preserved"); - return; - } - const JsonPrefsLoadResult json_result = loadMqttJsonFile(fs, "/mqtt.json", scratch); - if (json_result == JsonPrefsLoadResult::Loaded || - json_result == JsonPrefsLoadResult::LoadedWithRepairs) { - memcpy(&_mqtt_prefs, scratch, sizeof(_mqtt_prefs)); - delete scratch; + const JsonPrefsLoadResult json_result = + adoptMqttJsonFile(fs, "/mqtt.json", &_mqtt_prefs); + if (jsonPrefsLoaded(json_result)) { _legacy_tail.valid = false; if (json_result == JsonPrefsLoadResult::LoadedWithRepairs) { MESH_DEBUG_PRINTLN("MQTT: repaired out-of-range values in /mqtt.json"); @@ -818,10 +873,11 @@ void CommonCLI::loadMQTTPrefs( } return; } - delete scratch; _mqtt_prefs_hold = true; _legacy_tail.valid = false; - if (json_result == JsonPrefsLoadResult::UnsupportedVersion) { + if (json_result == JsonPrefsLoadResult::NoMemory) { + MESH_DEBUG_PRINTLN("MQTT: no memory to validate /mqtt.json; source preserved"); + } else if (json_result == JsonPrefsLoadResult::UnsupportedVersion) { MESH_DEBUG_PRINTLN("MQTT: /mqtt.json uses a future version; using defaults (file preserved)"); } else if (json_result == JsonPrefsLoadResult::FutureClaimed) { MESH_DEBUG_PRINTLN( diff --git a/src/helpers/MQTTPrefsRecovery.h b/src/helpers/MQTTPrefsRecovery.h index 37073209..4f6b15ac 100644 --- a/src/helpers/MQTTPrefsRecovery.h +++ b/src/helpers/MQTTPrefsRecovery.h @@ -7,9 +7,16 @@ // the primary name. On a reset, the loader uses this policy before decoding // the primary. FutureUsable is syntactically valid but belongs to newer // firmware. FutureClaimed and Indeterminate cannot be safely classified by -// this firmware, so recovery may use a known-good backup but must retain the +// this firmware, so recovery may run a known-good backup but must retain the // uncertain image and hold further writes. Preserve is definitively invalid or // unsupported and may be discarded only where the transaction policy permits. +// +// The filenames are the transaction state. A temp that exists while the primary +// name is empty says the commit had already passed the backup rename, and that +// is the only record of it — so an uncertain temp is answered with +// UseBackupHeld, which renames nothing. Publishing the backup instead would +// make the candidate look like an ordinary stale artifact to the next boot, +// which would delete it exactly when it finally became readable. namespace MQTTPrefsRecovery { enum class FileState : uint8_t { @@ -27,6 +34,10 @@ enum class Action : uint8_t { DiscardTemp, PromoteTemp, PromoteBackup, + // Run the backup where it lies, changing nothing on disk, and hold writes. + // Every later boot re-runs this policy against the same three names until one + // of them can classify the candidate. + UseBackupHeld, }; inline Action select(FileState primary, FileState temp, FileState backup) { @@ -50,10 +61,12 @@ inline Action select(FileState primary, FileState temp, FileState backup) { } // FutureClaimed and Indeterminate may be a completed image this firmware - // cannot classify. A known-good backup can run this boot, but without one - // preserve the only candidate under the authoritative name and hold writes. + // cannot classify. A known-good backup can run this boot, but it must run + // from its own name: promoting it would spend the empty primary name that + // marks the candidate as mid-commit. Without such a backup, preserve the only + // candidate under the authoritative name and hold writes. if (temp == FileState::FutureClaimed || temp == FileState::Indeterminate) { - return backup == FileState::Usable ? Action::PromoteBackup : Action::PromoteTemp; + return backup == FileState::Usable ? Action::UseBackupHeld : Action::PromoteTemp; } // No temp survived. The backup is the only recoverable image, even when it diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp index ddd6e184..a5b28be1 100644 --- a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -467,6 +467,9 @@ public: _files.erase("/mqtt_prefs.tmp"); return; } + if (action == Recovery::Action::UseBackupHeld) { + return; // production renames nothing and runs the backup where it lies + } if (action == Recovery::Action::PromoteBackup) { rename("/mqtt_prefs.bak", "/mqtt_prefs"); if (backup == Recovery::FileState::Usable && !Recovery::uncertain(temp) && @@ -930,15 +933,83 @@ TEST(MQTTPrefsAtomicStore, AmbiguousFutureOrOomTempIsNeverDeleted) { store.recover(Recovery::FileState::Missing, uncertain, Recovery::FileState::Usable); - // The supported backup can run this boot, but the candidate that this - // firmware could not classify remains available to newer firmware or a - // later boot with more heap. Its presence also blocks a new transaction. - EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + // The supported backup runs this boot from its own name. Nothing is + // renamed, so the empty primary name still records that the candidate had + // reached the publish phase. Its presence also blocks a new transaction. + EXPECT_FALSE(store.has("/mqtt_prefs")); EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.has("/mqtt_prefs.bak")); EXPECT_FALSE(store.canStartSave()); } } +TEST(MQTTPrefsAtomicStore, PreservedCandidateIsPromotedByTheBootThatCanReadIt) { + // The whole point of keeping an uncertain temp is that a later boot can act + // on it. Publishing the backup on the first boot would defeat that: the + // candidate would then look like a stale artifact next to a usable primary, + // and get deleted exactly when it finally became readable. + const struct { + Recovery::FileState first_boot; + Recovery::FileState second_boot; + // A promoted current-format image ends the transaction, so its backup goes + // too. A future-format one keeps the last readable image and stays held. + bool clears_backup; + } cases[] = { + // Classification scratch could not be allocated, then heap recovered. + {Recovery::FileState::Indeterminate, Recovery::FileState::Usable, true}, + // Downgraded firmware could not parse it, then the node rolled forward. + {Recovery::FileState::FutureClaimed, Recovery::FileState::Usable, true}, + // Still a newer schema on the second boot, but now classifiable. + {Recovery::FileState::FutureClaimed, Recovery::FileState::FutureUsable, false}, + }; + + for (const auto& test_case : cases) { + SpiffsMqttTransaction store; + store.cutAt(SpiffsMqttTransaction::Boundary::AfterBackupRename); + + store.recover(Recovery::FileState::Missing, test_case.first_boot, + Recovery::FileState::Usable); + ASSERT_TRUE(store.has("/mqtt_prefs.tmp")); + ASSERT_FALSE(store.has("/mqtt_prefs")); + + store.recover(Recovery::FileState::Missing, test_case.second_boot, + Recovery::FileState::Usable); + EXPECT_EQ(SpiffsMqttTransaction::newImage(), store.primary()); + EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); + EXPECT_EQ(test_case.clears_backup, !store.has("/mqtt_prefs.bak")); + EXPECT_EQ(test_case.clears_backup, store.canStartSave()); + } +} + +TEST(MQTTPrefsAtomicStore, CandidateThatProvesInvalidYieldsToTheHeldBackup) { + SpiffsMqttTransaction store; + store.cutAt(SpiffsMqttTransaction::Boundary::AfterBackupRename); + store.recover(Recovery::FileState::Missing, Recovery::FileState::Indeterminate, + Recovery::FileState::Usable); + + // The second boot can classify it and finds it definitively corrupt, so the + // last committed image is published and the candidate is dropped. The node + // leaves the held state without ever having discarded an unread candidate. + store.recover(Recovery::FileState::Missing, Recovery::FileState::Preserve, + Recovery::FileState::Usable); + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.canStartSave()); +} + +TEST(MQTTPrefsAtomicStore, UncertainTempWithNoUsableBackupStillOwnsThePrimaryName) { + // With no image that can run this boot, the candidate is the only thing left + // to protect, so it takes the authoritative name and CommonCLI holds it. + EXPECT_EQ(Recovery::Action::PromoteTemp, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::Indeterminate, + Recovery::FileState::Missing)); + EXPECT_EQ(Recovery::Action::PromoteTemp, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::FutureClaimed, + Recovery::FileState::Preserve)); +} + TEST(MQTTPrefsAtomicStore, UsablePrimaryDoesNotCleanIndeterminateArtifact) { SpiffsMqttTransaction store; store.cutDuringTempWrite(); From 28877f3017ce48d1b132d8464d1d1b441f753223 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 15 Aug 2026 21:41:20 -0700 Subject: [PATCH 25/93] fix(mqtt): report an undeletable rejected /mqtt.json temp as unresolved Cleanup before the commit phase ignored whether the temp was actually removed. On a fresh install a complete, byte-verified temp that failed schema verification (or whose read-back failed in finish()) could survive a failed remove() with no primary to outrank it, and boot recovery then promoted the very value the CLI had just reported as rolled back. Both cleanups now return a disposition: success means the temp is gone or an existing primary is authoritative. A false disposition maps to the new CleanupIndeterminate, which latches the same indeterminate reply as a commit that could not be rolled back. A short write is excluded, since it leaves structurally incomplete JSON that recovery classifies as invalid. Recovery also stops spending transaction state on an opaque backup: an uncertain temp beside a FutureUsable or uncertain backup now holds both names and runs defaults instead of promoting the candidate into the authoritative name, where the "any primary owns the name" rule would keep it even after a later boot proved it corrupt. --- src/helpers/CommonCLI.cpp | 49 ++++++-- src/helpers/MQTTPrefsAtomicStore.h | 38 ++++-- src/helpers/MQTTPrefsRecovery.h | 33 +++-- .../test_mqtt_prefs_atomic_store.cpp | 116 ++++++++++++++++-- 4 files changed, 197 insertions(+), 39 deletions(-) diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index c901fb45..bf015527 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -542,6 +542,13 @@ static MqttJsonRecovery recoverMqttJsonFiles(FILESYSTEM* fs) { MESH_DEBUG_PRINTLN("MQTT: unresolved /mqtt.json candidate; running the backup in place and holding writes"); return {true, true}; } + if (action == MQTTPrefsRecovery::Action::RunDefaultsHeld) { + // Same reasoning, with no image this firmware can run: renaming either file + // would spend transaction state that a later boot still needs. + MESH_DEBUG_PRINTLN("MQTT: unresolved /mqtt.json candidate and no runnable backup; " + "using defaults (files preserved)"); + return {true, false}; + } if (action == MQTTPrefsRecovery::Action::PromoteBackup) { if (fs->rename("/mqtt.json.bak", "/mqtt.json")) { if (backup == MQTTPrefsRecovery::FileState::Usable && @@ -618,7 +625,8 @@ static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt_prefs temp; files preserved"); return true; } - if (action == MQTTPrefsRecovery::Action::UseBackupHeld) { + if (action == MQTTPrefsRecovery::Action::UseBackupHeld || + action == MQTTPrefsRecovery::Action::RunDefaultsHeld) { // Unreachable for the binary format, whose classifier never reports an // uncertain state. Hold rather than fall through to "nothing to do" if a // later classifier change makes it reachable. @@ -766,26 +774,36 @@ public: return true; } - void discardFinishedTemp() { + // Drop a temp that was written and byte-verified but then rejected. Returns + // false when it is still on flash and no primary outranks it: recovery + // promotes a lone temp, so the caller cannot claim the change was undone. + bool discardFinishedTemp() { if (_open) _file.close(); _open = false; - if (_owns_temp && _fs->exists("/mqtt.json.tmp")) _fs->remove("/mqtt.json.tmp"); + bool removed = true; + if (_owns_temp && _fs->exists("/mqtt.json.tmp")) { + removed = _fs->remove("/mqtt.json.tmp"); + } _finished = false; _owns_temp = false; + return removed || _fs->exists("/mqtt.json"); } - void abort() { + // Same disposition contract as discardFinishedTemp(). Only unfinished staging + // is disposable here: once finish() has verified the temp, + // rollbackFailedCommit() has already decided its fate, and a temp that + // survived that is one recovery must resolve after reset. + bool abort() { if (_open) _file.close(); _open = false; - // Only unfinished staging is disposable here. Once finish() has verified - // the temp, rollbackFailedCommit() has already decided its fate, and a temp - // that survived that is one recovery must resolve after reset. + bool removed = true; if (_owns_temp && !_finished && _fs->exists("/mqtt.json.tmp")) { - _fs->remove("/mqtt.json.tmp"); + removed = _fs->remove("/mqtt.json.tmp"); } _finished = false; _owns_temp = false; _owns_backup = false; + return removed || _fs->exists("/mqtt.json"); } private: @@ -1130,6 +1148,12 @@ bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { return verify_result == JsonPrefsLoadResult::Loaded; }); + // A recovery file this firmware may still promote is holding the new image, + // so the change cannot be reported as rolled back. No further save can start + // until it is resolved: begin() refuses while either the temp or the backup + // exists, which is exactly the state that gets us here. + if (MQTTPrefsAtomicStore::saveOutcomeUnresolved(result)) _observer_save_indeterminate = true; + switch (result) { case MQTTPrefsAtomicStore::VerifiedImageResult::Committed: return true; @@ -1149,15 +1173,14 @@ bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { MESH_DEBUG_PRINTLN("MQTT: generated /mqtt.json temp failed schema validation; source preserved"); } break; + case MQTTPrefsAtomicStore::VerifiedImageResult::CleanupIndeterminate: + MESH_DEBUG_PRINTLN("MQTT: rejected /mqtt.json temp could not be removed and no primary outranks it; " + "recovery files preserved"); + break; case MQTTPrefsAtomicStore::VerifiedImageResult::CommitFailed: MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename; transaction rolled back"); break; case MQTTPrefsAtomicStore::VerifiedImageResult::CommitIndeterminate: - // The failed publish could not be undone, so a recovery file this - // firmware may still promote is holding the new image. No further save - // can start until it is resolved: begin() refuses while either the temp - // or the backup exists, which is exactly the state that gets us here. - _observer_save_indeterminate = true; MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt.json save failed during rename and could not be rolled back; " "recovery files preserved"); break; diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h index 91b7ae39..373dfa3b 100644 --- a/src/helpers/MQTTPrefsAtomicStore.h +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -13,22 +13,32 @@ namespace MQTTPrefsAtomicStore { // commit() allowed to publish it. A schema-verification failure discards the // finished temp; a commit failure is undone by rollbackFailedCommit(). // -// CommitFailed and CommitIndeterminate are distinct outcomes, not shades of the -// same one. Publishing moves the old primary aside before the verified temp -// takes its name, so a half-done commit leaves an image that boot recovery -// would promote. CommitFailed means that was undone and the change is really -// gone; CommitIndeterminate means it could not be, and the next boot may still -// come up with the new value. Callers must not report the two the same way. +// A failed save that is really gone and one that may still surface at the next +// boot are distinct outcomes, not shades of the same one. Publishing moves the +// old primary aside before the verified temp takes its name, so a half-done +// commit leaves an image that boot recovery would promote; CommitFailed means +// that was undone, CommitIndeterminate means it could not be. The same applies +// before the commit: a rejected temp that cleanup could not delete is still an +// image recovery may promote when no primary outranks it, which is +// CleanupIndeterminate. Callers must not report these as a rollback. enum class VerifiedImageResult : uint8_t { Committed, BeginFailed, WriteFailed, FinishFailed, VerifyFailed, + CleanupIndeterminate, CommitFailed, CommitIndeterminate, }; +// The change may still be on flash at the next boot, so the caller must not +// tell the operator it was rolled back. +inline bool saveOutcomeUnresolved(VerifiedImageResult result) { + return result == VerifiedImageResult::CleanupIndeterminate || + result == VerifiedImageResult::CommitIndeterminate; +} + template inline VerifiedImageResult writeVerifiedImage(Store& store, ImageWriter write_image, @@ -38,16 +48,24 @@ inline VerifiedImageResult writeVerifiedImage(Store& store, return VerifiedImageResult::BeginFailed; } if (!write_image()) { + // A short write leaves a structurally incomplete image that recovery + // classifies as invalid, so a temp that survives cleanup here cannot be + // promoted and the change really is gone. store.abort(); return VerifiedImageResult::WriteFailed; } if (!store.finish()) { - store.abort(); - return VerifiedImageResult::FinishFailed; + // Unlike a short write, this does not prove the bytes on disk are + // unusable: finish() also fails when a complete temp cannot be read back. + return store.abort() ? VerifiedImageResult::FinishFailed + : VerifiedImageResult::CleanupIndeterminate; } if (!verify_image()) { - store.discardFinishedTemp(); - return VerifiedImageResult::VerifyFailed; + // The temp is complete and byte-verified, only rejected by the schema. If + // it cannot be deleted and no primary outranks it, boot recovery promotes + // the very image this call is about to report as not saved. + return store.discardFinishedTemp() ? VerifiedImageResult::VerifyFailed + : VerifiedImageResult::CleanupIndeterminate; } if (!store.commit()) { // Undo the partial publish before answering. Only the store knows whether diff --git a/src/helpers/MQTTPrefsRecovery.h b/src/helpers/MQTTPrefsRecovery.h index 4f6b15ac..15eb014e 100644 --- a/src/helpers/MQTTPrefsRecovery.h +++ b/src/helpers/MQTTPrefsRecovery.h @@ -16,7 +16,9 @@ // is the only record of it — so an uncertain temp is answered with // UseBackupHeld, which renames nothing. Publishing the backup instead would // make the candidate look like an ordinary stale artifact to the next boot, -// which would delete it exactly when it finally became readable. +// which would delete it exactly when it finally became readable. When neither +// file can run this boot, RunDefaultsHeld keeps both names untouched for the +// same reason. namespace MQTTPrefsRecovery { enum class FileState : uint8_t { @@ -38,8 +40,15 @@ enum class Action : uint8_t { // Every later boot re-runs this policy against the same three names until one // of them can classify the candidate. UseBackupHeld, + // Neither the candidate nor the backup can run this boot. Change nothing, + // come up on defaults, and hold writes until a boot that can classify them. + RunDefaultsHeld, }; +inline bool uncertain(FileState state) { + return state == FileState::FutureClaimed || state == FileState::Indeterminate; +} + inline Action select(FileState primary, FileState temp, FileState backup) { // A primary of any kind owns the name. In particular, do not roll a newer // or corrupt primary back to an older backup just because it cannot be read @@ -63,10 +72,20 @@ inline Action select(FileState primary, FileState temp, FileState backup) { // FutureClaimed and Indeterminate may be a completed image this firmware // cannot classify. A known-good backup can run this boot, but it must run // from its own name: promoting it would spend the empty primary name that - // marks the candidate as mid-commit. Without such a backup, preserve the only - // candidate under the authoritative name and hold writes. - if (temp == FileState::FutureClaimed || temp == FileState::Indeterminate) { - return backup == FileState::Usable ? Action::UseBackupHeld : Action::PromoteTemp; + // marks the candidate as mid-commit. + // + // A backup this firmware cannot run is still the previous committed image, so + // the same argument applies to it: promoting the candidate would give it the + // authoritative name, and the "any primary owns the name" rule above would + // then keep it even on a boot that proves it corrupt. Spend the transaction + // state recorded in the filenames only once the backup is definitively no + // longer a usable fallback. + if (uncertain(temp)) { + if (backup == FileState::Usable) return Action::UseBackupHeld; + if (backup == FileState::FutureUsable || uncertain(backup)) { + return Action::RunDefaultsHeld; + } + return Action::PromoteTemp; } // No temp survived. The backup is the only recoverable image, even when it @@ -75,8 +94,4 @@ inline Action select(FileState primary, FileState temp, FileState backup) { return Action::None; } -inline bool uncertain(FileState state) { - return state == FileState::FutureClaimed || state == FileState::Indeterminate; -} - } // namespace MQTTPrefsRecovery diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp index a5b28be1..19794839 100644 --- a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -126,6 +126,8 @@ public: bool rollback_rename_fails = false; bool rollback_remove_fails = false; bool has_primary = true; + bool discard_remove_fails = false; + bool abort_remove_fails = false; }; explicit InMemoryJsonStore(FailurePoint failure) : _opts{failure, false, false, true} { @@ -157,8 +159,11 @@ public: bool finish() { ++finish_calls; _open = false; - if (_opts.failure == FailurePoint::Finish) return false; + // Production writes straight to /mqtt.json.tmp, so the bytes are on flash + // before finish() reads them back. A finish() failure can therefore be a + // failed read-back of an otherwise complete image, not a torn one. _files["/mqtt.json.tmp"] = _staging; + if (_opts.failure == FailurePoint::Finish) return false; _finished = true; return true; } @@ -196,18 +201,28 @@ public: return true; } - void discardFinishedTemp() { + // Both cleanups report the same disposition as production: false means a temp + // survived with no primary to outrank it, so boot recovery may still promote + // the image the caller is about to report as not saved. + bool discardFinishedTemp() { ++discard_calls; - _files.erase("/mqtt.json.tmp"); + const bool removed = !_opts.discard_remove_fails; + if (removed) _files.erase("/mqtt.json.tmp"); _finished = false; + return removed || has("/mqtt.json"); } - void abort() { + bool abort() { ++abort_calls; _open = false; _staging.clear(); - if (!_finished) _files.erase("/mqtt.json.tmp"); + bool removed = true; + if (!_finished && has("/mqtt.json.tmp")) { + removed = !_opts.abort_remove_fails; + if (removed) _files.erase("/mqtt.json.tmp"); + } _finished = false; + return removed || has("/mqtt.json"); } // Apply the boot-recovery policy to whatever the transaction left behind and @@ -467,8 +482,9 @@ public: _files.erase("/mqtt_prefs.tmp"); return; } - if (action == Recovery::Action::UseBackupHeld) { - return; // production renames nothing and runs the backup where it lies + if (action == Recovery::Action::UseBackupHeld || + action == Recovery::Action::RunDefaultsHeld) { + return; // production renames nothing, so the transaction state survives } if (action == Recovery::Action::PromoteBackup) { rename("/mqtt_prefs.bak", "/mqtt_prefs"); @@ -605,6 +621,49 @@ TEST(MQTTPrefsAtomicStore, UnrestorablePublishFailureIsReportedAsIndeterminate) EXPECT_EQ(InMemoryJsonStore::newImage(), first_save.imageAfterReboot()); } +TEST(MQTTPrefsAtomicStore, RejectedTempThatCannotBeDeletedIsReportedAsIndeterminate) { + // The temp is a complete, byte-verified, perfectly valid image: only the + // scratch allocation that reparses it failed. With no primary on a fresh + // install, a temp that cleanup cannot delete is exactly what boot recovery + // promotes, so this must not be answered as a rollback. + InMemoryJsonStore first_save(InMemoryJsonStore::Options{ + FailurePoint::Verify, false, false, /*has_primary=*/false, + /*discard_remove_fails=*/true}); + ASSERT_EQ(AtomicStore::VerifiedImageResult::CleanupIndeterminate, + runVerifiedJson(&first_save)); + EXPECT_TRUE(AtomicStore::saveOutcomeUnresolved( + AtomicStore::VerifiedImageResult::CleanupIndeterminate)); + EXPECT_TRUE(first_save.has("/mqtt.json.tmp")); + EXPECT_FALSE(first_save.canStartSave()); // retries fail until this resolves + EXPECT_EQ(InMemoryJsonStore::newImage(), first_save.imageAfterReboot()); + + // With a primary published, the same stuck temp cannot win recovery, so the + // change really is gone and the ordinary rejection verdict still holds. + InMemoryJsonStore with_primary(InMemoryJsonStore::Options{ + FailurePoint::Verify, false, false, /*has_primary=*/true, + /*discard_remove_fails=*/true}); + ASSERT_EQ(AtomicStore::VerifiedImageResult::VerifyFailed, + runVerifiedJson(&with_primary)); + EXPECT_FALSE(AtomicStore::saveOutcomeUnresolved( + AtomicStore::VerifiedImageResult::VerifyFailed)); + EXPECT_EQ(InMemoryJsonStore::oldImage(), with_primary.imageAfterReboot()); +} + +TEST(MQTTPrefsAtomicStore, UnverifiedTempThatCannotBeDeletedIsReportedAsIndeterminate) { + // finish() failing does not prove the bytes are torn — a complete image whose + // read-back failed still parses at the next boot. If abort() cannot remove it + // and no primary outranks it, the outcome is unresolved for the same reason. + InMemoryJsonStore store(InMemoryJsonStore::Options{ + FailurePoint::Finish, false, false, /*has_primary=*/false, + /*discard_remove_fails=*/false, /*abort_remove_fails=*/true}); + + ASSERT_EQ(AtomicStore::VerifiedImageResult::CleanupIndeterminate, + runVerifiedJson(&store)); + EXPECT_TRUE(store.has("/mqtt.json.tmp")); + EXPECT_FALSE(store.canStartSave()); + EXPECT_EQ(InMemoryJsonStore::newImage(), store.imageAfterReboot()); +} + TEST(MQTTPrefsAtomicStore, FirstSavePublishFailureLeavesNoImageToPromote) { InMemoryJsonStore store(InMemoryJsonStore::Options{ FailurePoint::Commit, false, false, /*has_primary=*/false}); @@ -1010,6 +1069,49 @@ TEST(MQTTPrefsAtomicStore, UncertainTempWithNoUsableBackupStillOwnsThePrimaryNam Recovery::FileState::Preserve)); } +TEST(MQTTPrefsAtomicStore, UncertainTempDoesNotSpendAnOpaqueBackup) { + // Newer firmware was replacing a future-version primary when power was lost. + // This boot can prove the backup is a syntactically valid future image but + // cannot run it, and cannot classify the candidate at all. + EXPECT_EQ(Recovery::Action::RunDefaultsHeld, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::FutureClaimed, + Recovery::FileState::FutureUsable)); + EXPECT_EQ(Recovery::Action::RunDefaultsHeld, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::Indeterminate, + Recovery::FileState::Indeterminate)); + // A backup that is definitively invalid has ceased to be a fallback, so the + // candidate may take the authoritative name. + EXPECT_EQ(Recovery::Action::PromoteTemp, + Recovery::select(Recovery::FileState::Missing, + Recovery::FileState::Indeterminate, + Recovery::FileState::Preserve)); +} + +TEST(MQTTPrefsAtomicStore, HeldOpaqueBackupSurvivesACandidateThatProvesInvalid) { + SpiffsMqttTransaction store; + store.cutAt(SpiffsMqttTransaction::Boundary::AfterBackupRename); + + // Nothing is renamed, so the empty primary name still records the interrupted + // commit and the previous committed image keeps its backup name. + store.recover(Recovery::FileState::Missing, Recovery::FileState::FutureClaimed, + Recovery::FileState::FutureUsable); + EXPECT_FALSE(store.has("/mqtt_prefs")); + EXPECT_TRUE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.has("/mqtt_prefs.bak")); + EXPECT_FALSE(store.canStartSave()); + + // A later boot understands both and finds the candidate corrupt. Because the + // first boot did not spend the primary name on it, the last committed image + // is still there to publish instead of being stranded behind a bad primary. + store.recover(Recovery::FileState::Missing, Recovery::FileState::Preserve, + Recovery::FileState::Usable); + EXPECT_EQ(SpiffsMqttTransaction::oldImage(), store.primary()); + EXPECT_FALSE(store.has("/mqtt_prefs.tmp")); + EXPECT_TRUE(store.canStartSave()); +} + TEST(MQTTPrefsAtomicStore, UsablePrimaryDoesNotCleanIndeterminateArtifact) { SpiffsMqttTransaction store; store.cutDuringTempWrite(); From 4bdbe33a4538a570fe970068c1c0fedb7c6366d1 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 17 Aug 2026 08:49:56 -0700 Subject: [PATCH 26/93] fix(mqtt): Align JWT reuse with renewal buffer --- src/helpers/MQTTConnectionPolicy.h | 14 +++++- src/helpers/bridges/MQTTBridge.cpp | 27 ++++++++---- .../test_mqtt_connection_policy.cpp | 44 +++++++++++++++---- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index 096c5d38..d7ff2f5e 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -160,16 +160,26 @@ static inline bool tokenNeedsRenewal(bool time_synced, uint32_t current_time, // A reconnect may keep its credentials only when their validity is known to // outlast the next handshake; uncertainty refreshes them before reconnecting. +// Clearing the renewal buffer is not enough: tokenNeedsRenewal() fires the moment +// remaining reaches it, so reuse must clear it by the handshake margin too, or the +// reused token is renewed — and bounced, on a broker enforcing exp — seconds later. +// Pass 0 for renewal_buffer_secs to gate on the flat margin alone. static inline bool canReuseJwtForReconnect(bool time_synced, bool has_token, bool force_mint, uint32_t current_time, - uint32_t token_expires_at) { + uint32_t token_expires_at, + uint32_t renewal_buffer_secs) { + // Saturate rather than wrap: a wrapped floor would silently weaken this gate. + const uint32_t floor_secs = + renewal_buffer_secs > UINT32_MAX - kJwtReconnectSafetyMarginSecs + ? UINT32_MAX + : renewal_buffer_secs + kJwtReconnectSafetyMarginSecs; return time_synced && has_token && !force_mint && token_expires_at >= kMinimumValidEpoch && current_time < token_expires_at && - (token_expires_at - current_time) > kJwtReconnectSafetyMarginSecs; + (token_expires_at - current_time) > floor_secs; } static inline bool renewalAttemptAllowed(uint32_t now, uint32_t last_attempt) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1ce0f1bb..efb33d69 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2001,8 +2001,11 @@ void MQTTBridge::teardownSlot(int index) { } // A stopped client needs connect(): reconnect() is a documented no-op on one, so reaching -// it here would strand the slot. The WiFi-transition teardown stops a client while leaving -// initial_connect_done set, so the ladder does see this state. +// it here would strand the slot. The producer is a failed esp_mqtt_client_start(), which +// leaves _started false while initial_connect_done stays set. Not the WiFi-drop teardown, +// which only stops slots still marked connected: a publishing slot's socket fails first, so +// the guard skips it — measured across a 62 s deauth, five slots, zero stops. An idle slot +// with no traffic to fail on is the one case that could still reach here that way. void MQTTBridge::reconnectSlotClient(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; @@ -2221,9 +2224,13 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns : 0; const bool force_mint_after_refusal = _slot_force_jwt_mint[index]; force_mint = force_mint || force_mint_after_refusal; + // Same buffer the renewal path uses, so a reconnect never keeps a token that + // the next maintenance pass would renew (and bounce) seconds later. + const uint32_t renewal_buffer_secs = MQTTConnectionPolicy::renewalBufferSecs( + static_cast(slotTokenLifetime(index))); const bool reuse_token = MQTTConnectionPolicy::canReuseJwtForReconnect( time_synced, has_token, force_mint, static_cast(current_time), - static_cast(expires_at)); + static_cast(expires_at), renewal_buffer_secs); const char* mint_reason = "none"; if (!reuse_token) { if (backoff_level < 0) { @@ -2238,8 +2245,12 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns mint_reason = "invalid-expiry"; } else if (current_time >= expires_at) { mint_reason = "expired"; + } else if (remaining_secs <= MQTTConnectionPolicy::kJwtReconnectSafetyMarginSecs) { + mint_reason = "safety-margin"; // too little left to outlast the handshake + } else if (remaining_secs <= renewal_buffer_secs) { + mint_reason = "renewal-due"; // the renewal path wants this token now } else { - mint_reason = "safety-margin"; + mint_reason = "renewal-imminent"; // it will, within the handshake margin } } const char* mint_result = reuse_token ? "REUSED" : "FAILED"; @@ -2282,8 +2293,8 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns if (slot_uses_jwt) { prepareJwtReconnect(true, -1); } - // Via the helper: reconnect() is a no-op on a client the WiFi-drop path - // stopped, which would probe forever without ever starting it. + // Via the helper: reconnect() is a no-op on a client whose start failed, + // which would probe forever without ever starting it. reconnectSlotClient(index); // If the connect callback fires and sets slot.connected = true, // it will clear circuit_breaker_tripped via the onConnect handler @@ -2319,8 +2330,8 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // Non-JWT slots — lightweight reconnect on existing client. MQTT_DEBUG_PRINTLN("MQTT%d reconnect (non-JWT, backoff %d)", index + 1, slot.reconnect_backoff); } - // Via the helper: reconnect() is a no-op on a client the WiFi-drop path - // stopped, which would back off forever without ever starting it. + // Via the helper: reconnect() is a no-op on a client whose start failed, + // which would back off forever without ever starting it. reconnectSlotClient(index); } } diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index c3cd469e..f92e5b89 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -141,16 +141,44 @@ TEST(MQTTConnectionPolicy, JwtReconnectReusesOnlyProvenValidCredentials) { const uint32_t now = 1735689600U; const uint32_t usable_expiry = now + Policy::kJwtReconnectSafetyMarginSecs + 1U; - EXPECT_TRUE(Policy::canReuseJwtForReconnect(true, true, false, now, usable_expiry)); + // Buffer 0 gates on the flat margin alone, which is what these cases cover. + EXPECT_TRUE(Policy::canReuseJwtForReconnect(true, true, false, now, usable_expiry, 0U)); EXPECT_FALSE(Policy::canReuseJwtForReconnect( - true, true, false, now, Policy::kMinimumValidEpoch - 1U)); - EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, 0U)); + true, true, false, now, Policy::kMinimumValidEpoch - 1U, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, 0U, 0U)); EXPECT_FALSE(Policy::canReuseJwtForReconnect( - true, true, false, now, now + Policy::kJwtReconnectSafetyMarginSecs)); - EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, now - 1U)); - EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, false, false, now, usable_expiry)); - EXPECT_FALSE(Policy::canReuseJwtForReconnect(false, true, false, now, usable_expiry)); - EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, true, now, usable_expiry)); + true, true, false, now, now + Policy::kJwtReconnectSafetyMarginSecs, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, now - 1U, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, false, false, now, usable_expiry, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(false, true, false, now, usable_expiry, 0U)); + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, true, now, usable_expiry, 0U)); +} + +TEST(MQTTConnectionPolicy, JwtReuseRefusesATokenTheRenewalPathIsAboutToBounce) { + const uint32_t now = 1735689600U; + const uint32_t buffer = Policy::renewalBufferSecs(3300U); // waev's 55-minute tokens + ASSERT_EQ(buffer, 300U); + const uint32_t floor_secs = buffer + Policy::kJwtReconnectSafetyMarginSecs; + + // The observed defect (d3, 2026-08-17T14:31): reused at 302 s remaining under the + // flat 60 s margin, then renewed five seconds later, which bounced the slot. + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, now + 302U, buffer)); + + // Anything the renewal path would act on is refused, and so is the slack above it. + EXPECT_FALSE(Policy::canReuseJwtForReconnect(true, true, false, now, now + buffer, buffer)); + EXPECT_FALSE( + Policy::canReuseJwtForReconnect(true, true, false, now, now + floor_secs, buffer)); + const uint32_t tightest_reuse = now + floor_secs + 1U; + EXPECT_TRUE(Policy::canReuseJwtForReconnect(true, true, false, now, tightest_reuse, buffer)); + + // What the margin above the buffer buys: the tightest token reuse will accept is still + // not due for renewal a full margin later, so no renewal can bounce in behind it. + EXPECT_FALSE(Policy::tokenNeedsRenewal( + true, now + Policy::kJwtReconnectSafetyMarginSecs, tightest_reuse, buffer)); + + // A buffer that would wrap the floor has to fail closed, not reuse a doomed token. + EXPECT_FALSE(Policy::canReuseJwtForReconnect( + true, true, false, now, now + 100000U, std::numeric_limits::max())); } TEST(MQTTConnectionPolicy, RenewalThrottleHasExactBoundaryAndHandlesRollover) { From 1fe78ef94628ad3b30d1f81e7fede9d5ea27f513 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 17 Aug 2026 14:34:19 -0700 Subject: [PATCH 27/93] fix(sensors): claim an I2C address after a successful init Several table entries share an address and not every driver verifies a chip ID: INA226::begin() only checks that the address ACKs, so an SHT4x at 0x44 was also registered as an INA226 and reported junk current on a second channel. Mark the address consumed once a driver initializes it so later entries cannot re-claim the same device. --- src/helpers/sensors/EnvironmentSensorManager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/helpers/sensors/EnvironmentSensorManager.cpp b/src/helpers/sensors/EnvironmentSensorManager.cpp index e2f0d33e..c3dfd9f7 100644 --- a/src/helpers/sensors/EnvironmentSensorManager.cpp +++ b/src/helpers/sensors/EnvironmentSensorManager.cpp @@ -650,6 +650,7 @@ bool EnvironmentSensorManager::begin() { continue; } MESH_DEBUG_PRINTLN("Found %s at address: %02X", def.name, def.address); + detected[def.address] = false; // consumed; later entries must not re-claim this device for (uint8_t sub = 0; sub < n && _active_sensor_count < MAX_ACTIVE_SENSORS; sub++) { _active_sensors[_active_sensor_count++] = { def.query, sub }; } From 44beeab00c1a37b8d6bad111ba949913a5a5b055 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 17 Aug 2026 14:59:59 -0700 Subject: [PATCH 28/93] fix(sensors): probe BMP/BME at both 0x76 and 0x77 Grove and other Bosch modules strap SDO high, so the 0x76-only table never initialized them. Add an alternate-address entry per Bosch sensor; the bus scan still gates every probe, and all four drivers verify a chip ID before claiming an address. Each sensor type has a single static driver instance, so skip an entry whose query is already active: the alternate address is a fallback, not a second device. --- .../sensors/EnvironmentSensorManager.cpp | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/helpers/sensors/EnvironmentSensorManager.cpp b/src/helpers/sensors/EnvironmentSensorManager.cpp index c3dfd9f7..54305692 100644 --- a/src/helpers/sensors/EnvironmentSensorManager.cpp +++ b/src/helpers/sensors/EnvironmentSensorManager.cpp @@ -539,6 +539,8 @@ static void query_bme680_bsec(uint8_t ch, uint8_t, CayenneLPP& lpp) { // are compiled in. The sentinel at the end keeps the array // non-empty regardless of which sensors are enabled. // +// Bosch BMP/BME SDO selects 0x76 or 0x77; probe both. +// // Ordering here determines channel assignment at runtime: // the first detected+initialized sensor gets channel 2, the // next gets channel 3, and so on. @@ -551,21 +553,27 @@ struct SensorDef { void (*query)(uint8_t channel, uint8_t sub_channel, CayenneLPP& telemetry); }; +#define TELEM_BOSCH_ALT_ADDR(addr) ((uint8_t)((addr) == 0x76 ? 0x77 : 0x76)) + static const SensorDef SENSOR_TABLE[] = { #if ENV_INCLUDE_AHTX0 { TELEM_AHTX_ADDRESS, "AHT10/AHT20", init_ahtx0, query_ahtx0 }, #endif #ifdef ENV_INCLUDE_BME680 - { TELEM_BME680_ADDRESS, "BME680", init_bme680, query_bme680 }, + { TELEM_BME680_ADDRESS, "BME680", init_bme680, query_bme680 }, + { TELEM_BOSCH_ALT_ADDR(TELEM_BME680_ADDRESS), "BME680", init_bme680, query_bme680 }, #endif #if ENV_INCLUDE_BME680_BSEC - { TELEM_BME680_ADDRESS, "BME680+BSEC", init_bme680_bsec, query_bme680_bsec }, + { TELEM_BME680_ADDRESS, "BME680+BSEC", init_bme680_bsec, query_bme680_bsec }, + { TELEM_BOSCH_ALT_ADDR(TELEM_BME680_ADDRESS), "BME680+BSEC", init_bme680_bsec, query_bme680_bsec }, #endif #if ENV_INCLUDE_BME280 - { TELEM_BME280_ADDRESS, "BME280", init_bme280, query_bme280 }, + { TELEM_BME280_ADDRESS, "BME280", init_bme280, query_bme280 }, + { TELEM_BOSCH_ALT_ADDR(TELEM_BME280_ADDRESS), "BME280", init_bme280, query_bme280 }, #endif #if ENV_INCLUDE_BMP280 - { TELEM_BMP280_ADDRESS, "BMP280", init_bmp280, query_bmp280 }, + { TELEM_BMP280_ADDRESS, "BMP280", init_bmp280, query_bmp280 }, + { TELEM_BOSCH_ALT_ADDR(TELEM_BMP280_ADDRESS), "BMP280", init_bmp280, query_bmp280 }, #endif #if ENV_INCLUDE_SHTC3 { 0x70, "SHTC3", init_shtc3, query_shtc3 }, @@ -603,6 +611,8 @@ static const SensorDef SENSOR_TABLE[] = { { 0, nullptr, nullptr, nullptr } // sentinel — keeps the array non-empty }; +#undef TELEM_BOSCH_ALT_ADDR + static const size_t SENSOR_TABLE_SIZE = (sizeof(SENSOR_TABLE) / sizeof(SENSOR_TABLE[0])) - 1; // ============================================================ @@ -640,6 +650,12 @@ bool EnvironmentSensorManager::begin() { _active_sensor_count = 0; for (size_t i = 0; i < SENSOR_TABLE_SIZE && _active_sensor_count < MAX_ACTIVE_SENSORS; i++) { const SensorDef& def = SENSOR_TABLE[i]; + // One static driver instance per type: an alternate address is a fallback, not a second device. + bool already_active = false; + for (int j = 0; j < _active_sensor_count; j++) { + if (_active_sensors[j].query == def.query) { already_active = true; break; } + } + if (already_active) continue; if (!detected[def.address]) { MESH_DEBUG_PRINTLN("%s not detected at I2C address %02X", def.name, def.address); continue; From 73480d50a343875820d54c307fc5dcf1d6e1a23c Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 18 Aug 2026 08:04:15 -0700 Subject: [PATCH 29/93] feat(mqtt): add ntxmesh broker preset --- MQTT_IMPLEMENTATION.md | 1 + src/helpers/MQTTPresets.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index f18fed75..ac74485d 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -161,6 +161,7 @@ below documents the current build. | `atvirastinklas` | `wss://mqtt-mc.atvirastinklas.lt:443` | JWT | — | | `gomesh` | `wss://mqtt.gomesh.dev:443` | JWT | — | | `idahomesh` | `wss://mqtt.idahomesh.org:443/mqtt` | JWT | — | +| `ntxmesh` | `wss://ntxmesh.dhovin.me:8883` | JWT | — | | `custom` | your own broker | User/pass, or JWT when `mqttN.audience` is set | `set mqttN.server` (see [custom broker setup](#custom-brokers)) | | `none` | (slot disabled) | — | — | diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 1083026d..176413cb 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -148,7 +148,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 34; +static const int MQTT_PRESET_COUNT = 35; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -195,6 +195,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { // JWT token auth; LE Gen-Y ECDSA chain (YE2 → Root YE → X2) still anchors at ISRG Root X1. { "gomesh", "wss://mqtt.gomesh.dev:443", "mqtt.gomesh.dev", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "idahomesh", "wss://mqtt.idahomesh.org:443/mqtt", "mqtt.idahomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "ntxmesh", "wss://ntxmesh.dhovin.me:8883", "ntxmesh.dhovin.me", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, }; // Find a preset by name, returns nullptr if not found From a0aec91c1784446a0613e1bc7f3d1145581f3651 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 18 Aug 2026 13:54:55 -0700 Subject: [PATCH 30/93] fix(mqtt): render the mbedtls diagnostic magnitude correctly ESP-IDF stores the mbedTLS stack error as a positive magnitude (it captures -ret), so negating it before printing produced "mbedtls:-0xFFFF8100" instead of "mbedtls:-0x7F00" for the record-buffer allocation failure. %04X is a minimum width, so nothing masked it. Normalisation moves into MQTTReplyFormat.h as mbedtlsErrorMagnitude() rather than staying inline in the bridge: inline is why this survived, since the existing test passes the already-correct magnitude straight into replyAppendf and never exercised the caller. It accepts either sign so a later SDK storing the real negative code still renders, and widens to int64_t before negating because negating INT32_MIN is undefined behaviour. MQTTReplyFormat.h also gains the stdint.h it was always missing: it compiled only because MQTTBridge.cpp pulls stdint in via other headers, and the host test includes the header standalone. --- src/helpers/MQTTReplyFormat.h | 9 ++++++++ src/helpers/bridges/MQTTBridge.cpp | 6 ++++-- .../test_mqtt_reply_format.cpp | 21 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/helpers/MQTTReplyFormat.h b/src/helpers/MQTTReplyFormat.h index bdb343d7..0ff5e38e 100644 --- a/src/helpers/MQTTReplyFormat.h +++ b/src/helpers/MQTTReplyFormat.h @@ -2,6 +2,7 @@ #include #include +#include #include // Bounded, clamping printf-append for the fixed-size CLI reply buffers used by @@ -41,3 +42,11 @@ static inline void replyAppendf(char* buf, size_t bufsize, int* pos, const char* *pos += n; if ((size_t)*pos >= bufsize) *pos = (int)bufsize - 1; // clamp truncated append } + +// Magnitude of an mbedTLS stack error, for printing as "-0x%04X". ESP-IDF 4.4 stores +// the positive magnitude (it captures -ret), so negating it produces a garbage unsigned; +// accepting either sign keeps this correct if a later SDK stores the negative code. +// The int64_t cast matters: negating INT32_MIN directly is undefined behaviour. +static inline uint32_t mbedtlsErrorMagnitude(int32_t stack_err) { + return (stack_err < 0) ? (uint32_t)(-(int64_t)stack_err) : (uint32_t)stack_err; +} diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index efb33d69..6013a4dd 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -559,9 +559,11 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) replyAppendf(buf, bufsize, &pos, ", tls:0x%04X", (unsigned)slot.last_tls_err); } } - // mbedTLS stack error (shown as negative hex per convention) + // mbedTLS stack error (shown as negative hex per convention). ESP-IDF stores the + // magnitude, not the negative mbedTLS code, so normalise instead of negating. if (slot.last_tls_stack_err != 0) { - replyAppendf(buf, bufsize, &pos, ", mbedtls:-0x%04X", (unsigned)(-slot.last_tls_stack_err)); + replyAppendf(buf, bufsize, &pos, ", mbedtls:-0x%04X", + (unsigned)mbedtlsErrorMagnitude(slot.last_tls_stack_err)); } // Socket errno if (slot.last_sock_errno != 0) { diff --git a/test/test_mqtt_reply_format/test_mqtt_reply_format.cpp b/test/test_mqtt_reply_format/test_mqtt_reply_format.cpp index 4b3726af..52ff33f8 100644 --- a/test/test_mqtt_reply_format/test_mqtt_reply_format.cpp +++ b/test/test_mqtt_reply_format/test_mqtt_reply_format.cpp @@ -88,6 +88,27 @@ TEST(ReplyAppendf, NoWritePastBufferAcrossOverflowingChain) { EXPECT_EQ(c.buf()[c.logical - 1], '\0'); // still NUL-terminated } +// The caller, not the helper, was where the mbedtls: line went wrong: production +// negated a value ESP-IDF already stores as a magnitude, so it rendered +// "mbedtls:-0xFFFF8100" instead of "-0x7F00". The chain test above never caught it +// because it passes the correct magnitude straight in. +TEST(MbedtlsErrorMagnitude, NormalisesEitherSignForTheNegativeHexConvention) { + EXPECT_EQ(mbedtlsErrorMagnitude(0x7F00), 0x7F00u); // ESP-IDF 4.4 stores +magnitude + EXPECT_EQ(mbedtlsErrorMagnitude(-0x7F00), 0x7F00u); // a later SDK may store the real code + EXPECT_EQ(mbedtlsErrorMagnitude(0), 0u); + + char buf[32]; + int pos = 0; + replyAppendf(buf, sizeof(buf), &pos, ", mbedtls:-0x%04X", + (unsigned)mbedtlsErrorMagnitude(0x7F00)); + EXPECT_STREQ(buf, ", mbedtls:-0x7F00"); +} + +// Negating INT32_MIN is undefined behaviour, which is why the helper widens first. +TEST(MbedtlsErrorMagnitude, HandlesInt32MinWithoutOverflow) { + EXPECT_EQ(mbedtlsErrorMagnitude(INT32_MIN), 2147483648u); +} + TEST(ReplyAppendf, ExactFitBoundary) { char buf[11]; // room for exactly "0123456789" + NUL int pos = 0; From a00bad2a4fbba7170a8b59ae2562cd9888193c37 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 19 Aug 2026 08:38:00 -0700 Subject: [PATCH 31/93] fix(mqtt): clear a slot that preferences say is disabled on bridge start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preset-application loop in begin() only had a positive branch, so when preferences named no preset the slot kept whatever the previous begin() left in RAM. Nothing else resets it: teardownSlot() deliberately preserves enabled and preset so a reconfigure can reuse the mbedTLS context, and the constructor clears them exactly once. A bridge restart could therefore resurrect a broker the operator had disabled and reconnect to it with the old credentials. Config fields only. The client belongs to destroySlotClients() and the token buffer to releaseSlotAuthToken(); clearing either here would strand a pointer esp-mqtt still holds in its config. Note the positive branch re-enabling a slot is correct and unchanged: a slot capped off at startup is disabled in RAM only, with preferences still naming a real preset, and the cap decision has to be re-made on each start. Reachable via WebConfig, where a pending full restart discards the per-slot restart mask, so a batch that disables a slot can end in a restart that never applied the disable. A plain CLI set cannot reach it: restartBridgeSlot() applies the change to the live bridge immediately, keeping RAM and preferences in agreement. Verified on hardware as a non-regression check for that reason — disable, restart, slot stays down; restore, restart, all slots return. --- src/helpers/bridges/MQTTBridge.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 6013a4dd..579242e4 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -940,6 +940,21 @@ void MQTTBridge::begin() { _slots[i].enabled = false; } } + } else { + // Prefs say this slot is off. Without this the slot keeps whatever the previous + // begin() left in RAM, so a restart resurrects a broker the operator disabled and + // reconnects to it with the old credentials. teardownSlot() deliberately preserves + // enabled/preset, and the constructor only clears them once, so nothing else does. + // Config fields only: the client belongs to destroySlotClients() and the token + // buffer to releaseSlotAuthToken(), and clearing either here would strand a pointer + // esp-mqtt still holds. + _slots[i].enabled = false; + _slots[i].preset = nullptr; + _slots[i].host[0] = '\0'; + _slots[i].username[0] = '\0'; + _slots[i].password[0] = '\0'; + _slots[i].audience[0] = '\0'; + _slots[i].port = 0; } } From 3c7eb13d1188b11eb0f7a7c3444c82a72b7fa502 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 19 Aug 2026 08:38:00 -0700 Subject: [PATCH 32/93] fix(mqtt): stop a wedged client before deleting it on the dirty-stop path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stop timeout bounded only the cooperative attempt. Its fallback tears down clients from the calling task, and both helpers skipped the stop entirely when the client reported not-connected — then deleted it anyway. That is the exact shape of the dangerous case: a client whose DISCONNECTED callback has already cleared its connected flag while it sits inside esp_mqtt_client_stop() reports not-connected, so the stop was skipped precisely when it mattered and the object was freed from under a live ESP-MQTT task. Both helpers take a force flag, and only the dirty branch passes it. force is deliberately not gated on connected(), and routes through the existing forceStop() rather than disconnect(), whose wait for the DISCONNECTED event is unbounded. Every other caller — the non-ESP32 release path, the Core-0 cooperative teardown, reconfigure, and the slot-cap path — keeps its current behaviour byte for byte. This does not make the path bounded. esp_mqtt_client_stop() waits on the client's STOPPED_BIT with portMAX_DELAY on the pinned framework, so a client wedged inside mbedTLS can still block the caller. The change trades a free-under-a-live-task for a wait, which is the safer of the two failure modes. Bounding it properly, and the surrounding destroy-while-still-stopping hazard, need the ownership handoff reworked; that is tracked separately. --- src/helpers/bridges/MQTTBridge.cpp | 28 +++++++++++++++++++++------- src/helpers/bridges/MQTTBridge.h | 8 ++++++-- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 579242e4..947722a3 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1175,8 +1175,10 @@ void MQTTBridge::LifecycleOps::releaseResources() { if (b->_mqtt_task_handle != nullptr) { vTaskDelete(b->_mqtt_task_handle); } - for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) b->teardownSlot(i); - b->destroySlotClients(); + // force: the task is already gone and the client is presumed wedged, so waiting on a + // DISCONNECTED event that may never arrive would hang this task (the app loop) forever. + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) b->teardownSlot(i, /*force=*/true); + b->destroySlotClients(/*force=*/true); } // Clean path (or a task that acked right at the deadline): the MQTT task // already disconnected/deleted its clients on Core 0 and self-terminated, so @@ -1724,11 +1726,17 @@ void MQTTBridge::releaseSlotAuthToken(int index) { slot.last_token_renewal = 0; } -void MQTTBridge::destroySlotClients() { +void MQTTBridge::destroySlotClients(bool force) { for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { MQTTSlot& slot = _slots[i]; if (slot.client != nullptr) { - if (slot.client->connected()) { + // force is deliberately NOT gated on connected(). The state it exists for — a client + // that already took its DISCONNECTED callback and is now stuck inside + // esp_mqtt_client_stop() — reports not-connected, so gating skipped the stop exactly + // when it mattered and left the object to be deleted from under a live IDF task. + if (force) { + slot.client->forceStop(); + } else if (slot.client->connected()) { slot.client->disconnect(); } #ifdef ESP_PLATFORM @@ -1986,12 +1994,18 @@ bool MQTTBridge::setupSlot(int index) { // the client object alive so a subsequent setupSlot() can reuse its mbedTLS // context. This is called both on reconfigure (preset change) and at shutdown; // destruction of the underlying client happens once in destroySlotClients(). -void MQTTBridge::teardownSlot(int index) { +void MQTTBridge::teardownSlot(int index, bool force) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; - if (slot.client && slot.client->connected()) { - slot.client->disconnect(); + // As in destroySlotClients(): force is not gated on connected(), because the wedged + // mid-stop state it exists for already reports not-connected. + if (slot.client && (force || slot.client->connected())) { + if (force) { + slot.client->forceStop(); + } else { + slot.client->disconnect(); + } #ifdef ESP_PLATFORM vTaskDelay(pdMS_TO_TICKS(50)); #else diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 0ad539e9..e94a5033 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -442,7 +442,10 @@ private: bool ensureSlotClient(int index); // Allocate this slot's persistent client + callbacks on first use bool ensureSlotAuthToken(int index); // Allocate this slot's JWT token buffer on first token creation void releaseSlotAuthToken(int index);// Free the token buffer (only with the client — see MQTTSlot) - void destroySlotClients(); // Delete all persistent clients (shutdown only) + // force=true stops each client without waiting for its DISCONNECTED event. Only the + // dirty-stop fallback passes it: disconnect()'s wait is unbounded, so a client already + // wedged in mbedTLS would block the caller — MyMesh::loop() — indefinitely. + void destroySlotClients(bool force = false); // Delete all persistent clients (shutdown only) bool setupSlot(int index); // Configure and connect the slot; false = not activated // Single definition of "this slot holds one of the _max_active_slots positions": // it is enabled and has been through a successful setupSlot(). Startup, the @@ -450,7 +453,8 @@ private: // exceeded by one route while another enforces it. int activatedSlotCount() const; bool canActivateSlot(int index) const; - void teardownSlot(int index); // Disconnect the slot's client (keeps the object alive) + // force as in destroySlotClients(): skip the unbounded wait, dirty-stop path only. + void teardownSlot(int index, bool force = false); // Disconnect the slot's client (keeps the object alive) // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a // no-op on a stopped client). See the definition. void reconnectSlotClient(int index); From 3666cb6da9c98fbea08061e3ba61a81035a9be45 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 20 Aug 2026 19:46:44 -0700 Subject: [PATCH 33/93] fix(mqtt): preserve WiFi outage state for alerts --- src/helpers/AlertReporter.cpp | 140 ++++++------------ src/helpers/AlertReporter.h | 13 +- src/helpers/MQTTConnectionPolicy.h | 18 +++ src/helpers/bridges/MQTTBridge.cpp | 43 ++++-- src/helpers/bridges/MQTTBridge.h | 22 ++- test/README.md | 3 +- .../test_mqtt_connection_policy.cpp | 15 ++ 7 files changed, 136 insertions(+), 118 deletions(-) diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index 4bce1fd0..b55ac90c 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -4,6 +4,9 @@ #include #include #include +#ifdef WITH_MQTT_BRIDGE +#include "AlertFaultPolicy.h" +#endif // Header layout for PAYLOAD_TYPE_GRP_TXT before encryption: // [0..3] timestamp (uint32_t LE) — also helps make packet_hash unique @@ -129,11 +132,9 @@ bool AlertReporter::resolveChannel(mesh::GroupChannel& out) const { void AlertReporter::onConfigChanged() { // Reset transient state so a config change re-arms the edge detector. #ifdef WITH_MQTT_BRIDGE - _wifi.state = OK; - _wifi.fired_at_ms = 0; + AlertFaultPolicy::reset(_wifi); for (size_t i = 0; i < sizeof(_mqtt) / sizeof(_mqtt[0]); i++) { - _mqtt[i].state = OK; - _mqtt[i].fired_at_ms = 0; + AlertFaultPolicy::reset(_mqtt[i]); } #endif } @@ -194,124 +195,75 @@ bool AlertReporter::sendText(const char* text) { return sendChannel(text); } -void AlertReporter::formatAge(unsigned long age_ms, char* out, size_t out_size) const { - unsigned long secs = age_ms / 1000UL; - unsigned long h = secs / 3600UL; - unsigned long m = (secs % 3600UL) / 60UL; - if (h > 0) { - snprintf(out, out_size, "%luh%lum", h, m); - } else { - snprintf(out, out_size, "%lum", m); - } -} - void AlertReporter::onLoop(unsigned long now_ms) { if (!_prefs || !_obs || !_obs->alert_enabled) return; if (!_mesh) return; - // Throttle: ~5 s cadence. The thresholds are minutes-scale so this is fine. - if ((long)(now_ms - _next_check_ms) < 0) return; - _next_check_ms = now_ms + 5000UL; + const uint32_t now = (uint32_t)now_ms; + if (!AlertFaultPolicy::checkDue(now, (uint32_t)_next_check_ms)) return; + _next_check_ms = AlertFaultPolicy::nextCheckMs(now); #ifdef WITH_MQTT_BRIDGE - // Clamp to a 60-minute floor regardless of what's in NodePrefs. The CLI - // already enforces this on set, but a stale prefs file or future field - // tweak shouldn't be able to drag the floor below 1 hour and let a - // flapping link spam the mesh. - // - // The rate limiter only applies between two real sends: fired_at_ms == 0 - // means "never fired since boot/config change", and treating it as a send - // at millis()==0 would suppress every first alert until uptime reaches - // min_interval (observed as a 30-minute alert.mqtt threshold not reporting - // until 60 minutes after a reboot). - uint16_t cfg_min = _obs->alert_min_interval_min; - if (cfg_min < 60) cfg_min = 60; - unsigned long min_interval_ms = (unsigned long)cfg_min * 60000UL; + const uint32_t min_interval_ms = + AlertFaultPolicy::minIntervalMs(_obs->alert_min_interval_min); // -------- WiFi fault -------- if (_obs->alert_wifi_minutes > 0) { - unsigned long wifi_disc_ms = MQTTBridge::getLastWifiDisconnectTime(); - unsigned long wifi_conn_ms = MQTTBridge::getWifiConnectedAtMillis(); - bool wifi_down = (wifi_disc_ms != 0 && wifi_conn_ms == 0); - unsigned long down_ms = wifi_down ? (now_ms - wifi_disc_ms) : 0; - unsigned long thresh_ms = (unsigned long)_obs->alert_wifi_minutes * 60000UL; - - if (_wifi.state == OK) { - if (wifi_down && down_ms >= thresh_ms && - (_wifi.fired_at_ms == 0 || (now_ms - _wifi.fired_at_ms) >= min_interval_ms)) { - char age[16]; - formatAge(down_ms, age, sizeof(age)); - uint8_t reason = MQTTBridge::getLastWifiDisconnectReason(); + if (_bridge != nullptr) { + const AlertFaultPolicy::OutageSnapshot snap = _bridge->getWifiOutageSnapshot(); + AlertFaultPolicy::TickResult r = AlertFaultPolicy::tick( + _wifi, now, snap, + AlertFaultPolicy::thresholdMs(_obs->alert_wifi_minutes), + min_interval_ms); + if (r.action == AlertFaultPolicy::Action::FireDown) { char text[80]; - if (reason != 0) { - snprintf(text, sizeof(text), "WiFi down %s (reason %u)", age, (unsigned)reason); - } else { - snprintf(text, sizeof(text), "WiFi down %s", age); - } + AlertFaultPolicy::formatWifiAlert(text, sizeof(text), r, snap); if (sendChannel(text)) { - _wifi.state = FIRING; - _wifi.fired_at_ms = now_ms; - _wifi.last_outage_started_ms = wifi_disc_ms; + AlertFaultPolicy::commitDown(_wifi, now, snap.started_ms); } - } - } else { // FIRING - if (!wifi_down) { - unsigned long total = (wifi_conn_ms != 0 && _wifi.last_outage_started_ms != 0) - ? (wifi_conn_ms - _wifi.last_outage_started_ms) : 0; - char age[16]; - formatAge(total, age, sizeof(age)); + } else if (r.action == AlertFaultPolicy::Action::FireRecovered) { char text[80]; - snprintf(text, sizeof(text), "WiFi recovered after %s", age); + AlertFaultPolicy::formatWifiAlert(text, sizeof(text), r, snap); sendChannel(text); - _wifi.state = OK; + AlertFaultPolicy::commitRecovered(_wifi); } } - } else if (_wifi.state == FIRING) { - _wifi.state = OK; // threshold disabled mid-fault: silently re-arm + } else { + AlertFaultPolicy::rearmIfDisabled(_wifi); } // -------- MQTT slot faults -------- if (_obs->alert_mqtt_minutes > 0 && _bridge != nullptr) { int n = MQTTBridge::getRuntimeSlotCount(); if (n > (int)(sizeof(_mqtt) / sizeof(_mqtt[0]))) n = (int)(sizeof(_mqtt) / sizeof(_mqtt[0])); - unsigned long thresh_ms = (unsigned long)_obs->alert_mqtt_minutes * 60000UL; + const uint32_t thresh_ms = AlertFaultPolicy::thresholdMs(_obs->alert_mqtt_minutes); for (int i = 0; i < n; i++) { - Fault& f = _mqtt[i]; + AlertFaultPolicy::Fault& f = _mqtt[i]; if (!_bridge->isSlotEnabledAndAttempted(i)) { - if (f.state == FIRING) f.state = OK; // slot disabled mid-fault + AlertFaultPolicy::rearmIfDisabled(f); continue; } - unsigned long outage_start = _bridge->getSlotCurrentOutageStartMs(i); - bool down = (outage_start != 0); - unsigned long down_ms = down ? (now_ms - outage_start) : 0; - - if (f.state == OK) { - if (down && down_ms >= thresh_ms && - (f.fired_at_ms == 0 || (now_ms - f.fired_at_ms) >= min_interval_ms)) { - char age[16]; - formatAge(down_ms, age, sizeof(age)); - char text[100]; - snprintf(text, sizeof(text), "MQTT slot %d (%s) down %s", - i + 1, _bridge->getSlotPresetName(i), age); - if (sendChannel(text)) { - f.state = FIRING; - f.fired_at_ms = now_ms; - f.last_outage_started_ms = outage_start; - } - } - } else { // FIRING - if (!down) { - unsigned long total = (f.last_outage_started_ms != 0) - ? (now_ms - f.last_outage_started_ms) : 0; - char age[16]; - formatAge(total, age, sizeof(age)); - char text[100]; - snprintf(text, sizeof(text), "MQTT slot %d (%s) recovered after %s", - i + 1, _bridge->getSlotPresetName(i), age); - sendChannel(text); - f.state = OK; + const uint32_t outage_start = (uint32_t)_bridge->getSlotCurrentOutageStartMs(i); + const AlertFaultPolicy::OutageSnapshot snap = + AlertFaultPolicy::fromStartMs(outage_start); + AlertFaultPolicy::TickResult r = AlertFaultPolicy::tick( + f, now, snap, thresh_ms, min_interval_ms); + if (r.action == AlertFaultPolicy::Action::FireDown) { + char text[100]; + AlertFaultPolicy::formatMqttDown(text, sizeof(text), i + 1, + _bridge->getSlotPresetName(i), + r.duration_ms); + if (sendChannel(text)) { + AlertFaultPolicy::commitDown(f, now, outage_start); } + } else if (r.action == AlertFaultPolicy::Action::FireRecovered) { + char text[100]; + AlertFaultPolicy::formatMqttRecovered(text, sizeof(text), i + 1, + _bridge->getSlotPresetName(i), + r.duration_ms); + sendChannel(text); + AlertFaultPolicy::commitRecovered(f); } } } diff --git a/src/helpers/AlertReporter.h b/src/helpers/AlertReporter.h index 390262b6..6f52b571 100644 --- a/src/helpers/AlertReporter.h +++ b/src/helpers/AlertReporter.h @@ -6,6 +6,7 @@ #ifdef WITH_MQTT_BRIDGE #include "bridges/MQTTBridge.h" +#include "AlertFaultPolicy.h" #endif /** @@ -89,14 +90,6 @@ public: private: bool resolveChannel(mesh::GroupChannel& out) const; bool sendChannel(const char* text); - void formatAge(unsigned long age_ms, char* out, size_t out_size) const; - - enum FaultState { OK, FIRING }; - struct Fault { - FaultState state; - unsigned long fired_at_ms; // millis() when we last sent a "down" alert - unsigned long last_outage_started_ms; // remembered so the recovered msg can quote duration - }; NodePrefs* _prefs; MQTTPrefs* _obs; @@ -104,8 +97,8 @@ private: CommonCLICallbacks* _callbacks; #ifdef WITH_MQTT_BRIDGE MQTTBridge* _bridge; - Fault _wifi; - Fault _mqtt[RUNTIME_MQTT_SLOTS]; + AlertFaultPolicy::Fault _wifi; + AlertFaultPolicy::Fault _mqtt[RUNTIME_MQTT_SLOTS]; #endif unsigned long _next_check_ms; }; diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index d7ff2f5e..cf459459 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -120,6 +120,24 @@ static inline uint8_t nextWifiBackoffAttempt(uint8_t attempt) { return attempt < 5 ? static_cast(attempt + 1) : attempt; } +// Current WiFi outage start (millis), or 0 while associated. handleWiFiConnection() +// applies this on each STA status observation. Reconnect attempts that fire +// while already down must pass connected=false and last_connected=false so the +// start is preserved — WiFi.disconnect() in the backoff ladder is not a new +// outage, and must not become the clock AlertReporter quotes as downtime. +static inline uint32_t wifiCurrentOutageStartMs(uint32_t now, bool connected, + bool last_connected, + uint32_t current_start, + bool initialized) { + if (!initialized) { + return connected ? 0U : now; + } + if (connected) { + return last_connected ? current_start : 0U; + } + return last_connected ? now : current_start; +} + // Each later slot expires up to five percent of the base lifetime earlier, // capped at five minutes per slot. Runtime slot indexes are bounded by the // persisted MQTT slot count; the final clamp also prevents underflow if this diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 947722a3..39c8f05b 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -662,7 +662,7 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg _snmp_agent(nullptr), #endif _last_wifi_check(0), _last_wifi_status(WL_DISCONNECTED), _wifi_status_initialized(false), - _wifi_disconnected_time(0), _last_wifi_reconnect_attempt(0), _wifi_reconnect_backoff_attempt(0), + _wifi_outage_bits{0}, _last_wifi_reconnect_attempt(0), _wifi_reconnect_backoff_attempt(0), _last_slot_reconnect_ms(0) #ifdef ESP_PLATFORM , _packet_queue_handle(nullptr), _mqtt_task_handle(nullptr), @@ -1257,16 +1257,23 @@ void MQTTBridge::initializeWiFiInTask() { switch(event) { case ARDUINO_EVENT_WIFI_STA_GOT_IP: MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); + setWifiOutage(AlertFaultPolicy::applyWifiGotIp(wifiOutage())); + _wifi_reconnect_backoff_attempt = 0; // Set flag to trigger NTP sync from loop() instead of doing it here if (!_ntp_synced && !_ntp_sync_pending) { _ntp_sync_pending = true; } break; - case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: - s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; - s_wifi_disconnect_time = millis(); + case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: { + const uint8_t reason = info.wifi_sta_disconnected.reason; + const unsigned long t = millis(); + s_wifi_disconnect_reason = reason; + s_wifi_disconnect_time = t; + setWifiOutage(AlertFaultPolicy::applyWifiDisconnectEvent( + (uint32_t)t, reason, wifiOutage())); MQTT_DEBUG_PRINTLN("WiFi disconnected: reason %d", s_wifi_disconnect_reason); break; + } default: break; } @@ -2795,11 +2802,19 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (!_wifi_status_initialized) { _last_wifi_status = current_wifi_status; _wifi_status_initialized = true; - if (current_wifi_status != WL_CONNECTED) { - _wifi_disconnected_time = now; - } + setWifiOutage(AlertFaultPolicy::applyWifiStatus( + (uint32_t)now, current_wifi_status == WL_CONNECTED, wifiOutage(), false)); } if (now - _last_wifi_check <= 10000) { + // Events own the snapshot between 10 s polls. If STA is associated again + // and GOT_IP was missed, still close the outage so a flap contained + // between polls does not look like one continuous downtime. + if (current_wifi_status == WL_CONNECTED) { + AlertFaultPolicy::OutageSnapshot snap = wifiOutage(); + if (snap.down) { + setWifiOutage(AlertFaultPolicy::applyWifiGotIp(snap)); + } + } return false; } _last_wifi_check = now; @@ -2807,7 +2822,8 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (current_wifi_status == WL_CONNECTED) { if (_last_wifi_status != WL_CONNECTED) { transitioned_to_connected = true; - _wifi_disconnected_time = 0; + setWifiOutage(AlertFaultPolicy::applyWifiStatus( + (uint32_t)now, true, wifiOutage(), true)); s_wifi_connected_at = now; _wifi_reconnect_backoff_attempt = 0; #ifdef ESP_PLATFORM @@ -2833,8 +2849,11 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { } _last_wifi_status = WL_CONNECTED; } else { - if (_last_wifi_status == WL_CONNECTED) { - _wifi_disconnected_time = now; + const bool last_connected = (_last_wifi_status == WL_CONNECTED); + AlertFaultPolicy::OutageSnapshot snap = AlertFaultPolicy::applyWifiStatus( + (uint32_t)now, false, wifiOutage(), true); + setWifiOutage(snap); + if (last_connected) { s_wifi_connected_at = 0; // Disconnect all slot clients when WiFi drops for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { @@ -2842,13 +2861,13 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { _slots[i].client->disconnect(); } } - } else if (_wifi_disconnected_time > 0) { + } else if (snap.down) { // Backoff ladder + wrap-safe timing live in MQTTConnectionPolicy (Phase 6), // exercised by host tests. Behavior is unchanged: both the link-down // duration and the since-last-attempt interval must clear the current rung // (elapsedMs is the wrap-safe form of the old ULONG_MAX branch). if (MQTTConnectionPolicy::wifiReconnectDue( - (uint32_t)now, (uint32_t)_wifi_disconnected_time, + (uint32_t)now, snap.started_ms, (uint32_t)_last_wifi_reconnect_attempt, _wifi_reconnect_backoff_attempt)) { _last_wifi_reconnect_attempt = now; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index e94a5033..e2d27253 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -11,6 +11,7 @@ #include "helpers/MQTTPacketFilter.h" #include "helpers/MQTTPresets.h" #include "helpers/MQTTLifecycle.h" +#include "helpers/AlertFaultPolicy.h" #include #ifdef WITH_SNMP @@ -409,11 +410,21 @@ private: unsigned long _last_wifi_check; wl_status_t _last_wifi_status; bool _wifi_status_initialized; - unsigned long _wifi_disconnected_time; // 0 when connected + // Packed OutageSnapshot; Core 0 (event + MQTT task) stores, Core 1 loads. + std::atomic _wifi_outage_bits; unsigned long _last_wifi_reconnect_attempt; uint8_t _wifi_reconnect_backoff_attempt; // 0..5 → 15s, 30s, 60s, 120s, 300s; reset on connect unsigned long _last_slot_reconnect_ms; // guards against concurrent TLS handshakes (15 s inter-slot gap) + AlertFaultPolicy::OutageSnapshot wifiOutage() const { + return AlertFaultPolicy::unpackOutageSnapshot( + _wifi_outage_bits.load(std::memory_order_acquire)); + } + void setWifiOutage(AlertFaultPolicy::OutageSnapshot snap) { + _wifi_outage_bits.store(AlertFaultPolicy::packOutageSnapshot(snap), + std::memory_order_release); + } + // Optional pointers for collecting stats internally (set by mesh if available) mesh::Dispatcher* _dispatcher; // For air times and errors mesh::Radio* _radio; // For noise floor @@ -643,6 +654,15 @@ public: static unsigned long getWifiConnectedAtMillis(); + /** + * Current WiFi outage snapshot for AlertReporter: down, started_ms, and the + * initiating disconnect reason. Distinct from getLastWifiDisconnectTime() / + * getLastWifiDisconnectReason(), which follow the most recent ESP-IDF + * DISCONNECTED event and are overwritten by STA-backoff WiFi.disconnect() + * (reason 8 / ASSOC_LEAVE). + */ + AlertFaultPolicy::OutageSnapshot getWifiOutageSnapshot() const { return wifiOutage(); } + /** * Per-slot outage accessors used by AlertReporter to detect prolonged * MQTT broker outages. Indices are 0..RUNTIME_MQTT_SLOTS-1. diff --git a/test/README.md b/test/README.md index 80182510..6fde2d7f 100644 --- a/test/README.md +++ b/test/README.md @@ -28,7 +28,8 @@ does not reflect the GoogleTest count — run the built binary directly | `test_webconfig_keys` | `src/helpers/WebConfigKeys.h` | POST-key allowlist, secret detection, admin-password classification/validation, slot-index bounds, and the short-key out-of-bounds guard (attacker-supplied keys) | | `test_topic_template` | `src/helpers/MQTTTopicTemplate.h` | `{iata}/{device}/{token}/{type}` expansion, overflow/NUL-termination, and a buffer-size fuzz | | `test_mqtt_topic_router` | `src/helpers/MQTTTopicRouter.h` | complete preset/custom topic-routing contract; MeshRank all types except raw; required identifiers; invalid inputs/slots; exact buffer boundaries | -| `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover | +| `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover; WiFi current-outage start sticky across STA reconnect attempts | +| `test_alert_fault_policy` | `src/helpers/AlertFaultPolicy.h` | WiFi/MQTT fault edge detector; `OutageSnapshot` (down / started_ms / initiating reason) fed to tick and `formatWifiAlert`; reason-8 reconnects change neither duration nor initiating reason; flap between status polls; down at `millis()==0`; packed 64-bit cross-task word; rate-limit floor and first-fire; 5 s poll cadence and `millis()` rollover | | `test_mqtt_packet_queue_policy` | `src/helpers/MQTTPacketQueuePolicy.h` | queue-full eviction; stale-disconnect flush; adaptive drain limits; bounded QoS0 retries; exact timing boundaries and 32-bit `millis()` rollover | | `test_mqtt_packet_filter` | `src/helpers/MQTTPacketFilter.h` | per-slot 0-15 allowlist parsing/formatting, numeric and named spellings; exact bounds; membership; candidate/eligible split and retry-completion policy; pre-queue union gate; default-mask detection | | `test_mqtt_runtime_buffer_lifecycle` | `src/helpers/MQTTRuntimeBufferLifecycle.h` | idempotent allocation/release; partial-allocation degradation; retry of only missing buffers | diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index f92e5b89..e5fef6d0 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -230,6 +230,21 @@ TEST(MQTTConnectionPolicy, WifiReconnectRequiresBothDownAndSinceAttemptToClearRu EXPECT_TRUE(Policy::wifiReconnectDue(1000U + 15000U, down_since, last_attempt, attempt)); } +TEST(MQTTConnectionPolicy, WifiOutageStartStickyAcrossReconnectAttempts) { + // First observe-down (or connected→down) records `now`. Further down samples + // — including STA backoff WiFi.disconnect() — must keep that start so + // AlertReporter quotes the outage, not the last attempt / boot event. + EXPECT_EQ(0U, Policy::wifiCurrentOutageStartMs(5000, true, false, 0, false)); + EXPECT_EQ(5000U, Policy::wifiCurrentOutageStartMs(5000, false, false, 0, false)); + + const uint32_t start = 9000U; + EXPECT_EQ(start, Policy::wifiCurrentOutageStartMs(start, false, true, 0, true)); + EXPECT_EQ(start, Policy::wifiCurrentOutageStartMs(start + 15000U, false, false, start, true)); + EXPECT_EQ(start, Policy::wifiCurrentOutageStartMs(start + 2U * 3600U * 1000U, + false, false, start, true)); + EXPECT_EQ(0U, Policy::wifiCurrentOutageStartMs(start + 1000U, true, false, start, true)); +} + TEST(MQTTConnectionPolicy, WifiReconnectDueSurvivesMillisRollover) { const uint32_t down_since = std::numeric_limits::max() - 100U; const uint32_t last_attempt = down_since; From 0a3b50bcf9e584a9c64e018c86c94587bd814a1e Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 20 Aug 2026 19:46:58 -0700 Subject: [PATCH 34/93] test(alert): add WiFi fault policy host tests --- src/helpers/AlertFaultPolicy.h | 277 +++++++++++++ .../test_alert_fault_policy.cpp | 389 ++++++++++++++++++ 2 files changed, 666 insertions(+) create mode 100644 src/helpers/AlertFaultPolicy.h create mode 100644 test/test_alert_fault_policy/test_alert_fault_policy.cpp diff --git a/src/helpers/AlertFaultPolicy.h b/src/helpers/AlertFaultPolicy.h new file mode 100644 index 00000000..d15aaf1f --- /dev/null +++ b/src/helpers/AlertFaultPolicy.h @@ -0,0 +1,277 @@ +#pragma once + +#include +#include +#include + +// Pure WiFi/MQTT fault-alert policy used by AlertReporter. Keeping the edge +// detector, duration math, rate-limit floor, and message formatting here lets +// host tests drive a fake clock without Mesh, MQTTBridge, or a radio. +// +// WiFi outage state is an OutageSnapshot: down, started_ms, and the initiating +// disconnect reason. That is the only data AlertReporter passes into tick() +// and formatWifiAlert(). Last ESP-IDF DISCONNECTED event time/reason are not +// inputs — using them made "WiFi down" quote uptime and reason 8 (ASSOC_LEAVE +// from STA-backoff WiFi.disconnect()). +namespace AlertFaultPolicy { + +static const uint32_t kCheckIntervalMs = 5000UL; +static const uint16_t kMinIntervalMinutes = 60; +static const uint32_t kMsPerMinute = 60000UL; + +enum class State : uint8_t { OK, FIRING }; + +struct Fault { + State state; + uint32_t fired_at_ms; // millis() of last successful "down" send; 0 = never + uint32_t last_outage_started_ms; // remembered so recovered can quote duration +}; + +// Current-outage view consumed by tick() and formatWifiAlert(). +// `down` is the authority: started_ms may be 0 when the STA dropped at +// millis()==0. While !down, packOutageSnapshot canonicalizes start/reason to 0. +struct OutageSnapshot { + bool down; + uint32_t started_ms; + uint8_t reason; // initiating reason; 0 = omit from the down message +}; + +enum class Action : uint8_t { None, FireDown, FireRecovered }; + +struct TickResult { + Action action; + uint32_t duration_ms; +}; + +static inline OutageSnapshot fromStartMs(uint32_t started_ms) { + OutageSnapshot snap{}; + snap.down = started_ms != 0; + snap.started_ms = started_ms; + return snap; +} + +// Single published word for the Core 0 writers (WiFi event / MQTT task) and +// the Core 1 reader (AlertReporter). 32-bit started_ms + 8-bit reason + down +// fit in 64 bits; atomic load/store of this word is the cross-task boundary. +static const uint64_t kOutageDownBit = 1ULL << 40; + +static inline uint64_t packOutageSnapshot(OutageSnapshot s) { + if (!s.down) { + s.started_ms = 0; + s.reason = 0; + } + uint64_t v = (uint64_t)s.started_ms; + v |= (uint64_t)s.reason << 32; + if (s.down) v |= kOutageDownBit; + return v; +} + +static inline OutageSnapshot unpackOutageSnapshot(uint64_t v) { + OutageSnapshot s{}; + s.started_ms = (uint32_t)v; + s.reason = (uint8_t)(v >> 32); + s.down = (v & kOutageDownBit) != 0; + if (!s.down) { + s.started_ms = 0; + s.reason = 0; + } + return s; +} + +// Status poll (handleWiFiConnection). `down` is independent of started_ms so a +// drop at millis()==0 is still down. Uses snapshot.down — not a separate +// last_connected flag — so a DISCONNECTED event that already opened the +// outage is not treated as a fresh connected→down edge on the next poll. +static inline OutageSnapshot applyWifiStatus(uint32_t now, bool connected, + OutageSnapshot cur, + bool initialized) { + if (connected) { + if (!initialized || cur.down) { + return OutageSnapshot{}; + } + return cur; + } + if (!initialized || !cur.down) { + OutageSnapshot snap{}; + snap.down = true; + snap.started_ms = now; // 0 is a legal start + snap.reason = cur.reason; + return snap; + } + return cur; +} + +// STA_GOT_IP (and any observe-connected between 10 s polls). A disconnect + +// reconnect that never shares a status poll with the drop must still close +// the outage, or the next DISCONNECTED event keeps the first start/reason. +static inline OutageSnapshot applyWifiGotIp(OutageSnapshot /*cur*/) { + return OutageSnapshot{}; +} + +// ESP-IDF STA_DISCONNECTED event. Opens the outage if needed. While already +// down, the start is frozen and the first non-zero reason is kept — later +// reason 8 (ASSOC_LEAVE from WiFi.disconnect()) cannot replace it. +static inline OutageSnapshot applyWifiDisconnectEvent(uint32_t now, uint8_t reason, + OutageSnapshot cur) { + if (!cur.down) { + OutageSnapshot snap{}; + snap.down = true; + snap.started_ms = now; + snap.reason = reason; + return snap; + } + if (cur.reason == 0 && reason != 0) { + cur.reason = reason; + } + return cur; +} + +// Unsigned subtraction is the standard millis() idiom and remains correct +// across one 32-bit counter rollover. +static inline uint32_t elapsedMs(uint32_t now, uint32_t then) { + return now - then; +} + +// Construction leaves next_check_ms at 0, so the first poll is always due. +// The signed delta matches AlertReporter's original (long)(now - next) < 0 +// skip, which is wrap-safe for intervals well under 2^31 ms. +static inline bool checkDue(uint32_t now, uint32_t next_check_ms) { + return (int32_t)(now - next_check_ms) >= 0; +} + +static inline uint32_t nextCheckMs(uint32_t now) { + return now + kCheckIntervalMs; +} + +// Stale prefs or a future field tweak cannot drag the floor below 1 hour. +static inline uint32_t minIntervalMs(uint16_t cfg_minutes) { + uint16_t minutes = cfg_minutes < kMinIntervalMinutes ? kMinIntervalMinutes + : cfg_minutes; + return (uint32_t)minutes * kMsPerMinute; +} + +static inline uint32_t thresholdMs(uint16_t minutes) { + return (uint32_t)minutes * kMsPerMinute; +} + +static inline uint32_t downDurationMs(uint32_t now, const OutageSnapshot& snap) { + return snap.down ? elapsedMs(now, snap.started_ms) : 0; +} + +static inline bool rateLimitAllows(uint32_t now, uint32_t fired_at_ms, + uint32_t min_interval_ms) { + // fired_at_ms == 0 means never fired since boot/config change. Treating it + // as a send at millis()==0 would suppress the first alert until uptime + // reached min_interval. + return fired_at_ms == 0 || elapsedMs(now, fired_at_ms) >= min_interval_ms; +} + +// Decide whether to emit a down/recovered message. Does not mutate `f`: +// FireDown is committed only after a successful send; FireRecovered is +// committed after the send attempt (success or not), matching production. +static inline TickResult tick(const Fault& f, uint32_t now, + const OutageSnapshot& snap, uint32_t thresh_ms, + uint32_t min_interval_ms) { + TickResult result = {Action::None, 0}; + if (f.state == State::OK) { + const uint32_t down_ms = downDurationMs(now, snap); + if (snap.down && down_ms >= thresh_ms && + rateLimitAllows(now, f.fired_at_ms, min_interval_ms)) { + result.action = Action::FireDown; + result.duration_ms = down_ms; + } + } else if (!snap.down) { + result.action = Action::FireRecovered; + // FIRING always went through commitDown, so last_outage_started_ms is a + // real start — including 0 when the outage began at millis()==0. + result.duration_ms = elapsedMs(now, f.last_outage_started_ms); + } + return result; +} + +static inline void commitDown(Fault& f, uint32_t now, uint32_t outage_start_ms) { + f.state = State::FIRING; + f.fired_at_ms = now; + f.last_outage_started_ms = outage_start_ms; +} + +static inline void commitRecovered(Fault& f) { + f.state = State::OK; +} + +static inline void reset(Fault& f) { + f.state = State::OK; + f.fired_at_ms = 0; +} + +static inline void rearmIfDisabled(Fault& f) { + if (f.state == State::FIRING) f.state = State::OK; +} + +static inline void formatAge(uint32_t age_ms, char* out, size_t out_size) { + if (!out || out_size == 0) return; + uint32_t secs = age_ms / 1000U; + uint32_t h = secs / 3600U; + uint32_t m = (secs % 3600U) / 60U; + if (h > 0) { + snprintf(out, out_size, "%uh%um", (unsigned)h, (unsigned)m); + } else { + snprintf(out, out_size, "%um", (unsigned)m); + } +} + +static inline void formatWifiDown(char* out, size_t out_size, uint32_t duration_ms, + uint8_t reason) { + if (!out || out_size == 0) return; + char age[16]; + formatAge(duration_ms, age, sizeof(age)); + if (reason != 0) { + snprintf(out, out_size, "WiFi down %s (reason %u)", age, (unsigned)reason); + } else { + snprintf(out, out_size, "WiFi down %s", age); + } +} + +static inline void formatWifiRecovered(char* out, size_t out_size, + uint32_t duration_ms) { + if (!out || out_size == 0) return; + char age[16]; + formatAge(duration_ms, age, sizeof(age)); + snprintf(out, out_size, "WiFi recovered after %s", age); +} + +// Production formatting entry: the same (TickResult, OutageSnapshot) pair +// AlertReporter feeds after tick(). Returns false when there is no message. +static inline bool formatWifiAlert(char* out, size_t out_size, const TickResult& r, + const OutageSnapshot& snap) { + if (r.action == Action::FireDown) { + formatWifiDown(out, out_size, r.duration_ms, snap.reason); + return true; + } + if (r.action == Action::FireRecovered) { + formatWifiRecovered(out, out_size, r.duration_ms); + return true; + } + return false; +} + +static inline void formatMqttDown(char* out, size_t out_size, int slot_1based, + const char* preset_name, uint32_t duration_ms) { + if (!out || out_size == 0) return; + char age[16]; + formatAge(duration_ms, age, sizeof(age)); + snprintf(out, out_size, "MQTT slot %d (%s) down %s", slot_1based, + preset_name ? preset_name : "?", age); +} + +static inline void formatMqttRecovered(char* out, size_t out_size, int slot_1based, + const char* preset_name, + uint32_t duration_ms) { + if (!out || out_size == 0) return; + char age[16]; + formatAge(duration_ms, age, sizeof(age)); + snprintf(out, out_size, "MQTT slot %d (%s) recovered after %s", slot_1based, + preset_name ? preset_name : "?", age); +} + +} // namespace AlertFaultPolicy diff --git a/test/test_alert_fault_policy/test_alert_fault_policy.cpp b/test/test_alert_fault_policy/test_alert_fault_policy.cpp new file mode 100644 index 00000000..42ba27d7 --- /dev/null +++ b/test/test_alert_fault_policy/test_alert_fault_policy.cpp @@ -0,0 +1,389 @@ +// Host tests for AlertReporter's fault edge detector, duration math, and +// message formatting (src/helpers/AlertFaultPolicy.h). Tick and formatWifiAlert +// share an OutageSnapshot (down, started_ms, initiating reason) — the same +// data MQTTBridge feeds production. +#include +#include +#include +#include + +#include "helpers/AlertFaultPolicy.h" + +namespace Alert = AlertFaultPolicy; + +namespace { + +const uint32_t kWifiThresh = Alert::thresholdMs(30); // default alert.wifi +const uint32_t kMinInterval = Alert::minIntervalMs(60); +const uint8_t kBeaconTimeout = 200; // WIFI_REASON_BEACON_TIMEOUT +const uint8_t kAssocLeave = 8; // WIFI_REASON_ASSOC_LEAVE (WiFi.disconnect()) + +Alert::Fault OkFault() { + Alert::Fault f{}; + f.state = Alert::State::OK; + return f; +} + +Alert::OutageSnapshot Down(uint32_t started_ms, uint8_t reason = 0) { + Alert::OutageSnapshot snap{}; + snap.down = true; + snap.started_ms = started_ms; + snap.reason = reason; + return snap; +} + +Alert::OutageSnapshot Up() { return {}; } + +// Same sequence AlertReporter uses on the WiFi path: tick, format from the +// snapshot, commit on FireDown / FireRecovered. +bool ProductionWifiAlert(Alert::Fault& f, uint32_t now, + const Alert::OutageSnapshot& snap, char* text, + size_t text_size) { + Alert::TickResult r = Alert::tick(f, now, snap, kWifiThresh, kMinInterval); + if (!Alert::formatWifiAlert(text, text_size, r, snap)) return false; + if (r.action == Alert::Action::FireDown) { + Alert::commitDown(f, now, snap.started_ms); + } else { + Alert::commitRecovered(f); + } + return true; +} + +} // namespace + +TEST(AlertFaultPolicy, DownDurationUsesCurrentOutageStartNotLastEvent) { + Alert::Fault f = OkFault(); + const uint32_t boot_event = 1; + const uint32_t outage_start = 10 * 60000; + const uint32_t now = outage_start + kWifiThresh; + const uint32_t last_reconnect_event = now - 5000; + + Alert::TickResult r = + Alert::tick(f, now, Down(outage_start, kBeaconTimeout), kWifiThresh, + kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); + EXPECT_EQ(kWifiThresh, r.duration_ms); + EXPECT_NE(now - boot_event, r.duration_ms); + EXPECT_NE(now - last_reconnect_event, r.duration_ms); +} + +TEST(AlertFaultPolicy, + ProductionFlowReason8ReconnectsPreserveDurationAndInitiatingReason) { + Alert::OutageSnapshot snap{}; + snap = Alert::applyWifiStatus(1000, true, snap, false); + EXPECT_FALSE(snap.down); + EXPECT_EQ(0U, snap.started_ms); + EXPECT_EQ(0, snap.reason); + + const uint32_t drop = 10 * 60000U; + snap = Alert::applyWifiDisconnectEvent(drop, kBeaconTimeout, snap); + snap = Alert::applyWifiStatus(drop + 100, false, snap, true); + EXPECT_TRUE(snap.down); + EXPECT_EQ(drop, snap.started_ms); + EXPECT_EQ(kBeaconTimeout, snap.reason); + + for (int i = 1; i <= 24; ++i) { + const uint32_t t = drop + (uint32_t)i * 300000U; + snap = Alert::applyWifiDisconnectEvent(t, kAssocLeave, snap); + snap = Alert::applyWifiStatus(t + 1, false, snap, true); + } + EXPECT_EQ(drop, snap.started_ms); + EXPECT_EQ(kBeaconTimeout, snap.reason); + + const uint32_t two_h_two_m = (2U * 3600U + 2U * 60U) * 1000U; + const uint32_t now = drop + two_h_two_m; + Alert::Fault f = OkFault(); + char text[80]; + ASSERT_TRUE(ProductionWifiAlert(f, now, snap, text, sizeof(text))); + EXPECT_STREQ("WiFi down 2h2m (reason 200)", text); + EXPECT_EQ(Alert::State::FIRING, f.state); + EXPECT_EQ(drop, f.last_outage_started_ms); +} + +TEST(AlertFaultPolicy, PollFirstEventFillsReasonThenReason8DoesNotOverwrite) { + Alert::OutageSnapshot snap{}; + snap = Alert::applyWifiStatus(1000, true, snap, false); + const uint32_t drop = 5000; + snap = Alert::applyWifiStatus(drop, false, snap, true); + EXPECT_TRUE(snap.down); + EXPECT_EQ(drop, snap.started_ms); + EXPECT_EQ(0, snap.reason); + + snap = Alert::applyWifiDisconnectEvent(drop + 20, kBeaconTimeout, snap); + EXPECT_EQ(kBeaconTimeout, snap.reason); + EXPECT_EQ(drop, snap.started_ms); + + snap = Alert::applyWifiDisconnectEvent(drop + 15000, kAssocLeave, snap); + EXPECT_EQ(kBeaconTimeout, snap.reason); + EXPECT_EQ(drop, snap.started_ms); +} + +TEST(AlertFaultPolicy, RecoverClearsSnapshotReasonAndStart) { + Alert::OutageSnapshot snap = Down(5000, kBeaconTimeout); + snap = Alert::applyWifiStatus(8000, true, snap, true); + EXPECT_FALSE(snap.down); + EXPECT_EQ(0U, snap.started_ms); + EXPECT_EQ(0, snap.reason); +} + +TEST(AlertFaultPolicy, DisconnectAndReconnectBetweenStatusPollsIsANewOutage) { + Alert::OutageSnapshot snap{}; + snap = Alert::applyWifiStatus(1000, true, snap, false); + + snap = Alert::applyWifiDisconnectEvent(2000, kBeaconTimeout, snap); + EXPECT_TRUE(snap.down); + EXPECT_EQ(2000U, snap.started_ms); + EXPECT_EQ(kBeaconTimeout, snap.reason); + + // GOT_IP with no status poll in between — the flap is fully between polls. + snap = Alert::applyWifiGotIp(snap); + EXPECT_FALSE(snap.down); + EXPECT_EQ(0U, snap.started_ms); + EXPECT_EQ(0, snap.reason); + + const uint8_t kAssocExpire = 4; + snap = Alert::applyWifiDisconnectEvent(3500, kAssocExpire, snap); + EXPECT_TRUE(snap.down); + EXPECT_EQ(3500U, snap.started_ms); + EXPECT_EQ(kAssocExpire, snap.reason); + EXPECT_NE(kBeaconTimeout, snap.reason); +} + +TEST(AlertFaultPolicy, StatusDetectedDownAtMillisZeroIsStillDown) { + Alert::OutageSnapshot snap{}; + snap = Alert::applyWifiStatus(0, false, snap, false); + EXPECT_TRUE(snap.down); + EXPECT_EQ(0U, snap.started_ms); + + Alert::Fault f = OkFault(); + Alert::TickResult r = + Alert::tick(f, kWifiThresh, snap, kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); + EXPECT_EQ(kWifiThresh, r.duration_ms); +} + +TEST(AlertFaultPolicy, RecoveryAfterDownCommittedAtMillisZeroQuotesElapsedTime) { + Alert::Fault f = OkFault(); + Alert::OutageSnapshot snap = Down(0, kBeaconTimeout); + const uint32_t down_at = kWifiThresh; + Alert::TickResult down = + Alert::tick(f, down_at, snap, kWifiThresh, kMinInterval); + ASSERT_EQ(Alert::Action::FireDown, down.action); + Alert::commitDown(f, down_at, snap.started_ms); + EXPECT_EQ(0U, f.last_outage_started_ms); + + const uint32_t recovered_at = (2U * 3600U + 5U * 60U) * 1000U; + Alert::TickResult rec = + Alert::tick(f, recovered_at, Up(), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireRecovered, rec.action); + EXPECT_EQ(recovered_at, rec.duration_ms); + + char text[80]; + ASSERT_TRUE(Alert::formatWifiAlert(text, sizeof(text), rec, Up())); + EXPECT_STREQ("WiFi recovered after 2h5m", text); +} + +TEST(AlertFaultPolicy, PackedSnapshotIsTheCoherentCrossTaskWord) { + Alert::OutageSnapshot down_at_zero = Down(0, kBeaconTimeout); + std::atomic cell{0}; + cell.store(Alert::packOutageSnapshot(down_at_zero), std::memory_order_release); + Alert::OutageSnapshot loaded = Alert::unpackOutageSnapshot( + cell.load(std::memory_order_acquire)); + EXPECT_TRUE(loaded.down); + EXPECT_EQ(0U, loaded.started_ms); + EXPECT_EQ(kBeaconTimeout, loaded.reason); + + cell.store(Alert::packOutageSnapshot(Up()), std::memory_order_release); + loaded = Alert::unpackOutageSnapshot(cell.load(std::memory_order_acquire)); + EXPECT_FALSE(loaded.down); + EXPECT_EQ(0U, loaded.started_ms); + EXPECT_EQ(0, loaded.reason); + + Alert::OutageSnapshot dirty_up{}; + dirty_up.down = false; + dirty_up.started_ms = 12345; + dirty_up.reason = kAssocLeave; + loaded = Alert::unpackOutageSnapshot(Alert::packOutageSnapshot(dirty_up)); + EXPECT_FALSE(loaded.down); + EXPECT_EQ(0U, loaded.started_ms); + EXPECT_EQ(0, loaded.reason); +} + +TEST(AlertFaultPolicy, DoesNotFireBelowThreshold) { + Alert::Fault f = OkFault(); + Alert::TickResult r = + Alert::tick(f, 1000 + kWifiThresh - 1, Down(1000), kWifiThresh, + kMinInterval); + EXPECT_EQ(Alert::Action::None, r.action); +} + +TEST(AlertFaultPolicy, FiresAtExactThreshold) { + Alert::Fault f = OkFault(); + Alert::TickResult r = + Alert::tick(f, 1000 + kWifiThresh, Down(1000), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); + EXPECT_EQ(kWifiThresh, r.duration_ms); +} + +TEST(AlertFaultPolicy, FirstFireIgnoresMinIntervalUptime) { + Alert::Fault f = OkFault(); + EXPECT_EQ(0U, f.fired_at_ms); + const uint32_t start = 1000; + const uint32_t now = start + kWifiThresh; + EXPECT_LT(now, kMinInterval); + Alert::TickResult r = + Alert::tick(f, now, Down(start), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); +} + +TEST(AlertFaultPolicy, RateLimitBlocksRepeatUntilFloorElapses) { + Alert::Fault f = OkFault(); + const uint32_t start = 1000; + const uint32_t first = start + kWifiThresh; + Alert::commitDown(f, first, start); + + Alert::commitRecovered(f); + Alert::TickResult r = Alert::tick(f, first + kMinInterval - 1, Down(start), + kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::None, r.action); + + Alert::TickResult due = Alert::tick(f, first + kMinInterval, Down(start), + kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, due.action); +} + +TEST(AlertFaultPolicy, MinIntervalClampsBelowOneHour) { + EXPECT_EQ(60U * 60000U, Alert::minIntervalMs(0)); + EXPECT_EQ(60U * 60000U, Alert::minIntervalMs(30)); + EXPECT_EQ(60U * 60000U, Alert::minIntervalMs(59)); + EXPECT_EQ(60U * 60000U, Alert::minIntervalMs(60)); + EXPECT_EQ(120U * 60000U, Alert::minIntervalMs(120)); +} + +TEST(AlertFaultPolicy, RecoveredDurationUsesRememberedOutageStart) { + Alert::Fault f = OkFault(); + const uint32_t start = 10 * 60000; + const uint32_t down_at = start + kWifiThresh; + Alert::commitDown(f, down_at, start); + + const uint32_t recovered_at = start + (2U * 3600U + 5U * 60U) * 1000U; + Alert::TickResult r = + Alert::tick(f, recovered_at, Up(), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireRecovered, r.action); + EXPECT_EQ(recovered_at - start, r.duration_ms); + + char text[80]; + ASSERT_TRUE(Alert::formatWifiAlert(text, sizeof(text), r, Up())); + EXPECT_STREQ("WiFi recovered after 2h5m", text); +} + +TEST(AlertFaultPolicy, SecondOutageUsesNewStartNotTheFirst) { + Alert::Fault f = OkFault(); + const uint32_t first_start = 1000; + Alert::commitDown(f, first_start + kWifiThresh, first_start); + Alert::TickResult recovered = Alert::tick( + f, first_start + kWifiThresh + 1000, Up(), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireRecovered, recovered.action); + Alert::commitRecovered(f); + + const uint32_t second_start = first_start + kWifiThresh + 5000; + const uint32_t second_now = second_start + kWifiThresh + kMinInterval; + Alert::TickResult r = Alert::tick(f, second_now, Down(second_start), + kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); + EXPECT_EQ(second_now - second_start, r.duration_ms); + EXPECT_NE(second_now - first_start, r.duration_ms); +} + +TEST(AlertFaultPolicy, TickDoesNotMutateUntilCommit) { + Alert::Fault f = OkFault(); + const uint32_t start = 1000; + Alert::tick(f, start + kWifiThresh, Down(start), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::State::OK, f.state); + EXPECT_EQ(0U, f.fired_at_ms); + + Alert::commitDown(f, start + kWifiThresh, start); + Alert::TickResult recovered = + Alert::tick(f, start + kWifiThresh + 1, Up(), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireRecovered, recovered.action); + EXPECT_EQ(Alert::State::FIRING, f.state); + Alert::commitRecovered(f); + EXPECT_EQ(Alert::State::OK, f.state); +} + +TEST(AlertFaultPolicy, RearmClearsFiringWhenThresholdDisabled) { + Alert::Fault f = OkFault(); + Alert::commitDown(f, 1000, 1); + Alert::rearmIfDisabled(f); + EXPECT_EQ(Alert::State::OK, f.state); +} + +TEST(AlertFaultPolicy, ResetRearmsWithoutForcingASend) { + Alert::Fault f = OkFault(); + Alert::commitDown(f, 1000, 1); + Alert::reset(f); + EXPECT_EQ(Alert::State::OK, f.state); + EXPECT_EQ(0U, f.fired_at_ms); +} + +TEST(AlertFaultPolicy, FormatAgeMinutesAndHours) { + char buf[16]; + Alert::formatAge(0, buf, sizeof(buf)); + EXPECT_STREQ("0m", buf); + Alert::formatAge(47U * 60000U, buf, sizeof(buf)); + EXPECT_STREQ("47m", buf); + Alert::formatAge((1U * 3600U + 3U * 60U) * 1000U, buf, sizeof(buf)); + EXPECT_STREQ("1h3m", buf); +} + +TEST(AlertFaultPolicy, FormatWifiAlertUsesSnapshotReason) { + Alert::Fault f = OkFault(); + Alert::TickResult r = Alert::tick(f, 1000 + kWifiThresh, + Down(1000, kBeaconTimeout), kWifiThresh, + kMinInterval); + char text[80]; + ASSERT_TRUE(Alert::formatWifiAlert(text, sizeof(text), r, + Down(1000, kBeaconTimeout))); + EXPECT_STREQ("WiFi down 30m (reason 200)", text); + + Alert::formatWifiDown(text, sizeof(text), 47U * 60000U, 0); + EXPECT_STREQ("WiFi down 47m", text); +} + +TEST(AlertFaultPolicy, FormatMqttSlotMessages) { + char text[100]; + Alert::formatMqttDown(text, sizeof(text), 1, "analyzer-us", 30U * 60000U); + EXPECT_STREQ("MQTT slot 1 (analyzer-us) down 30m", text); + Alert::formatMqttRecovered(text, sizeof(text), 1, "analyzer-us", + 4U * 3600U * 1000U + 45U * 60000U); + EXPECT_STREQ("MQTT slot 1 (analyzer-us) recovered after 4h45m", text); +} + +TEST(AlertFaultPolicy, CheckDueMatchesFiveSecondCadenceAndWrap) { + EXPECT_TRUE(Alert::checkDue(0, 0)); + EXPECT_TRUE(Alert::checkDue(1000, 0)); + const uint32_t next = Alert::nextCheckMs(1000); + EXPECT_EQ(6000U, next); + EXPECT_FALSE(Alert::checkDue(5999, next)); + EXPECT_TRUE(Alert::checkDue(6000, next)); + + const uint32_t before_wrap = std::numeric_limits::max() - 1000U; + const uint32_t wrapped_next = Alert::nextCheckMs(before_wrap); + EXPECT_FALSE(Alert::checkDue(before_wrap + 4999U, wrapped_next)); + EXPECT_TRUE(Alert::checkDue(before_wrap + 5000U, wrapped_next)); +} + +TEST(AlertFaultPolicy, DownDurationSurvivesMillisRollover) { + Alert::Fault f = OkFault(); + const uint32_t start = std::numeric_limits::max() - 1000U; + const uint32_t now = start + kWifiThresh; + Alert::TickResult r = + Alert::tick(f, now, Down(start), kWifiThresh, kMinInterval); + EXPECT_EQ(Alert::Action::FireDown, r.action); + EXPECT_EQ(kWifiThresh, r.duration_ms); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 2358b750aec4145e46d46ae23ac5252067792761 Mon Sep 17 00:00:00 2001 From: Adam Gessaman Date: Thu, 20 Aug 2026 20:49:42 -0700 Subject: [PATCH 35/93] feat(mqtt): add Heltec V4 R8 configurations for MQTT observer and room server --- CHANGELOG.md | 4 + variants/heltec_v4_r8/platformio.ini | 168 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 993a13a8..816ae1e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ existing behavior · **Internal** = refactor / under-the-hood · **Docs** = docu **Build** / **CI** = build system & automation. **⬆ Upstream sync** marks a merge of the upstream MeshCore `dev` branch, which generally pulls in a new MeshCore software version. +### August 2026 + +- **New** · `platformio` — MQTT observer repeater and room-server configs for Heltec V4 R8 (OLED and TFT) 2026-08-20 + ### June 2026 - **New** — AlertReporter integration in MyMesh for Room Server 2026-06-17 · `985fda13` diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index f523ebf9..6f8d5a7f 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -101,6 +101,48 @@ lib_deps = ${esp32_ota.lib_deps} bakercp/CRC32 @ ^2.0.0 +[env:heltec_v4_r8_repeater_observer_mqtt] +extends = heltec_v4_r8_oled +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${heltec_v4_r8_oled.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; Keep default observer profile less verbose to reduce runtime contention. +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${heltec_v4_r8_oled.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${heltec_v4_r8_oled.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + [env:heltec_v4_r8_room_server] extends = heltec_v4_r8_oled build_flags = @@ -118,6 +160,48 @@ lib_deps = ${heltec_v4_r8_oled.lib_deps} ${esp32_ota.lib_deps} +[env:heltec_v4_r8_room_server_observer_mqtt] +extends = heltec_v4_r8_oled +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${heltec_v4_r8_oled.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"Heltec R8 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D WITH_MQTT_BRIDGE=1 + -D MAX_NEIGHBOURS=50 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${heltec_v4_r8_oled.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${heltec_v4_r8_oled.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + [env:heltec_v4_r8_terminal_chat] extends = heltec_v4_r8_oled build_flags = @@ -226,6 +310,48 @@ lib_deps = ${esp32_ota.lib_deps} bakercp/CRC32 @ ^2.0.0 +[env:heltec_v4_r8_tft_repeater_observer_mqtt] +extends = heltec_v4_r8_tft +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${heltec_v4_r8_tft.build_flags} + -D DISPLAY_CLASS=ST7789LCDDisplay + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; Keep default observer profile less verbose to reduce runtime contention. +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${heltec_v4_r8_tft.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + [env:heltec_v4_r8_tft_room_server] extends = heltec_v4_r8_tft build_flags = @@ -243,6 +369,48 @@ lib_deps = ${heltec_v4_r8_tft.lib_deps} ${esp32_ota.lib_deps} +[env:heltec_v4_r8_tft_room_server_observer_mqtt] +extends = heltec_v4_r8_tft +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${heltec_v4_r8_tft.build_flags} + -D DISPLAY_CLASS=ST7789LCDDisplay + -D ADVERT_NAME='"Heltec R8 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D WITH_MQTT_BRIDGE=1 + -D MAX_NEIGHBOURS=50 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${heltec_v4_r8_tft.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + [env:heltec_v4_r8_tft_terminal_chat] extends = heltec_v4_r8_tft build_flags = From 8fc0733299d79bb9ae327d1730327cfdee0e76b6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 20 Aug 2026 21:13:58 -0700 Subject: [PATCH 36/93] fix(examples): add MQTT teardown delay before OTA allocation --- examples/simple_repeater/MyMesh.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 22645503..859076e9 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -69,6 +69,11 @@ // channel so an update is never stalled indefinitely. #define OTA_TX_DRAIN_TIMEOUT_MS 5000 +// Bench mitigation for a ThinkNode M7 cache-error race between a clean MQTT +// teardown and the OTA worker allocation. 25 ms still failed intermittently; +// 100 ms completed five consecutive one-slot OTA cycles without a crash. +#define OTA_MQTT_STOP_SETTLE_MS 100 + #define LAZY_CONTACTS_WRITE_DELAY 5000 void MyMesh::putNeighbour(const mesh::Identity &id, uint32_t timestamp, float snr) { @@ -1731,6 +1736,9 @@ void MyMesh::loop() { // duty-limited channel) is lost when the flash spins the loop and reboots. drainOutbound(OTA_TX_DRAIN_TIMEOUT_MS); setBridgeState(false); + // TODO: Replace this timed settle with a proven MQTT task/client/callback + // quiescence barrier once the teardown race's root cause is identified. + delay(OTA_MQTT_STOP_SETTLE_MS); char ota_reply[160]; // OTA teardown barrier (Phase 5): only flash after a CLEAN MQTT shutdown. // A timed-out/forced stop leaves mbedTLS/heap ownership uncertain — writing From 0cc6fdfea69738c2696b3d1abfb0de77110a4919 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 20 Aug 2026 21:48:29 -0700 Subject: [PATCH 37/93] Add Heltec V4 R8 FEM LNA controls --- variants/heltec_v4_r8/HeltecV4R8Board.cpp | 18 ++++++++++++++++++ variants/heltec_v4_r8/HeltecV4R8Board.h | 3 +++ variants/heltec_v4_r8/LoRaFEMControl.cpp | 1 + variants/heltec_v4_r8/LoRaFEMControl.h | 6 ++++-- variants/heltec_v4_r8/platformio.ini | 6 +++--- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.cpp b/variants/heltec_v4_r8/HeltecV4R8Board.cpp index 1fb123b2..11aa9f64 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.cpp +++ b/variants/heltec_v4_r8/HeltecV4R8Board.cpp @@ -84,3 +84,21 @@ const char* HeltecV4R8Board::getManufacturerName() const { return "Heltec V4 R8 OLED"; #endif } + +bool HeltecV4R8Board::setLoRaFemLnaEnabled(bool enable) { + if (!loRaFEMControl.isLnaCanControl()) { + return false; + } + + loRaFEMControl.setLNAEnable(enable); + loRaFEMControl.setRxModeEnable(); + return true; +} + +bool HeltecV4R8Board::canControlLoRaFemLna() const { + return loRaFEMControl.isLnaCanControl(); +} + +bool HeltecV4R8Board::isLoRaFemLnaEnabled() const { + return loRaFEMControl.isLNAEnabled(); +} diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.h b/variants/heltec_v4_r8/HeltecV4R8Board.h index 20811abb..d8617b5b 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.h +++ b/variants/heltec_v4_r8/HeltecV4R8Board.h @@ -25,6 +25,9 @@ public: void onAfterTransmit(void) override; void enterDeepSleep(uint32_t secs, int pin_wake_btn = -1); void powerOff() override; + bool setLoRaFemLnaEnabled(bool enable) override; + bool canControlLoRaFemLna() const override; + bool isLoRaFemLnaEnabled() const override; uint16_t getBattMilliVolts() override; bool setAdcMultiplier(float multiplier) override { if (multiplier == 0.0f) { diff --git a/variants/heltec_v4_r8/LoRaFEMControl.cpp b/variants/heltec_v4_r8/LoRaFEMControl.cpp index bb530de3..2d0feba6 100644 --- a/variants/heltec_v4_r8/LoRaFEMControl.cpp +++ b/variants/heltec_v4_r8/LoRaFEMControl.cpp @@ -21,6 +21,7 @@ void LoRaFEMControl::init(void) { digitalWrite(P_LORA_KCT8103L_PA_CSD, HIGH); pinMode(P_LORA_KCT8103L_PA_CTX, OUTPUT); digitalWrite(P_LORA_KCT8103L_PA_CTX, lna_enabled ? LOW : HIGH); + setLnaCanControl(true); } void LoRaFEMControl::setSleepModeEnable(void) { diff --git a/variants/heltec_v4_r8/LoRaFEMControl.h b/variants/heltec_v4_r8/LoRaFEMControl.h index 961cfd07..b55e1553 100644 --- a/variants/heltec_v4_r8/LoRaFEMControl.h +++ b/variants/heltec_v4_r8/LoRaFEMControl.h @@ -15,10 +15,12 @@ public: void setRxModeEnable(void); void setRxModeEnableWhenMCUSleep(void); void setLNAEnable(bool enabled); - bool isLnaCanControl(void) { return true; } - void setLnaCanControl(bool can_control) { } + bool isLnaCanControl(void) const { return lna_can_control; } + void setLnaCanControl(bool can_control) { lna_can_control = can_control; } + bool isLNAEnabled(void) const { return lna_enabled; } LoRaFEMType getFEMType(void) const { return KCT8103L_PA; } private: bool lna_enabled = false; + bool lna_can_control = false; }; diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index 6f8d5a7f..70e5f702 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -17,9 +17,9 @@ build_flags = -D P_LORA_SCLK=9 -D P_LORA_MISO=11 -D P_LORA_MOSI=10 - -D P_LORA_PA_POWER=7 - -D P_LORA_KCT8103L_PA_CSD=2 - -D P_LORA_KCT8103L_PA_CTX=5 + -D P_LORA_PA_POWER=7 ; VFEM_Ctrl - LDO power enable + -D P_LORA_KCT8103L_PA_CSD=2 ; FEM_EN / CSD (HIGH=on) + -D P_LORA_KCT8103L_PA_CTX=5 ; PA_CTX (LOW=RX LNA, HIGH=RX bypass) -D P_LORA_TX_LED=46 -D PIN_USER_BTN=0 -D PIN_VEXT_EN=40 From c5d987df7055660e4af465e59e89c26c5a7d9762 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 23 Aug 2026 15:42:52 -0700 Subject: [PATCH 38/93] fix(boards): reset Heltec V4 R8 TFT on GPIO 21 GPIO 21 is the Expansion Kit V2 panel reset, not touch reset. Leaving RST unwired left the ST7789 blank after a bogus touch pulse. --- src/helpers/ui/ST7789LCDDisplay.cpp | 24 +++++++++++++++-------- variants/heltec_v4_r8/HeltecV4R8Board.cpp | 2 ++ variants/heltec_v4_r8/platformio.ini | 3 +-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index a9f30dd5..6e0862c3 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -4,6 +4,10 @@ #define PIN_TFT_MISO -1 #endif +#ifndef PIN_TFT_LEDA_CTL_ACTIVE + #define PIN_TFT_LEDA_CTL_ACTIVE HIGH +#endif + #ifndef DISPLAY_ROTATION #define DISPLAY_ROTATION 3 #endif @@ -36,11 +40,14 @@ ColorVal UIColor::corp_blue = 0x001A; bool ST7789LCDDisplay::begin() { if (!_isOn) { - if (_peripher_power) _peripher_power->claim(); + if (_peripher_power) { + _peripher_power->claim(); + delay(100); + } if (PIN_TFT_LEDA_CTL != -1) { pinMode(PIN_TFT_LEDA_CTL, OUTPUT); - digitalWrite(PIN_TFT_LEDA_CTL, HIGH); + digitalWrite(PIN_TFT_LEDA_CTL, !PIN_TFT_LEDA_CTL_ACTIVE); } // Im not sure if this is just a t-deck problem or not, if your display is slow try this. @@ -55,9 +62,13 @@ bool ST7789LCDDisplay::begin() { display.fillScreen(ST77XX_BLACK); display.setTextColor(ST77XX_WHITE); - display.setTextSize(2 * DISPLAY_SCALE_X); + display.setTextSize(2 * DISPLAY_SCALE_X); display.cp437(true); // Use full 256 char 'Code Page 437' font - + + if (PIN_TFT_LEDA_CTL != -1) { + digitalWrite(PIN_TFT_LEDA_CTL, PIN_TFT_LEDA_CTL_ACTIVE); + } + _isOn = true; } @@ -71,14 +82,11 @@ void ST7789LCDDisplay::turnOn() { void ST7789LCDDisplay::turnOff() { if (_isOn) { if (PIN_TFT_LEDA_CTL != -1) { - digitalWrite(PIN_TFT_LEDA_CTL, HIGH); + digitalWrite(PIN_TFT_LEDA_CTL, !PIN_TFT_LEDA_CTL_ACTIVE); } if (PIN_TFT_RST != -1) { digitalWrite(PIN_TFT_RST, LOW); } - if (PIN_TFT_LEDA_CTL != -1) { - digitalWrite(PIN_TFT_LEDA_CTL, LOW); - } _isOn = false; if (_peripher_power) _peripher_power->release(); diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.cpp b/variants/heltec_v4_r8/HeltecV4R8Board.cpp index 11aa9f64..580662b3 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.cpp +++ b/variants/heltec_v4_r8/HeltecV4R8Board.cpp @@ -8,6 +8,8 @@ void HeltecV4R8Board::begin() { loRaFEMControl.init(); + // Expansion Kit CHSC6X touch RST/INT are unwired. GPIO 21 is TFT RST + // (PIN_TFT_RST), owned by ST7789LCDDisplay. Do not pulse it here. #ifdef PIN_TOUCH_RST pinMode(PIN_TOUCH_RST, OUTPUT); digitalWrite(PIN_TOUCH_RST, HIGH); diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index 70e5f702..246eb785 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -65,7 +65,7 @@ build_flags = -D PIN_BOARD_SCL=18 -D DISPLAY_SCALE_X=2.5 -D DISPLAY_SCALE_Y=3.75 - -D PIN_TFT_RST=-1 + -D PIN_TFT_RST=21 -D PIN_TFT_VDD_CTL=-1 -D PIN_TFT_LEDA_CTL=44 -D PIN_TFT_LEDA_CTL_ACTIVE=HIGH @@ -75,7 +75,6 @@ build_flags = -D PIN_TFT_SDA=15 -D PIN_TFT_MISO=45 -D PIN_BUZZER=4 - -D PIN_TOUCH_RST=21 build_src_filter = ${Heltec_v4_r8.build_src_filter} + lib_deps = From 4a58d8076c806ad64766281e9d3023cdbd80e0ab Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 23 Aug 2026 16:33:41 -0700 Subject: [PATCH 39/93] fix(boards): initialize Heltec V4 R8 TFT --- src/helpers/ui/ST7789LCDDisplay.cpp | 21 +++++++++++++++++---- variants/heltec_v4_r8/HeltecV4R8Board.cpp | 12 ++++++++++-- variants/heltec_v4_r8/platformio.ini | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index 6e0862c3..137dfa15 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -4,10 +4,6 @@ #define PIN_TFT_MISO -1 #endif -#ifndef PIN_TFT_LEDA_CTL_ACTIVE - #define PIN_TFT_LEDA_CTL_ACTIVE HIGH -#endif - #ifndef DISPLAY_ROTATION #define DISPLAY_ROTATION 3 #endif @@ -42,12 +38,18 @@ bool ST7789LCDDisplay::begin() { if (!_isOn) { if (_peripher_power) { _peripher_power->claim(); + #ifdef HELTEC_V4_R8_TFT delay(100); + #endif } if (PIN_TFT_LEDA_CTL != -1) { pinMode(PIN_TFT_LEDA_CTL, OUTPUT); + #ifdef HELTEC_V4_R8_TFT digitalWrite(PIN_TFT_LEDA_CTL, !PIN_TFT_LEDA_CTL_ACTIVE); + #else + digitalWrite(PIN_TFT_LEDA_CTL, HIGH); + #endif } // Im not sure if this is just a t-deck problem or not, if your display is slow try this. @@ -65,9 +67,11 @@ bool ST7789LCDDisplay::begin() { display.setTextSize(2 * DISPLAY_SCALE_X); display.cp437(true); // Use full 256 char 'Code Page 437' font + #ifdef HELTEC_V4_R8_TFT if (PIN_TFT_LEDA_CTL != -1) { digitalWrite(PIN_TFT_LEDA_CTL, PIN_TFT_LEDA_CTL_ACTIVE); } + #endif _isOn = true; } @@ -82,11 +86,20 @@ void ST7789LCDDisplay::turnOn() { void ST7789LCDDisplay::turnOff() { if (_isOn) { if (PIN_TFT_LEDA_CTL != -1) { + #ifdef HELTEC_V4_R8_TFT digitalWrite(PIN_TFT_LEDA_CTL, !PIN_TFT_LEDA_CTL_ACTIVE); + #else + digitalWrite(PIN_TFT_LEDA_CTL, HIGH); + #endif } if (PIN_TFT_RST != -1) { digitalWrite(PIN_TFT_RST, LOW); } + #ifndef HELTEC_V4_R8_TFT + if (PIN_TFT_LEDA_CTL != -1) { + digitalWrite(PIN_TFT_LEDA_CTL, LOW); + } + #endif _isOn = false; if (_peripher_power) _peripher_power->release(); diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.cpp b/variants/heltec_v4_r8/HeltecV4R8Board.cpp index 580662b3..34d6b992 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.cpp +++ b/variants/heltec_v4_r8/HeltecV4R8Board.cpp @@ -1,5 +1,9 @@ #include "HeltecV4R8Board.h" +#if defined(HELTEC_V4_R8_TFT) && defined(DISPLAY_CLASS) + #include +#endif + void HeltecV4R8Board::begin() { ESP32Board::begin(); @@ -8,8 +12,8 @@ void HeltecV4R8Board::begin() { loRaFEMControl.init(); - // Expansion Kit CHSC6X touch RST/INT are unwired. GPIO 21 is TFT RST - // (PIN_TFT_RST), owned by ST7789LCDDisplay. Do not pulse it here. + // GPIO 21 is shared by LCD_RST and TP_RST. Let ST7789LCDDisplay own the + // reset sequence; no separate touch reset is needed. #ifdef PIN_TOUCH_RST pinMode(PIN_TOUCH_RST, OUTPUT); digitalWrite(PIN_TOUCH_RST, HIGH); @@ -42,6 +46,10 @@ void HeltecV4R8Board::onAfterTransmit(void) { } void HeltecV4R8Board::enterDeepSleep(uint32_t secs, int pin_wake_btn) { +#if defined(HELTEC_V4_R8_TFT) && defined(DISPLAY_CLASS) + display.turnOff(); +#endif + esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON); rtc_gpio_set_direction((gpio_num_t)P_LORA_DIO_1, RTC_GPIO_MODE_INPUT_ONLY); diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index 246eb785..e732e571 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -68,7 +68,7 @@ build_flags = -D PIN_TFT_RST=21 -D PIN_TFT_VDD_CTL=-1 -D PIN_TFT_LEDA_CTL=44 - -D PIN_TFT_LEDA_CTL_ACTIVE=HIGH + -D PIN_TFT_LEDA_CTL_ACTIVE=LOW -D PIN_TFT_CS=47 -D PIN_TFT_DC=48 -D PIN_TFT_SCL=16 From 85fd9b09be47c63c77358eb2c92209cc547c7b98 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 07:44:16 -0700 Subject: [PATCH 40/93] fix(boards): update Heltec V4 R8 TFT LEDA control to active HIGH --- variants/heltec_v4_r8/platformio.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index e732e571..246eb785 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -68,7 +68,7 @@ build_flags = -D PIN_TFT_RST=21 -D PIN_TFT_VDD_CTL=-1 -D PIN_TFT_LEDA_CTL=44 - -D PIN_TFT_LEDA_CTL_ACTIVE=LOW + -D PIN_TFT_LEDA_CTL_ACTIVE=HIGH -D PIN_TFT_CS=47 -D PIN_TFT_DC=48 -D PIN_TFT_SCL=16 From fc361ca94be22aa8458f74fa4570bdf10da2129d Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 09:21:38 -0700 Subject: [PATCH 41/93] feat(display): add R8 portrait observer layouts --- examples/simple_repeater/UITask.cpp | 87 +++++++++++- examples/simple_repeater/UITask.h | 9 +- examples/simple_room_server/UITask.cpp | 83 ++++++++++- examples/simple_room_server/UITask.h | 9 +- src/helpers/ui/DisplayFrameSignature.h | 23 +++ src/helpers/ui/DisplayViewport.h | 41 ++++++ src/helpers/ui/ST7789LCDDisplay.cpp | 134 ++++++++++++++++++ src/helpers/ui/ST7789LCDDisplay.h | 8 ++ test/README.md | 1 + .../test_display_viewport.cpp | 83 +++++++++++ variants/heltec_v4_r8/platformio.ini | 16 +++ 11 files changed, 482 insertions(+), 12 deletions(-) create mode 100644 src/helpers/ui/DisplayFrameSignature.h create mode 100644 src/helpers/ui/DisplayViewport.h create mode 100644 test/test_display_viewport/test_display_viewport.cpp diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index 2cd75ee4..e8f48ed3 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -3,6 +3,10 @@ #include #include +#ifdef DISPLAY_REDRAW_ON_CHANGE +#include +#endif + #ifndef USER_BTN_PRESSED #define USER_BTN_PRESSED LOW #endif @@ -40,6 +44,9 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi _started_at = millis(); _node_prefs = node_prefs; _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif #if defined(PIN_USER_BTN) && defined(DISPLAY_CLASS) user_btn.begin(); @@ -163,6 +170,54 @@ void UITask::renderCurrScreen() { } } +#ifdef DISPLAY_REDRAW_ON_CHANGE +uint32_t UITask::getFrameSignature() { + uint32_t signature = DisplayFrameSignature::INITIAL; + char tmp[80]; + + if (millis() < _started_at + BOOT_SCREEN_MILLIS) { + signature = DisplayFrameSignature::append(signature, "boot"); + return DisplayFrameSignature::append(signature, _version_info); + } + + if (_powering_off_at > 0) { + return DisplayFrameSignature::append(signature, "powering-off"); + } + +#ifdef WITH_WEBCONFIG + if (WebConfigServer::isRebootPending()) { + return DisplayFrameSignature::append(signature, "rebooting"); + } + + char wc_ssid[33], wc_ip[16]; + if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + signature = DisplayFrameSignature::append(signature, "setup"); + signature = DisplayFrameSignature::append(signature, wc_ssid); + return DisplayFrameSignature::append(signature, wc_ip); + } +#endif + + signature = DisplayFrameSignature::append(signature, "home"); + signature = DisplayFrameSignature::append(signature, _node_prefs->node_name); + snprintf(tmp, sizeof(tmp), "FREQ: %06.3f SF%d", _node_prefs->freq, _node_prefs->sf); + signature = DisplayFrameSignature::append(signature, tmp); + snprintf(tmp, sizeof(tmp), "BW: %03.2f CR: %d", _node_prefs->bw, _node_prefs->cr); + signature = DisplayFrameSignature::append(signature, tmp); + +#ifdef WITH_MQTT_BRIDGE + if (WiFi.status() == WL_CONNECTED) { + IPAddress ip = WiFi.localIP(); + snprintf(tmp, sizeof(tmp), "IP: %d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]); + signature = DisplayFrameSignature::append(signature, tmp); + } else { + signature = DisplayFrameSignature::append(signature, "wifi-disconnected"); + } +#endif + + return signature; +} +#endif + void UITask::loop() { #if defined(PIN_USER_BTN) && defined(DISPLAY_CLASS) int ev = user_btn.check(); @@ -171,6 +226,9 @@ void UITask::loop() { // TODO: any action ? } else { _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif } _auto_off = millis() + AUTO_OFF_MILLIS; // extend auto-off timer } else if (ev == BUTTON_EVENT_LONG_PRESS) { @@ -184,21 +242,40 @@ void UITask::loop() { // While the setup portal is up there's no user button to wake the screen // reliably - keep it on so the join instructions stay visible. if (WebConfigServer::getSetupInfo(NULL, 0, NULL, 0)) { - if (!_display->isOn()) _display->turnOn(); + if (!_display->isOn()) { + _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif + } _auto_off = millis() + AUTO_OFF_MILLIS; } #endif if (_display->isOn()) { if (millis() >= _next_refresh) { - _display->startFrame(); - renderCurrScreen(); - _display->endFrame(); + bool redraw = true; +#ifdef DISPLAY_REDRAW_ON_CHANGE + uint32_t frame_signature = getFrameSignature(); + redraw = !_frame_valid || frame_signature != _last_frame_signature; +#endif + if (redraw) { + _display->startFrame(); + renderCurrScreen(); + _display->endFrame(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _last_frame_signature = frame_signature; + _frame_valid = true; +#endif + } - _next_refresh = millis() + 1000; // refresh every second + _next_refresh = millis() + 1000; // check for visible changes every second } if (millis() > _auto_off) { _display->turnOff(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif } } diff --git a/examples/simple_repeater/UITask.h b/examples/simple_repeater/UITask.h index d8e3ce1d..f0ce4221 100644 --- a/examples/simple_repeater/UITask.h +++ b/examples/simple_repeater/UITask.h @@ -13,10 +13,17 @@ class UITask { unsigned long _powering_off_at = 0; unsigned long _started_at = 0; +#ifdef DISPLAY_REDRAW_ON_CHANGE + uint32_t _last_frame_signature = 0; + bool _frame_valid = false; + + uint32_t getFrameSignature(); +#endif + void renderCurrScreen(); public: UITask(mesh::MainBoard& board, DisplayDriver& display) : _board(&board), _display(&display) { _next_read = _next_refresh = 0; } void begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version); void loop(); -}; \ No newline at end of file +}; diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 97562297..884ab528 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -2,6 +2,10 @@ #include #include +#ifdef DISPLAY_REDRAW_ON_CHANGE +#include +#endif + #ifndef USER_BTN_PRESSED #define USER_BTN_PRESSED LOW #endif @@ -36,6 +40,9 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi _auto_off = millis() + AUTO_OFF_MILLIS; _node_prefs = node_prefs; _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif // strip off dash and commit hash by changing dash to null terminator // e.g: v1.2.3-abcdef -> v1.2.3 @@ -144,6 +151,50 @@ void UITask::renderCurrScreen() { } } +#ifdef DISPLAY_REDRAW_ON_CHANGE +uint32_t UITask::getFrameSignature() { + uint32_t signature = DisplayFrameSignature::INITIAL; + char tmp[80]; + + if (millis() < BOOT_SCREEN_MILLIS) { + signature = DisplayFrameSignature::append(signature, "boot"); + return DisplayFrameSignature::append(signature, _version_info); + } + +#ifdef WITH_WEBCONFIG + if (WebConfigServer::isRebootPending()) { + return DisplayFrameSignature::append(signature, "rebooting"); + } + + char wc_ssid[33], wc_ip[16]; + if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + signature = DisplayFrameSignature::append(signature, "setup"); + signature = DisplayFrameSignature::append(signature, wc_ssid); + return DisplayFrameSignature::append(signature, wc_ip); + } +#endif + + signature = DisplayFrameSignature::append(signature, "home"); + signature = DisplayFrameSignature::append(signature, _node_prefs->node_name); + snprintf(tmp, sizeof(tmp), "FREQ: %06.3f SF%d", _node_prefs->freq, _node_prefs->sf); + signature = DisplayFrameSignature::append(signature, tmp); + snprintf(tmp, sizeof(tmp), "BW: %03.2f CR: %d", _node_prefs->bw, _node_prefs->cr); + signature = DisplayFrameSignature::append(signature, tmp); + +#ifdef WITH_MQTT_BRIDGE + if (WiFi.status() == WL_CONNECTED) { + IPAddress ip = WiFi.localIP(); + snprintf(tmp, sizeof(tmp), "IP: %d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]); + signature = DisplayFrameSignature::append(signature, tmp); + } else { + signature = DisplayFrameSignature::append(signature, "wifi-disconnected"); + } +#endif + + return signature; +} +#endif + void UITask::loop() { #ifdef PIN_USER_BTN if (millis() >= _next_read) { @@ -154,6 +205,9 @@ void UITask::loop() { // TODO: any action ? } else { _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif } _auto_off = millis() + AUTO_OFF_MILLIS; // extend auto-off timer } @@ -167,21 +221,40 @@ void UITask::loop() { // While the setup portal is up there's no user button to wake the screen // reliably - keep it on so the join instructions stay visible. if (WebConfigServer::getSetupInfo(NULL, 0, NULL, 0)) { - if (!_display->isOn()) _display->turnOn(); + if (!_display->isOn()) { + _display->turnOn(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif + } _auto_off = millis() + AUTO_OFF_MILLIS; } #endif if (_display->isOn()) { if (millis() >= _next_refresh) { - _display->startFrame(); - renderCurrScreen(); - _display->endFrame(); + bool redraw = true; +#ifdef DISPLAY_REDRAW_ON_CHANGE + uint32_t frame_signature = getFrameSignature(); + redraw = !_frame_valid || frame_signature != _last_frame_signature; +#endif + if (redraw) { + _display->startFrame(); + renderCurrScreen(); + _display->endFrame(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _last_frame_signature = frame_signature; + _frame_valid = true; +#endif + } - _next_refresh = millis() + 1000; // refresh every second + _next_refresh = millis() + 1000; // check for visible changes every second } if (millis() > _auto_off) { _display->turnOff(); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif } } } diff --git a/examples/simple_room_server/UITask.h b/examples/simple_room_server/UITask.h index a27259f1..7ef88778 100644 --- a/examples/simple_room_server/UITask.h +++ b/examples/simple_room_server/UITask.h @@ -10,10 +10,17 @@ class UITask { NodePrefs* _node_prefs; char _version_info[32]; +#ifdef DISPLAY_REDRAW_ON_CHANGE + uint32_t _last_frame_signature = 0; + bool _frame_valid = false; + + uint32_t getFrameSignature(); +#endif + void renderCurrScreen(); public: UITask(DisplayDriver& display) : _display(&display) { _next_read = _next_refresh = 0; } void begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version); void loop(); -}; \ No newline at end of file +}; diff --git a/src/helpers/ui/DisplayFrameSignature.h b/src/helpers/ui/DisplayFrameSignature.h new file mode 100644 index 00000000..9cf545d6 --- /dev/null +++ b/src/helpers/ui/DisplayFrameSignature.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace DisplayFrameSignature { + +static constexpr uint32_t INITIAL = 2166136261u; + +inline uint32_t append(uint32_t signature, const char *text) { + if (text) { + while (*text) { + signature ^= static_cast(*text++); + signature *= 16777619u; + } + } + + // Separate adjacent fields so { "ab", "c" } differs from { "a", "bc" }. + signature ^= 0xFFu; + signature *= 16777619u; + return signature; +} + +} // namespace DisplayFrameSignature diff --git a/src/helpers/ui/DisplayViewport.h b/src/helpers/ui/DisplayViewport.h new file mode 100644 index 00000000..a3812c3e --- /dev/null +++ b/src/helpers/ui/DisplayViewport.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace DisplayViewport { + +struct Geometry { + int16_t logical_width; + int16_t logical_height; + int16_t physical_width; + int16_t physical_height; + + int16_t mapX(int16_t x) const { + return static_cast((static_cast(x) * physical_width) / logical_width); + } + + int16_t mapY(int16_t y) const { + return static_cast((static_cast(y) * physical_height) / logical_height); + } + + uint16_t spanX(int16_t x, int16_t width) const { return static_cast(mapX(x + width) - mapX(x)); } + + uint16_t spanY(int16_t y, int16_t height) const { + return static_cast(mapY(y + height) - mapY(y)); + } + + uint16_t logicalWidthForPhysical(uint16_t width) const { + return static_cast((static_cast(width) * logical_width + physical_width - 1) / + physical_width); + } +}; + +inline uint8_t selectTextScale(uint16_t width_at_scale_one, uint8_t preferred_scale, uint8_t minimum_scale, + uint16_t available_width) { + if (static_cast(width_at_scale_one) * preferred_scale <= available_width) { + return preferred_scale; + } + return minimum_scale; +} + +} // namespace DisplayViewport diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index 137dfa15..1d00fd15 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -1,5 +1,9 @@ #include "ST7789LCDDisplay.h" +#ifdef ST7789_PORTRAIT_PROFILE + #include "DisplayViewport.h" +#endif + #ifndef PIN_TFT_MISO #define PIN_TFT_MISO -1 #endif @@ -19,6 +23,16 @@ #define DISPLAY_WIDTH 240 #define DISPLAY_HEIGHT 320 +#ifdef ST7789_PORTRAIT_PROFILE + #ifndef ST7789_PORTRAIT_TEXT_SCALE + #define ST7789_PORTRAIT_TEXT_SCALE 2 + #endif + +static DisplayViewport::Geometry portraitViewport(int16_t physical_width, int16_t physical_height) { + return {128, 64, physical_width, physical_height}; +} +#endif + bool ST7789LCDDisplay::i2c_probe(TwoWire& wire, uint8_t addr) { return true; } @@ -64,7 +78,13 @@ bool ST7789LCDDisplay::begin() { display.fillScreen(ST77XX_BLACK); display.setTextColor(ST77XX_WHITE); + #ifdef ST7789_PORTRAIT_PROFILE + _logical_text_size = 1; + display.setTextSize(ST7789_PORTRAIT_TEXT_SCALE); + display.setTextWrap(false); + #else display.setTextSize(2 * DISPLAY_SCALE_X); + #endif display.cp437(true); // Use full 256 char 'Code Page 437' font #ifdef HELTEC_V4_R8_TFT @@ -113,12 +133,22 @@ void ST7789LCDDisplay::clear() { void ST7789LCDDisplay::startFrame(ColorVal bkg) { display.fillScreen(bkg); display.setTextColor(_color = UIColor::primary_txt); +#ifdef ST7789_PORTRAIT_PROFILE + _logical_text_size = 1; + display.setTextSize(ST7789_PORTRAIT_TEXT_SCALE); +#else display.setTextSize(1 * DISPLAY_SCALE_X); // This one affects size of Please wait... message +#endif display.cp437(true); // Use full 256 char 'Code Page 437' font } void ST7789LCDDisplay::setTextSize(int sz) { +#ifdef ST7789_PORTRAIT_PROFILE + _logical_text_size = sz > 0 ? static_cast(sz) : 1; + display.setTextSize(_logical_text_size * ST7789_PORTRAIT_TEXT_SCALE); +#else display.setTextSize(sz * DISPLAY_SCALE_X); +#endif } void ST7789LCDDisplay::setColor(ColorVal c) { @@ -126,24 +156,70 @@ void ST7789LCDDisplay::setColor(ColorVal c) { } void ST7789LCDDisplay::setCursor(int x, int y) { +#ifdef ST7789_PORTRAIT_PROFILE + DisplayViewport::Geometry viewport = portraitViewport(display.width(), display.height()); + display.setCursor(viewport.mapX(x), viewport.mapY(y)); +#else display.setCursor(x * DISPLAY_SCALE_X, y * DISPLAY_SCALE_Y); +#endif } void ST7789LCDDisplay::print(const char* str) { +#ifdef ST7789_PORTRAIT_PROFILE + int16_t cursor_x = display.getCursorX(); + if (cursor_x < 0) { + cursor_x = 0; + display.setCursor(cursor_x, display.getCursorY()); + } + if (cursor_x >= display.width()) return; + + printFitted(str, static_cast(display.width() - cursor_x)); +#else display.print(str); +#endif } void ST7789LCDDisplay::fillRect(int x, int y, int w, int h) { +#ifdef ST7789_PORTRAIT_PROFILE + DisplayViewport::Geometry viewport = portraitViewport(display.width(), display.height()); + display.fillRect(viewport.mapX(x), viewport.mapY(y), viewport.spanX(x, w), viewport.spanY(y, h), _color); +#else display.fillRect(x * DISPLAY_SCALE_X, y * DISPLAY_SCALE_Y, w * DISPLAY_SCALE_X, h * DISPLAY_SCALE_Y, _color); +#endif } void ST7789LCDDisplay::drawRect(int x, int y, int w, int h) { +#ifdef ST7789_PORTRAIT_PROFILE + DisplayViewport::Geometry viewport = portraitViewport(display.width(), display.height()); + display.drawRect(viewport.mapX(x), viewport.mapY(y), viewport.spanX(x, w), viewport.spanY(y, h), _color); +#else display.drawRect(x * DISPLAY_SCALE_X, y * DISPLAY_SCALE_Y, w * DISPLAY_SCALE_X, h * DISPLAY_SCALE_Y, _color); +#endif } void ST7789LCDDisplay::drawXbm(int x, int y, const uint8_t* bits, int w, int h) { uint8_t byteWidth = (w + 7) / 8; +#ifdef ST7789_PORTRAIT_PROFILE + DisplayViewport::Geometry viewport = portraitViewport(display.width(), display.height()); + int16_t physical_y = viewport.mapY(y); + + for (int j = 0; j < h; j++) { + // Scale both bitmap axes from the logical X ratio so logo pixels stay square. + int16_t y0 = physical_y + viewport.mapX(j); + int16_t y1 = physical_y + viewport.mapX(j + 1); + for (int i = 0; i < w; i++) { + uint8_t byte = bits[j * byteWidth + i / 8]; + bool pixelOn = byte & (0x80 >> (i & 7)); + + if (pixelOn) { + int16_t x0 = viewport.mapX(x + i); + int16_t x1 = viewport.mapX(x + i + 1); + display.fillRect(x0, y0, x1 - x0, y1 - y0, _color); + } + } + } +#else for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { uint8_t byte = bits[j * byteWidth + i / 8]; @@ -158,16 +234,74 @@ void ST7789LCDDisplay::drawXbm(int x, int y, const uint8_t* bits, int w, int h) } } } +#endif } uint16_t ST7789LCDDisplay::getTextWidth(const char* str) { +#ifdef ST7789_PORTRAIT_PROFILE + uint8_t physical_scale = selectTextScale(str, display.width()); + uint16_t physical_width = measureTextWidth(str, physical_scale); + if (physical_width > display.width()) physical_width = display.width(); + + DisplayViewport::Geometry viewport = portraitViewport(display.width(), display.height()); + return viewport.logicalWidthForPhysical(physical_width); +#else int16_t x1, y1; uint16_t w, h; display.getTextBounds(str, 0, 0, &x1, &y1, &w, &h); return w / DISPLAY_SCALE_X; +#endif } +#ifdef ST7789_PORTRAIT_PROFILE +uint16_t ST7789LCDDisplay::measureTextWidth(const char* str, uint8_t physical_scale) { + int16_t x1, y1; + uint16_t w, h; + display.setTextSize(physical_scale); + display.getTextBounds(str, 0, 0, &x1, &y1, &w, &h); + return w; +} + +uint8_t ST7789LCDDisplay::selectTextScale(const char* str, uint16_t available_width) { + uint16_t width_at_scale_one = measureTextWidth(str, 1); + uint8_t preferred_scale = _logical_text_size * ST7789_PORTRAIT_TEXT_SCALE; + return DisplayViewport::selectTextScale(width_at_scale_one, preferred_scale, _logical_text_size, + available_width); +} + +void ST7789LCDDisplay::printFitted(const char* str, uint16_t available_width) { + if (!str || available_width == 0) return; + + uint8_t physical_scale = selectTextScale(str, available_width); + if (measureTextWidth(str, physical_scale) <= available_width) { + display.print(str); + return; + } + + static const char* ellipsis = "..."; + uint16_t ellipsis_width = measureTextWidth(ellipsis, physical_scale); + if (ellipsis_width > available_width) return; + + char fitted[256]; + size_t len = strlen(str); + if (len > sizeof(fitted) - 4) len = sizeof(fitted) - 4; + memcpy(fitted, str, len); + fitted[len] = 0; + + while (len > 0 && measureTextWidth(fitted, physical_scale) + ellipsis_width > available_width) { + --len; + while (len > 0 && (static_cast(fitted[len]) & 0xC0) == 0x80) + --len; + fitted[len] = 0; + } + + memcpy(fitted + len, ellipsis, 4); + display.setTextSize(physical_scale); + display.print(fitted); +} +#endif + void ST7789LCDDisplay::endFrame() { // display.display(); } diff --git a/src/helpers/ui/ST7789LCDDisplay.h b/src/helpers/ui/ST7789LCDDisplay.h index b5127d35..f6d7446b 100644 --- a/src/helpers/ui/ST7789LCDDisplay.h +++ b/src/helpers/ui/ST7789LCDDisplay.h @@ -16,6 +16,14 @@ class ST7789LCDDisplay : public DisplayDriver { uint16_t _color; RefCountedDigitalPin* _peripher_power; +#ifdef ST7789_PORTRAIT_PROFILE + uint8_t _logical_text_size = 1; + + uint16_t measureTextWidth(const char* str, uint8_t physical_scale); + uint8_t selectTextScale(const char* str, uint16_t available_width); + void printFitted(const char* str, uint16_t available_width); +#endif + bool i2c_probe(TwoWire& wire, uint8_t addr); public: #ifdef USE_PIN_TFT diff --git a/test/README.md b/test/README.md index 6fde2d7f..901227ef 100644 --- a/test/README.md +++ b/test/README.md @@ -30,6 +30,7 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_topic_router` | `src/helpers/MQTTTopicRouter.h` | complete preset/custom topic-routing contract; MeshRank all types except raw; required identifiers; invalid inputs/slots; exact buffer boundaries | | `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover; WiFi current-outage start sticky across STA reconnect attempts | | `test_alert_fault_policy` | `src/helpers/AlertFaultPolicy.h` | WiFi/MQTT fault edge detector; `OutageSnapshot` (down / started_ms / initiating reason) fed to tick and `formatWifiAlert`; reason-8 reconnects change neither duration nor initiating reason; flap between status polls; down at `millis()==0`; packed 64-bit cross-task word; rate-limit floor and first-fire; 5 s poll cadence and `millis()` rollover | +| `test_display_viewport` | `src/helpers/ui/DisplayViewport.h`, `src/helpers/ui/DisplayFrameSignature.h` | logical-to-physical portrait mapping; fractional span coverage; fitted-width conversion; preferred/fallback text scaling; stable visible-frame change detection | | `test_mqtt_packet_queue_policy` | `src/helpers/MQTTPacketQueuePolicy.h` | queue-full eviction; stale-disconnect flush; adaptive drain limits; bounded QoS0 retries; exact timing boundaries and 32-bit `millis()` rollover | | `test_mqtt_packet_filter` | `src/helpers/MQTTPacketFilter.h` | per-slot 0-15 allowlist parsing/formatting, numeric and named spellings; exact bounds; membership; candidate/eligible split and retry-completion policy; pre-queue union gate; default-mask detection | | `test_mqtt_runtime_buffer_lifecycle` | `src/helpers/MQTTRuntimeBufferLifecycle.h` | idempotent allocation/release; partial-allocation degradation; retry of only missing buffers | diff --git a/test/test_display_viewport/test_display_viewport.cpp b/test/test_display_viewport/test_display_viewport.cpp new file mode 100644 index 00000000..35378bd0 --- /dev/null +++ b/test/test_display_viewport/test_display_viewport.cpp @@ -0,0 +1,83 @@ +#include "helpers/ui/DisplayFrameSignature.h" +#include "helpers/ui/DisplayViewport.h" + +#include + +namespace { + +const DisplayViewport::Geometry kPortrait{ 128, 64, 240, 320 }; + +} // namespace + +TEST(DisplayViewport, MapsLogicalBoundsToPortraitPanel) { + EXPECT_EQ(0, kPortrait.mapX(0)); + EXPECT_EQ(240, kPortrait.mapX(128)); + EXPECT_EQ(0, kPortrait.mapY(0)); + EXPECT_EQ(320, kPortrait.mapY(64)); +} + +TEST(DisplayViewport, MapsObserverRowsAcrossPortraitHeight) { + EXPECT_EQ(0, kPortrait.mapY(0)); + EXPECT_EQ(70, kPortrait.mapY(14)); + EXPECT_EQ(100, kPortrait.mapY(20)); + EXPECT_EQ(120, kPortrait.mapY(24)); + EXPECT_EQ(150, kPortrait.mapY(30)); + EXPECT_EQ(200, kPortrait.mapY(40)); + EXPECT_EQ(240, kPortrait.mapY(48)); + EXPECT_EQ(250, kPortrait.mapY(50)); +} + +TEST(DisplayViewport, FractionalHorizontalSpansHaveNoGapsOrOverlaps) { + int16_t previous_end = 0; + int total_width = 0; + + for (int x = 0; x < 128; ++x) { + int16_t start = kPortrait.mapX(x); + int16_t end = kPortrait.mapX(x + 1); + EXPECT_EQ(previous_end, start); + EXPECT_TRUE(end - start == 1 || end - start == 2); + total_width += end - start; + previous_end = end; + } + + EXPECT_EQ(240, previous_end); + EXPECT_EQ(240, total_width); +} + +TEST(DisplayViewport, ConvertsFittedPhysicalWidthBackToLogicalWidth) { + EXPECT_EQ(0, kPortrait.logicalWidthForPhysical(0)); + EXPECT_EQ(64, kPortrait.logicalWidthForPhysical(120)); + EXPECT_EQ(122, kPortrait.logicalWidthForPhysical(228)); + EXPECT_EQ(128, kPortrait.logicalWidthForPhysical(240)); +} + +TEST(DisplayViewport, SelectsPreferredTextScaleOnlyWhenItFits) { + EXPECT_EQ(2, DisplayViewport::selectTextScale(114, 2, 1, 240)); + EXPECT_EQ(1, DisplayViewport::selectTextScale(150, 2, 1, 240)); + EXPECT_EQ(1, DisplayViewport::selectTextScale(192, 2, 1, 229)); + EXPECT_EQ(1, DisplayViewport::selectTextScale(300, 2, 1, 240)); +} + +TEST(DisplayFrameSignature, ChangesWithVisibleContent) { + uint32_t initial = DisplayFrameSignature::INITIAL; + uint32_t home = DisplayFrameSignature::append(initial, "home"); + + EXPECT_EQ(home, DisplayFrameSignature::append(initial, "home")); + EXPECT_NE(home, DisplayFrameSignature::append(initial, "setup")); + EXPECT_NE(DisplayFrameSignature::append(home, "10.0.0.1"), DisplayFrameSignature::append(home, "10.0.0.2")); +} + +TEST(DisplayFrameSignature, KeepsAdjacentFieldsDistinct) { + uint32_t first = DisplayFrameSignature::append(DisplayFrameSignature::INITIAL, "ab"); + first = DisplayFrameSignature::append(first, "c"); + + uint32_t second = DisplayFrameSignature::append(DisplayFrameSignature::INITIAL, "a"); + second = DisplayFrameSignature::append(second, "bc"); + + EXPECT_NE(first, second); +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index 246eb785..9025d422 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -334,6 +334,7 @@ build_flags = -D ESP32_CPU_FREQ=160 -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm -D WITH_SNMP=1 + -D DISPLAY_REDRAW_ON_CHANGE=1 build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + @@ -351,6 +352,13 @@ lib_deps = paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent +[env:heltec_v4_r8_tft_portrait_repeater_observer_mqtt] +extends = env:heltec_v4_r8_tft_repeater_observer_mqtt +build_flags = + ${env:heltec_v4_r8_tft_repeater_observer_mqtt.build_flags} + -D ST7789_PORTRAIT_PROFILE=1 + -D DISPLAY_ROTATION=2 + [env:heltec_v4_r8_tft_room_server] extends = heltec_v4_r8_tft build_flags = @@ -393,6 +401,7 @@ build_flags = -D ESP32_CPU_FREQ=160 -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm -D WITH_SNMP=1 + -D DISPLAY_REDRAW_ON_CHANGE=1 build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + @@ -410,6 +419,13 @@ lib_deps = paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent +[env:heltec_v4_r8_tft_portrait_room_server_observer_mqtt] +extends = env:heltec_v4_r8_tft_room_server_observer_mqtt +build_flags = + ${env:heltec_v4_r8_tft_room_server_observer_mqtt.build_flags} + -D ST7789_PORTRAIT_PROFILE=1 + -D DISPLAY_ROTATION=2 + [env:heltec_v4_r8_tft_terminal_chat] extends = heltec_v4_r8_tft build_flags = From fcd92e985f8cd9fbcba38e4927874dcae7455e21 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 13:39:16 -0700 Subject: [PATCH 42/93] feat(display): add R8 observer TFT dashboard, touch toggle and display.timeout Replace the sparse Heltec V4 R8 observer home screen with a padded dark analytics dashboard, add manual display control, and make blanking a runtime setting. Dashboard (DISPLAY_ACTIVITY_DASHBOARD, the four R8 TFT observer envs): - RadioActivityWindow: 20 one-minute buckets of valid RX packets, no heap. The caller's 32-bit millis() is extended to a monotonic 64-bit clock, so nothing downstream has a rollover case; an always-on node passes 2^32 ms after ~49.7 days, which would otherwise re-enter warm-up and divide 20 minutes of traffic by seconds. Rates use 19 whole minutes plus the elapsed part of the current one rather than a fixed 1200 s. - ObserverDashboard: header, radio strip, headline totals, a 20-bar packets-per-minute graph and RF/status footers, with separate portrait and landscape layouts. A text row is a fixed 16 px, which is 3.2 logical units in portrait but 4.27 in landscape, so one shared grid would overlap. Text is trimmed by character budget, not measured width: getTextWidth() reports an over-long string at the portrait driver's fallback scale, so DisplayDriver::drawTextEllipsized() under-trims and the row renders at half height. - Six per-row signatures computed from what is actually drawn, so only the rows whose pixels changed repaint. No startFrame(), no whole-screen clear. Link state moved out of the full-frame signature, so a DHCP renewal or WiFi flap repaints one footer row instead of the panel. - Dark theme by retuning the UIColor statics at runtime, which needs no display-driver edit and carries boot, setup, reboot and power-off with it. Touch and button (DISPLAY_TOUCH_TOGGLE): - CHSC6X at I2C 0x2E, polled; TP_INT is unusable (optional R13, and GPIO 43 is U0TXD). The point-count byte is tested against a valid count, never against non-zero: an idle read returns 0xFF, which reads as a finger held down forever and latches the tap detector after one event. - turnOff() no longer parks PIN_TFT_RST low on this board. GPIO 21 is a shared LCD_RST/TP_RST net, so doing that held the touch controller in reset for as long as the display was off. Verified against Heltec's expansion-board and mainboard schematics and the V4-R8 datasheet pinout, which also correct the pin comment in HeltecV4R8Board.cpp. - The USER button click now toggles the display too; it previously did nothing whenever the display was already on. display.timeout: - `set display.timeout ` / `get display.timeout`, 0 = stay on, 60 s default, 3600 max. Read live, so a change applies without a reboot and restarts the countdown rather than firing on the old deadline. - Stored in MQTTPrefs (/mqtt.json), keeping NodePrefs aligned with upstream. Runtime-only: LegacyV1MQTTPrefs and the four frozen binary payload sizes are unchanged. No JSON format-version bump - the loader skips keys no def() claims, so older firmware reads newer files and this firmware reads older ones with the default applied. Both directions are covered by tests. - Joins the observer atomic-setter contract, so a failed save rolls the live value back instead of only claiming to. New periodic work uses a wrap-safe deadline check; `millis() >= deadline` fires every loop for a whole interval before each rollover. Adds test_radio_activity_window, test_observer_dashboard (driving the real renderer against a recording DisplayDriver in both orientation profiles) and test_touch_tap_detector. 440 native cases pass. --- examples/simple_repeater/MyMesh.cpp | 5 + examples/simple_repeater/MyMesh.h | 15 + examples/simple_repeater/UITask.cpp | 155 ++++- examples/simple_repeater/UITask.h | 48 ++ examples/simple_repeater/main.cpp | 6 + examples/simple_room_server/MyMesh.cpp | 5 + examples/simple_room_server/MyMesh.h | 15 + examples/simple_room_server/UITask.cpp | 155 ++++- examples/simple_room_server/UITask.h | 48 ++ examples/simple_room_server/main.cpp | 6 + src/helpers/CommonCLI_Observer.cpp | 29 +- src/helpers/MQTTDefaults.h | 2 + src/helpers/MQTTPrefsSerializer.h | 23 +- src/helpers/MQTTPrefsStorage.h | 8 + src/helpers/RadioActivityWindow.h | 207 +++++++ src/helpers/ui/CHSC6XTouch.h | 108 ++++ src/helpers/ui/ObserverDashboard.h | 524 ++++++++++++++++ src/helpers/ui/ST7789LCDDisplay.cpp | 7 +- src/helpers/ui/TouchTapDetector.h | 61 ++ test/README.md | 3 + .../test_mqtt_prefs_serializer.cpp | 67 ++ test/test_observer_dashboard/MockDisplay.h | 122 ++++ .../test_observer_dashboard.cpp | 581 ++++++++++++++++++ .../test_radio_activity_window.cpp | 376 ++++++++++++ .../test_touch_tap_detector.cpp | 123 ++++ variants/heltec_v4_r8/HeltecV4R8Board.cpp | 19 +- variants/heltec_v4_r8/platformio.ini | 16 + 27 files changed, 2709 insertions(+), 25 deletions(-) create mode 100644 src/helpers/RadioActivityWindow.h create mode 100644 src/helpers/ui/CHSC6XTouch.h create mode 100644 src/helpers/ui/ObserverDashboard.h create mode 100644 src/helpers/ui/TouchTapDetector.h create mode 100644 test/test_observer_dashboard/MockDisplay.h create mode 100644 test/test_observer_dashboard/test_observer_dashboard.cpp create mode 100644 test/test_radio_activity_window/test_radio_activity_window.cpp create mode 100644 test/test_touch_tap_detector/test_touch_tap_detector.cpp diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 859076e9..6c96112a 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -540,6 +540,11 @@ void MyMesh::logRxRaw(float snr, float rssi, const uint8_t raw[], int len) { } void MyMesh::logRx(mesh::Packet *pkt, int len, float score) { +#ifdef DISPLAY_ACTIVITY_DASHBOARD + // Valid parsed RF packet: the only event the dashboard's window counts. + _activity.recordPacket(millis(), (uint16_t)len, _radio->getEstAirtimeFor(len), + (int8_t)(pkt->getSNR() * 4.0f), (int16_t)_radio->getLastRSSI()); +#endif #ifdef WITH_MQTT_BRIDGE // MQTT bridge: always feed RX packets — bridge decides based on mqtt.rx setting if (bridge) bridge->onPacketReceived(pkt); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index e4c3a3f7..aabd605a 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -35,6 +35,10 @@ #include "helpers/SNMPAgent.h" #endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD +#include +#endif + #include #include #include @@ -101,6 +105,9 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint64_t uptime_millis; unsigned long next_local_advert, next_flood_advert; bool _logging; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow _activity; // rolling RF receive window, for the TFT dashboard +#endif NodePrefs _prefs; ClientACL acl; CommonCLI _cli; @@ -282,6 +289,14 @@ public: return &_prefs; } +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow* getActivityWindow() { return &_activity; } +#endif + +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* getObserverPrefs() { return _cli.getObserverPrefs(); } +#endif + void savePrefs() override { _cli.savePrefs(_fs); } diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index e8f48ed3..ce46b9c7 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -16,7 +16,31 @@ #include // defines WITH_WEBCONFIG on ESP32 #endif -#define AUTO_OFF_MILLIS 20000 // 20 seconds +#ifndef AUTO_OFF_MILLIS +#define AUTO_OFF_MILLIS 20000 // 20 seconds; 0 keeps the screen on +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE +#define TOUCH_POLL_MILLIS 50 +#endif + +// Wrap-safe deadline test. `millis() >= deadline` fires early for the whole +// interval before a rollover, because the deadline has already wrapped to a +// small value while millis() is still near UINT32_MAX; the signed difference +// stays correct across it. +static inline bool millisReached(unsigned long now, unsigned long deadline) { + return (int32_t)((uint32_t)now - (uint32_t)deadline) >= 0; +} + +// `display.timeout` when the observer prefs are available, otherwise the +// compiled-in default. Read on every use so a `set display.timeout` takes +// effect immediately. +unsigned long UITask::displayTimeoutMillis() const { +#ifdef WITH_MQTT_BRIDGE + if (_observer_prefs) return (unsigned long)_observer_prefs->display_timeout_secs * 1000UL; +#endif + return AUTO_OFF_MILLIS; +} #define BOOT_SCREEN_MILLIS 4000 // 4 seconds #define POWEROFF_DELAY 3000 @@ -40,10 +64,17 @@ static const uint8_t meshcore_logo [] PROGMEM = { void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version) { _prevBtnState = HIGH; - _auto_off = millis() + AUTO_OFF_MILLIS; + _timeout_seen = displayTimeoutMillis(); + _auto_off = millis() + displayTimeoutMillis(); _started_at = millis(); _node_prefs = node_prefs; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + ObserverDashboard::applyDarkPalette(); // retunes UIColor for this target only +#endif _display->turnOn(); +#ifdef DISPLAY_TOUCH_TOGGLE + _touch.begin(); +#endif #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; #endif @@ -67,6 +98,9 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi void UITask::renderCurrScreen() { char tmp[80]; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif if (millis() < _started_at + BOOT_SCREEN_MILLIS) { // boot screen // meshcore logo _display->setColor(UIColor::corp_blue); @@ -140,6 +174,10 @@ void UITask::renderCurrScreen() { _display->print(wc_ip); return; } +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + renderDashboard(); + return; #endif // node name _display->setCursor(0, 0); @@ -204,7 +242,7 @@ uint32_t UITask::getFrameSignature() { snprintf(tmp, sizeof(tmp), "BW: %03.2f CR: %d", _node_prefs->bw, _node_prefs->cr); signature = DisplayFrameSignature::append(signature, tmp); -#ifdef WITH_MQTT_BRIDGE +#if defined(WITH_MQTT_BRIDGE) && !defined(DISPLAY_ACTIVITY_DASHBOARD) if (WiFi.status() == WL_CONNECTED) { IPAddress ip = WiFi.localIP(); snprintf(tmp, sizeof(tmp), "IP: %d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]); @@ -218,19 +256,94 @@ uint32_t UITask::getFrameSignature() { } #endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD +#define ACTIVITY_REFRESH_MILLIS 5000 + +bool UITask::buildDashboardContext(ObserverDashboard::Context* ctx) { + if (_node_prefs == NULL) return false; + ctx->node_name = _node_prefs->node_name; + ctx->role_label = "REPEATER"; + ctx->freq = _node_prefs->freq; + ctx->sf = _node_prefs->sf; + ctx->bw = _node_prefs->bw; +#ifdef WITH_MQTT_BRIDGE + ctx->link_up = (WiFi.status() == WL_CONNECTED); +#else + ctx->link_up = false; +#endif + return true; +} + +void UITask::renderDashboard() { + ObserverDashboard::Context ctx; + if (!buildDashboardContext(&ctx)) return; + + RadioActivitySnapshot snap; + if (_activity) { + _activity->snapshot(millis(), &snap); + } else { + memset(&snap, 0, sizeof(snap)); + } + + const ObserverDashboard::Layout& layout = ObserverDashboard::activeLayout(); + ObserverDashboard::drawFull(*_display, layout, ctx, snap); + ObserverDashboard::allRowSignatures(layout, ctx, snap, _row_signatures); + _rows_valid = true; + _next_activity = millis() + ACTIVITY_REFRESH_MILLIS; +} + +// Repaints just the analytics rows whose contents moved. No startFrame(), so +// the header, the radio strip and the rest of the panel are never cleared. +void UITask::updateActivityRows() { + if (!_rows_valid || _activity == NULL) return; // not showing the dashboard + + ObserverDashboard::Context ctx; + if (!buildDashboardContext(&ctx)) return; + + RadioActivitySnapshot snap; + _activity->snapshot(millis(), &snap); + ObserverDashboard::drawChangedRows(*_display, ObserverDashboard::activeLayout(), ctx, snap, + _row_signatures); +} +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE +void UITask::toggleDisplay() { + if (_display->isOn()) { + _display->turnOff(); + } else { + _display->turnOn(); + } +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; // wake draws one complete current frame +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif + _auto_off = millis() + displayTimeoutMillis(); +} +#endif + void UITask::loop() { #if defined(PIN_USER_BTN) && defined(DISPLAY_CLASS) int ev = user_btn.check(); if (ev == BUTTON_EVENT_CLICK) { +#ifdef DISPLAY_TOUCH_TOGGLE + toggleDisplay(); // same action as tapping the panel +#else if (_display->isOn()) { // TODO: any action ? } else { _display->turnOn(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } - _auto_off = millis() + AUTO_OFF_MILLIS; // extend auto-off timer + _auto_off = millis() + displayTimeoutMillis(); // extend auto-off timer +#endif } else if (ev == BUTTON_EVENT_LONG_PRESS) { _display->turnOn(); Serial.println("Powering Off"); @@ -246,9 +359,22 @@ void UITask::loop() { _display->turnOn(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } - _auto_off = millis() + AUTO_OFF_MILLIS; + _auto_off = millis() + displayTimeoutMillis(); + } +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + { + unsigned long now = millis(); + if (millisReached(now, _next_touch)) { + _next_touch = now + TOUCH_POLL_MILLIS; + if (_touch.checkTap(now)) toggleDisplay(); + } } #endif @@ -268,13 +394,30 @@ void UITask::loop() { _frame_valid = true; #endif } +#ifdef DISPLAY_ACTIVITY_DASHBOARD + else if (millisReached(millis(), _next_activity)) { + updateActivityRows(); + _next_activity = millis() + ACTIVITY_REFRESH_MILLIS; + } +#endif _next_refresh = millis() + 1000; // check for visible changes every second } - if (millis() > _auto_off) { + // `_auto_off` is only armed on activity, so a timeout changed at runtime has + // to restart the countdown here - otherwise 0 -> 60 blanks instantly off a + // boot-time deadline, and 60 -> 3600 still blanks at the old 60 s mark. + unsigned long timeout = displayTimeoutMillis(); + if (timeout != _timeout_seen) { + _timeout_seen = timeout; + _auto_off = millis() + timeout; + } + if (timeout > 0 && millisReached(millis(), _auto_off)) { _display->turnOff(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } } diff --git a/examples/simple_repeater/UITask.h b/examples/simple_repeater/UITask.h index f0ce4221..ad6c5de1 100644 --- a/examples/simple_repeater/UITask.h +++ b/examples/simple_repeater/UITask.h @@ -3,6 +3,19 @@ #include #include +#ifdef DISPLAY_ACTIVITY_DASHBOARD + #ifndef DISPLAY_REDRAW_ON_CHANGE + #error "DISPLAY_ACTIVITY_DASHBOARD needs DISPLAY_REDRAW_ON_CHANGE: without it every frame clears the whole screen" + #endif + #include + #include + +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + #include +#endif + class UITask { mesh::MainBoard* _board; DisplayDriver* _display; @@ -20,10 +33,45 @@ class UITask { uint32_t getFrameSignature(); #endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow* _activity = NULL; + unsigned long _next_activity = 0; + uint32_t _row_signatures[ObserverDashboard::ROW_COUNT] = {0}; + bool _rows_valid = false; // true only while the dashboard is the drawn screen + + bool buildDashboardContext(ObserverDashboard::Context* ctx); + void renderDashboard(); + void updateActivityRows(); +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + CHSC6XTouch _touch; + unsigned long _next_touch = 0; + + void toggleDisplay(); +#endif + +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* _observer_prefs = NULL; +#endif + unsigned long _timeout_seen = 0; // to notice a live `display.timeout` change + + unsigned long displayTimeoutMillis() const; + void renderCurrScreen(); public: UITask(mesh::MainBoard& board, DisplayDriver& display) : _board(&board), _display(&display) { _next_read = _next_refresh = 0; } void begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version); +#ifdef WITH_MQTT_BRIDGE + // Supplies `display.timeout`, which is read live so a config change applies + // without a reboot. Call before begin(). + void setObserverPrefs(MQTTPrefs* prefs) { _observer_prefs = prefs; } +#endif + +#ifdef DISPLAY_ACTIVITY_DASHBOARD + void setActivityWindow(RadioActivityWindow* activity) { _activity = activity; } +#endif + void loop(); }; diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index 06873d31..fa56772a 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -115,7 +115,13 @@ void setup() { #ifdef DISPLAY_CLASS if (display_ready) { +#ifdef WITH_MQTT_BRIDGE + ui_task.setObserverPrefs(the_mesh.getObserverPrefs()); +#endif ui_task.begin(the_mesh.getNodePrefs(), FIRMWARE_BUILD_DATE, FIRMWARE_VERSION); +#ifdef DISPLAY_ACTIVITY_DASHBOARD + ui_task.setActivityWindow(the_mesh.getActivityWindow()); +#endif } #endif diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 02efbd43..a89dca23 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -246,6 +246,11 @@ void MyMesh::logRxRaw(float snr, float rssi, const uint8_t raw[], int len) { } void MyMesh::logRx(mesh::Packet *pkt, int len, float score) { +#ifdef DISPLAY_ACTIVITY_DASHBOARD + // Valid parsed RF packet: the only event the dashboard's window counts. + _activity.recordPacket(millis(), (uint16_t)len, _radio->getEstAirtimeFor(len), + (int8_t)(pkt->getSNR() * 4.0f), (int16_t)_radio->getLastRSSI()); +#endif #ifdef WITH_MQTT_BRIDGE // MQTT bridge: always feed RX packets — bridge decides based on mqtt.rx setting if (_prefs.bridge_enabled && bridge) bridge->onPacketReceived(pkt); diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 65de77ed..47f2f68d 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -15,6 +15,10 @@ #include #include #include +#ifdef DISPLAY_ACTIVITY_DASHBOARD +#include +#endif + #include #include #include @@ -117,6 +121,9 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint64_t uptime_millis; unsigned long next_local_advert, next_flood_advert; bool _logging; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow _activity; // rolling RF receive window, for the TFT dashboard +#endif bool region_load_active; NodePrefs _prefs; TransportKeyStore key_store; @@ -286,6 +293,14 @@ public: return &_prefs; } +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow* getActivityWindow() { return &_activity; } +#endif + +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* getObserverPrefs() { return _cli.getObserverPrefs(); } +#endif + void savePrefs() override { _cli.savePrefs(_fs); } diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 884ab528..3f124cb1 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -15,7 +15,31 @@ #include // defines WITH_WEBCONFIG on ESP32 #endif -#define AUTO_OFF_MILLIS 20000 // 20 seconds +#ifndef AUTO_OFF_MILLIS +#define AUTO_OFF_MILLIS 20000 // 20 seconds; 0 keeps the screen on +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE +#define TOUCH_POLL_MILLIS 50 +#endif + +// Wrap-safe deadline test. `millis() >= deadline` fires early for the whole +// interval before a rollover, because the deadline has already wrapped to a +// small value while millis() is still near UINT32_MAX; the signed difference +// stays correct across it. +static inline bool millisReached(unsigned long now, unsigned long deadline) { + return (int32_t)((uint32_t)now - (uint32_t)deadline) >= 0; +} + +// `display.timeout` when the observer prefs are available, otherwise the +// compiled-in default. Read on every use so a `set display.timeout` takes +// effect immediately. +unsigned long UITask::displayTimeoutMillis() const { +#ifdef WITH_MQTT_BRIDGE + if (_observer_prefs) return (unsigned long)_observer_prefs->display_timeout_secs * 1000UL; +#endif + return AUTO_OFF_MILLIS; +} #define BOOT_SCREEN_MILLIS 4000 // 4 seconds // 'meshcore', 128x13px @@ -37,9 +61,16 @@ static const uint8_t meshcore_logo [] PROGMEM = { void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version) { _prevBtnState = HIGH; - _auto_off = millis() + AUTO_OFF_MILLIS; + _timeout_seen = displayTimeoutMillis(); + _auto_off = millis() + displayTimeoutMillis(); _node_prefs = node_prefs; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + ObserverDashboard::applyDarkPalette(); // retunes UIColor for this target only +#endif _display->turnOn(); +#ifdef DISPLAY_TOUCH_TOGGLE + _touch.begin(); +#endif #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; #endif @@ -59,6 +90,9 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi void UITask::renderCurrScreen() { char tmp[80]; +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif if (millis() < BOOT_SCREEN_MILLIS) { // boot screen // meshcore logo _display->setColor(UIColor::corp_blue); @@ -121,6 +155,10 @@ void UITask::renderCurrScreen() { _display->print(wc_ip); return; } +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + renderDashboard(); + return; #endif // node name _display->setCursor(0, 0); @@ -181,7 +219,7 @@ uint32_t UITask::getFrameSignature() { snprintf(tmp, sizeof(tmp), "BW: %03.2f CR: %d", _node_prefs->bw, _node_prefs->cr); signature = DisplayFrameSignature::append(signature, tmp); -#ifdef WITH_MQTT_BRIDGE +#if defined(WITH_MQTT_BRIDGE) && !defined(DISPLAY_ACTIVITY_DASHBOARD) if (WiFi.status() == WL_CONNECTED) { IPAddress ip = WiFi.localIP(); snprintf(tmp, sizeof(tmp), "IP: %d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]); @@ -195,21 +233,96 @@ uint32_t UITask::getFrameSignature() { } #endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD +#define ACTIVITY_REFRESH_MILLIS 5000 + +bool UITask::buildDashboardContext(ObserverDashboard::Context* ctx) { + if (_node_prefs == NULL) return false; + ctx->node_name = _node_prefs->node_name; + ctx->role_label = "ROOM SERVER"; + ctx->freq = _node_prefs->freq; + ctx->sf = _node_prefs->sf; + ctx->bw = _node_prefs->bw; +#ifdef WITH_MQTT_BRIDGE + ctx->link_up = (WiFi.status() == WL_CONNECTED); +#else + ctx->link_up = false; +#endif + return true; +} + +void UITask::renderDashboard() { + ObserverDashboard::Context ctx; + if (!buildDashboardContext(&ctx)) return; + + RadioActivitySnapshot snap; + if (_activity) { + _activity->snapshot(millis(), &snap); + } else { + memset(&snap, 0, sizeof(snap)); + } + + const ObserverDashboard::Layout& layout = ObserverDashboard::activeLayout(); + ObserverDashboard::drawFull(*_display, layout, ctx, snap); + ObserverDashboard::allRowSignatures(layout, ctx, snap, _row_signatures); + _rows_valid = true; + _next_activity = millis() + ACTIVITY_REFRESH_MILLIS; +} + +// Repaints just the analytics rows whose contents moved. No startFrame(), so +// the header, the radio strip and the rest of the panel are never cleared. +void UITask::updateActivityRows() { + if (!_rows_valid || _activity == NULL) return; // not showing the dashboard + + ObserverDashboard::Context ctx; + if (!buildDashboardContext(&ctx)) return; + + RadioActivitySnapshot snap; + _activity->snapshot(millis(), &snap); + ObserverDashboard::drawChangedRows(*_display, ObserverDashboard::activeLayout(), ctx, snap, + _row_signatures); +} +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE +void UITask::toggleDisplay() { + if (_display->isOn()) { + _display->turnOff(); + } else { + _display->turnOn(); + } +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; // wake draws one complete current frame +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif + _auto_off = millis() + displayTimeoutMillis(); +} +#endif + void UITask::loop() { #ifdef PIN_USER_BTN if (millis() >= _next_read) { int btnState = digitalRead(PIN_USER_BTN); if (btnState != _prevBtnState) { if (btnState == USER_BTN_PRESSED) { // pressed? +#ifdef DISPLAY_TOUCH_TOGGLE + toggleDisplay(); // same action as tapping the panel +#else if (_display->isOn()) { // TODO: any action ? } else { _display->turnOn(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } - _auto_off = millis() + AUTO_OFF_MILLIS; // extend auto-off timer + _auto_off = millis() + displayTimeoutMillis(); // extend auto-off timer +#endif } _prevBtnState = btnState; } @@ -225,9 +338,22 @@ void UITask::loop() { _display->turnOn(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } - _auto_off = millis() + AUTO_OFF_MILLIS; + _auto_off = millis() + displayTimeoutMillis(); + } +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + { + unsigned long now = millis(); + if (millisReached(now, _next_touch)) { + _next_touch = now + TOUCH_POLL_MILLIS; + if (_touch.checkTap(now)) toggleDisplay(); + } } #endif @@ -247,13 +373,30 @@ void UITask::loop() { _frame_valid = true; #endif } +#ifdef DISPLAY_ACTIVITY_DASHBOARD + else if (millisReached(millis(), _next_activity)) { + updateActivityRows(); + _next_activity = millis() + ACTIVITY_REFRESH_MILLIS; + } +#endif _next_refresh = millis() + 1000; // check for visible changes every second } - if (millis() > _auto_off) { + // `_auto_off` is only armed on activity, so a timeout changed at runtime has + // to restart the countdown here - otherwise 0 -> 60 blanks instantly off a + // boot-time deadline, and 60 -> 3600 still blanks at the old 60 s mark. + unsigned long timeout = displayTimeoutMillis(); + if (timeout != _timeout_seen) { + _timeout_seen = timeout; + _auto_off = millis() + timeout; + } + if (timeout > 0 && millisReached(millis(), _auto_off)) { _display->turnOff(); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; #endif } } diff --git a/examples/simple_room_server/UITask.h b/examples/simple_room_server/UITask.h index 7ef88778..37005e18 100644 --- a/examples/simple_room_server/UITask.h +++ b/examples/simple_room_server/UITask.h @@ -3,6 +3,19 @@ #include #include +#ifdef DISPLAY_ACTIVITY_DASHBOARD + #ifndef DISPLAY_REDRAW_ON_CHANGE + #error "DISPLAY_ACTIVITY_DASHBOARD needs DISPLAY_REDRAW_ON_CHANGE: without it every frame clears the whole screen" + #endif + #include + #include + +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + #include +#endif + class UITask { DisplayDriver* _display; unsigned long _next_read, _next_refresh, _auto_off; @@ -17,10 +30,45 @@ class UITask { uint32_t getFrameSignature(); #endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + RadioActivityWindow* _activity = NULL; + unsigned long _next_activity = 0; + uint32_t _row_signatures[ObserverDashboard::ROW_COUNT] = {0}; + bool _rows_valid = false; // true only while the dashboard is the drawn screen + + bool buildDashboardContext(ObserverDashboard::Context* ctx); + void renderDashboard(); + void updateActivityRows(); +#endif + +#ifdef DISPLAY_TOUCH_TOGGLE + CHSC6XTouch _touch; + unsigned long _next_touch = 0; + + void toggleDisplay(); +#endif + +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs* _observer_prefs = NULL; +#endif + unsigned long _timeout_seen = 0; // to notice a live `display.timeout` change + + unsigned long displayTimeoutMillis() const; + void renderCurrScreen(); public: UITask(DisplayDriver& display) : _display(&display) { _next_read = _next_refresh = 0; } void begin(NodePrefs* node_prefs, const char* build_date, const char* firmware_version); +#ifdef WITH_MQTT_BRIDGE + // Supplies `display.timeout`, which is read live so a config change applies + // without a reboot. Call before begin(). + void setObserverPrefs(MQTTPrefs* prefs) { _observer_prefs = prefs; } +#endif + +#ifdef DISPLAY_ACTIVITY_DASHBOARD + void setActivityWindow(RadioActivityWindow* activity) { _activity = activity; } +#endif + void loop(); }; diff --git a/examples/simple_room_server/main.cpp b/examples/simple_room_server/main.cpp index 6b5f74a9..48f48799 100644 --- a/examples/simple_room_server/main.cpp +++ b/examples/simple_room_server/main.cpp @@ -91,7 +91,13 @@ void setup() { #ifdef DISPLAY_CLASS if (display_ready) { +#ifdef WITH_MQTT_BRIDGE + ui_task.setObserverPrefs(the_mesh.getObserverPrefs()); +#endif ui_task.begin(the_mesh.getNodePrefs(), FIRMWARE_BUILD_DATE, FIRMWARE_VERSION); +#ifdef DISPLAY_ACTIVITY_DASHBOARD + ui_task.setActivityWindow(the_mesh.getActivityWindow()); +#endif } #endif diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index c6fcb606..4cf66845 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -116,7 +116,8 @@ static bool isObserverPrefsSetCommand(const char* config) { strncmp(config, "mqtt", 4) == 0 || strncmp(config, "wifi.", 5) == 0 || strncmp(config, "timezone", 8) == 0 || - strncmp(config, "alert", 5) == 0; + strncmp(config, "alert", 5) == 0 || + strncmp(config, "display.", 8) == 0; } // Keep observer setters atomic from the caller's perspective. The live object @@ -265,6 +266,30 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } } } + } else if (memcmp(config, "display.timeout ", 16) == 0) { + const char* val = &config[16]; + bool all_digits = (*val != '\0'); + for (const char* sp = val; *sp; sp++) { + if (*sp < '0' || *sp > '9') { all_digits = false; break; } + } + if (*val == '\0') { + strcpy(reply, "Error: missing display.timeout seconds"); + } else if (!all_digits) { + sprintf(reply, "Error: display.timeout must be an integer 0-%d", DISPLAY_TIMEOUT_MAX_SECS); + } else { + long secs = atol(val); + if (secs > DISPLAY_TIMEOUT_MAX_SECS) { + sprintf(reply, "Error: display.timeout must be 0-%d seconds", DISPLAY_TIMEOUT_MAX_SECS); + } else { + _mqtt_prefs.display_timeout_secs = (uint16_t)secs; + if (!persistObserverPrefs(reply)) return true; + if (secs == 0) { + strcpy(reply, "OK - display stays on"); + } else { + sprintf(reply, "OK - display off after %ld s", secs); + } + } + } #ifdef WITH_MQTT_BRIDGE } else if (strcmp(config, "mqtt.origin") == 0) { _mqtt_prefs.mqtt_origin[0] = '\0'; @@ -860,6 +885,8 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, _mqtt_prefs.snmp_enabled ? "> on" : "> off"); } else if (memcmp(config, "radio.watchdog", 14) == 0) { sprintf(reply, "> %d", (uint32_t)_mqtt_prefs.radio_watchdog_minutes); + } else if (memcmp(config, "display.timeout", 15) == 0) { + sprintf(reply, "> %d", (uint32_t)_mqtt_prefs.display_timeout_secs); #ifdef WITH_MQTT_BRIDGE } else if (memcmp(config, "mqtt.origin", 11) == 0) { char effective_origin[32]; diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h index 21c53bc4..70140133 100644 --- a/src/helpers/MQTTDefaults.h +++ b/src/helpers/MQTTDefaults.h @@ -109,6 +109,8 @@ static inline void applyMQTTDefaults(MQTTPrefs* prefs) { // (not 0) so an in-lineage upgrade from a pre-neighbors payload is sane. prefs->mqtt_neighbors_enabled = 0; prefs->mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; + + prefs->display_timeout_secs = DISPLAY_TIMEOUT_DEFAULT_SECS; } #endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h index 4d182c44..42e56477 100644 --- a/src/helpers/MQTTPrefsSerializer.h +++ b/src/helpers/MQTTPrefsSerializer.h @@ -366,6 +366,24 @@ class MQTTPrefsSerializer : public ConfigSerializer { } }; + class DisplayPrefs : public ConfigSerializer { + MQTTPrefs* _prefs; + int32_t _timeout_s; + bool _seen_timeout = false; + protected: + void structure() override { defStrict("timeout_s", _timeout_s, _seen_timeout); } + public: + explicit DisplayPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _timeout_s(prefs->display_timeout_secs) {} + void apply(bool* repaired) { + if (_timeout_s < 0 || _timeout_s > DISPLAY_TIMEOUT_MAX_SECS) { + _timeout_s = DISPLAY_TIMEOUT_DEFAULT_SECS; + *repaired = true; + } + _prefs->display_timeout_secs = static_cast(_timeout_s); + } + }; + MQTTPrefs* _prefs; int32_t _version = MQTT_PREFS_JSON_FORMAT_VERSION; bool _seen_version = false; @@ -375,6 +393,7 @@ class MQTTPrefsSerializer : public ConfigSerializer { SnmpPrefs _snmp; RadioPrefs _radio; AlertPrefs _alert; + DisplayPrefs _display; protected: void structure() override { @@ -385,6 +404,7 @@ protected: def("snmp", _snmp); def("radio", _radio); def("alert", _alert); + def("display", _display); } public: @@ -392,7 +412,7 @@ public: : _prefs(prefs), _wifi(prefs), _time(prefs, repair_defaults ? repair_defaults : prefs), _mqtt(prefs, repair_defaults ? repair_defaults : prefs), _snmp(prefs), - _radio(prefs), _alert(prefs) {} + _radio(prefs), _alert(prefs), _display(prefs) {} bool hasSupportedVersion() const { return _seen_version && _version == MQTT_PREFS_JSON_FORMAT_VERSION; @@ -410,6 +430,7 @@ public: _snmp.apply(repaired); _radio.apply(repaired); _alert.apply(repaired); + _display.apply(repaired); return true; } diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h index 0bc0867e..1ee4a99f 100644 --- a/src/helpers/MQTTPrefsStorage.h +++ b/src/helpers/MQTTPrefsStorage.h @@ -117,8 +117,16 @@ struct MQTTPrefs { // Per-slot payload-type allow masks. Bit N controls MeshCore packet type N // for both packets and raw MQTT topics. uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; + + // Seconds of inactivity before the display blanks; 0 keeps it lit. Runtime + // only - deliberately absent from LegacyV1MQTTPrefs, so the frozen binary + // layout and its four payload sizes are unchanged. + uint16_t display_timeout_secs; }; +static const uint16_t DISPLAY_TIMEOUT_DEFAULT_SECS = 60; +static const uint16_t DISPLAY_TIMEOUT_MAX_SECS = 3600; + // Frozen payload written by the version-1 binary format. Keep this distinct // from the runtime MQTTPrefs type: JSON persistence must not make the runtime // object's padding or member order an on-flash ABI again. Legacy decoding reads diff --git a/src/helpers/RadioActivityWindow.h b/src/helpers/RadioActivityWindow.h new file mode 100644 index 00000000..36032b75 --- /dev/null +++ b/src/helpers/RadioActivityWindow.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include + +// Fixed-memory rolling window of RF receive activity, bucketed by minute. +// +// Pure logic: no Arduino, radio, display or role headers. Callers supply a +// millisecond counter and the per-packet values. +// +// The caller's 32-bit millis() is extended to a monotonic 64-bit clock on entry +// (see tick()), so nothing downstream has a rollover case. Unsigned-subtraction +// tricks are not enough here: they only survive a single wrap crossing, while an +// always-on observer accumulates uptime past 2^32 ms (~49.7 days), at which +// point a 32-bit tracker age would collapse back to a small value and the +// window would re-enter warm-up and divide 20 minutes of traffic by seconds. + +#define RADIO_ACTIVITY_BUCKETS 20 +#define RADIO_ACTIVITY_BUCKET_MS 60000UL + +// Beyond this, "time since last packet" stops being reported rather than shown +// as a stale or (after a 49-day rollover) nonsensical age. +#define RADIO_ACTIVITY_MAX_AGE_MS (100UL * RADIO_ACTIVITY_BUCKET_MS) + +struct RadioActivitySnapshot { + uint32_t packets; + uint32_t wire_bytes; + uint32_t airtime_ms; + int32_t snr_q4_sum; + int32_t rssi_sum; + + // Span the totals actually cover: 19 whole minutes plus the elapsed part of + // the current one, so it tops out just under 20 minutes and never claims + // coverage the ring does not have. + uint32_t window_ms; + uint32_t tracking_ms; // how long the tracker has been running + + uint32_t last_packet_age_ms; + bool has_last_packet; // false until the first packet, and once stale + + uint16_t peak_per_min; + uint16_t buckets[RADIO_ACTIVITY_BUCKETS]; // [0] oldest .. [N-1] current minute + + bool isEmpty() const { return packets == 0; } + + bool isWarmingUp() const { + return tracking_ms < (uint32_t)RADIO_ACTIVITY_BUCKETS * RADIO_ACTIVITY_BUCKET_MS; + } + uint32_t warmupMinutes() const { return tracking_ms / RADIO_ACTIVITY_BUCKET_MS; } + + // Derived values, in integer fixed point so host tests are exact and the + // formatting path stays off the FPU. All are zero when the window is empty. + uint32_t packetsPerMinuteX10() const { + if (window_ms == 0) return 0; + return (uint32_t)(((uint64_t)packets * RADIO_ACTIVITY_BUCKET_MS * 10) / window_ms); + } + uint32_t bytesPerSecondX10() const { + if (window_ms == 0) return 0; + return (uint32_t)(((uint64_t)wire_bytes * 1000 * 10) / window_ms); + } + uint32_t avgBytesPerPacket() const { + if (packets == 0) return 0; + return (wire_bytes + packets / 2) / packets; + } + // Receive airtime as tenths of a percent of the window. + uint32_t airtimePercentX10() const { + if (window_ms == 0) return 0; + return (uint32_t)(((uint64_t)airtime_ms * 1000) / window_ms); + } + // Average SNR in tenths of a dB (sums are quarter-dB units). + int32_t avgSnrX10() const { + if (packets == 0) return 0; + return (snr_q4_sum * 10) / ((int32_t)packets * 4); + } + int32_t avgRssi() const { + if (packets == 0) return 0; + return rssi_sum / (int32_t)packets; + } +}; + +class RadioActivityWindow { +public: + RadioActivityWindow() { reset(0); } + + void reset(uint32_t now_ms) { + memset(_buckets, 0, sizeof(_buckets)); + _head = 0; + _now_ms = 0; + _last_input_ms = now_ms; + _bucket_start_ms = 0; + _tracking_since_ms = 0; + _last_packet_ms = 0; + _ever_received = false; + } + + void recordPacket(uint32_t now_ms, uint16_t wire_bytes, uint32_t airtime_ms, int8_t snr_q4, + int16_t rssi_dbm) { + advance(now_ms); + + Bucket& b = _buckets[_head]; + // Saturated: drop the event whole so the bucket's averages stay consistent + // with its packet count. Unreachable at any real LoRa packet rate. + if (b.packets == 0xFFFF) return; + + b.packets++; + b.wire_bytes += wire_bytes; + b.airtime_ms += airtime_ms; + b.snr_q4_sum += snr_q4; + b.rssi_sum += rssi_dbm; + + _last_packet_ms = _now_ms; + _ever_received = true; + } + + void snapshot(uint32_t now_ms, RadioActivitySnapshot* out) { + advance(now_ms); + memset(out, 0, sizeof(*out)); + + for (int i = 0; i < RADIO_ACTIVITY_BUCKETS; i++) { + const Bucket& b = _buckets[(_head + 1 + i) % RADIO_ACTIVITY_BUCKETS]; + out->buckets[i] = b.packets; + out->packets += b.packets; + out->wire_bytes += b.wire_bytes; + out->airtime_ms += b.airtime_ms; + out->snr_q4_sum += b.snr_q4_sum; + out->rssi_sum += b.rssi_sum; + if (b.packets > out->peak_per_min) out->peak_per_min = b.packets; + } + + uint64_t elapsed_in_current = _now_ms - _bucket_start_ms; // < BUCKET_MS after advance() + uint64_t max_span = + (uint64_t)(RADIO_ACTIVITY_BUCKETS - 1) * RADIO_ACTIVITY_BUCKET_MS + elapsed_in_current; + uint64_t tracking = _now_ms - _tracking_since_ms; + const uint64_t full_span = (uint64_t)RADIO_ACTIVITY_BUCKETS * RADIO_ACTIVITY_BUCKET_MS; + + // Clamped, so the 32-bit snapshot fields stay in range on a long-lived node. + // Past full_span the exact tracker age is not needed: the window is warm. + out->tracking_ms = (uint32_t)(tracking < full_span ? tracking : full_span); + out->window_ms = (uint32_t)(tracking < max_span ? tracking : max_span); + + if (_ever_received) { + uint64_t age = _now_ms - _last_packet_ms; + if (age <= RADIO_ACTIVITY_MAX_AGE_MS) { + out->last_packet_age_ms = (uint32_t)age; + out->has_last_packet = true; + } + } + } + +private: + struct Bucket { + uint32_t wire_bytes; + uint32_t airtime_ms; + int32_t snr_q4_sum; + int32_t rssi_sum; + uint16_t packets; + uint16_t _reserved; + }; + + Bucket _buckets[RADIO_ACTIVITY_BUCKETS]; + uint64_t _now_ms; // monotonic clock, extended from the caller's + uint64_t _bucket_start_ms; // start of the current (newest) minute + uint64_t _tracking_since_ms; + uint64_t _last_packet_ms; + uint32_t _last_input_ms; // last 32-bit value the caller handed in + uint8_t _head; // ring index of the current minute + bool _ever_received; + + // Accumulates the delta since the previous call, which is correct across one + // millis() wrap. Read as signed so a caller handing back a slightly older + // reading counts as no time passing, rather than as a ~49-day leap forward + // that would expire the whole ring. Successive calls must therefore be less + // than 2^31 ms (~24.8 days) apart - guaranteed while the tracker is being + // serviced, and when it is not the ring is empty anyway. + void tick(uint32_t now_ms) { + int32_t delta = (int32_t)(now_ms - _last_input_ms); + if (delta <= 0) return; // stale or repeated reading: no time has passed + _now_ms += (uint32_t)delta; + _last_input_ms = now_ms; + } + + // Retires expired buckets lazily, advancing the boundary by whole BUCKET_MS + // steps so the minute phase is preserved across gaps. + void advance(uint32_t now_ms) { + tick(now_ms); + + uint64_t elapsed = _now_ms - _bucket_start_ms; + if (elapsed < RADIO_ACTIVITY_BUCKET_MS) return; + + uint64_t steps = elapsed / RADIO_ACTIVITY_BUCKET_MS; + _bucket_start_ms += steps * RADIO_ACTIVITY_BUCKET_MS; + + if (steps >= RADIO_ACTIVITY_BUCKETS) { + memset(_buckets, 0, sizeof(_buckets)); + _head = 0; + _tracking_since_ms = _bucket_start_ms; + return; + } + + for (uint64_t i = 0; i < steps; i++) { + _head = (uint8_t)((_head + 1) % RADIO_ACTIVITY_BUCKETS); + memset(&_buckets[_head], 0, sizeof(Bucket)); + } + } +}; + +static_assert(sizeof(RadioActivityWindow) <= 1024, "RadioActivityWindow must stay under 1 KiB"); diff --git a/src/helpers/ui/CHSC6XTouch.h b/src/helpers/ui/CHSC6XTouch.h new file mode 100644 index 00000000..c4c5cd2e --- /dev/null +++ b/src/helpers/ui/CHSC6XTouch.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include + +#include "TouchTapDetector.h" + +// Minimal polled driver for the CHSC6X capacitive touch controller on the +// Heltec V4 R8 Expansion Kit V2 panel. +// +// Only "is a finger down" is needed to toggle the display, so no coordinates +// and no calibration are read. TP_INT is not used: on this board it is an +// optional link (R13) on GPIO 43, which is also U0TXD - see HeltecV4R8Board.cpp +// for the verified pin map. + +#ifndef CHSC6X_I2C_ADDR +#define CHSC6X_I2C_ADDR 0x2E +#endif + +#define CHSC6X_READ_LEN 5 +#define CHSC6X_MAX_POINTS 1 + +class CHSC6XTouch { +public: + // Probes the bus. Returns false (and disables itself) when nothing answers, + // so a board without the touch panel simply carries on without it. + bool begin(TwoWire& wire = Wire) { + _wire = &wire; + _wire->beginTransmission((uint8_t)CHSC6X_I2C_ADDR); + _present = (_wire->endTransmission() == 0); + _detector.reset(millis()); + + #if defined(DISPLAY_TOUCH_DEBUG) && defined(PIN_TOUCH_INT) + pinMode(PIN_TOUCH_INT, INPUT_PULLUP); + #endif + + if (_present) { + Serial.printf("Touch: CHSC6X found at 0x%02X\n", CHSC6X_I2C_ADDR); + } else { + // Report what is actually on the bus, so an unexpected controller or + // address can be identified from a normal boot log. + Serial.printf("Touch: nothing at 0x%02X; I2C bus holds:", CHSC6X_I2C_ADDR); + for (uint8_t addr = 8; addr < 0x78; addr++) { + _wire->beginTransmission(addr); + if (_wire->endTransmission() == 0) Serial.printf(" 0x%02X", addr); + } + Serial.println(); + } + return _present; + } + + bool isPresent() const { return _present; } + + // True exactly once per new touch. + bool checkTap(uint32_t now_ms) { + if (!_present) return false; + return _detector.update(now_ms, readPressed()); + } + +private: + TwoWire* _wire = NULL; + bool _present = false; + TouchTapDetector _detector; + + bool readPressed() { + uint8_t got = _wire->requestFrom((uint8_t)CHSC6X_I2C_ADDR, (uint8_t)CHSC6X_READ_LEN); + if (got != CHSC6X_READ_LEN) { + while (_wire->available()) _wire->read(); // drain a short read + logRaw(got, NULL); + return false; + } + + uint8_t buf[CHSC6X_READ_LEN]; + for (uint8_t i = 0; i < CHSC6X_READ_LEN; i++) buf[i] = (uint8_t)_wire->read(); + logRaw(got, buf); + + // buf[0] is the reported touch-point count (buf[2]/buf[4] are x/y). It must + // be tested against a *valid* count, not merely against zero: an idle or + // NACKed read can come back as 0xFF, which "non-zero" reads as a finger + // held down forever - the tap detector then fires once and, seeing no + // release, never fires again. + return buf[0] >= 1 && buf[0] <= CHSC6X_MAX_POINTS; + } + +#ifdef DISPLAY_TOUCH_DEBUG + int16_t _logged = -1; + + // Logs on change only, so a normal boot stays quiet. + void logRaw(uint8_t got, const uint8_t* buf) { + int16_t key = buf ? (int16_t)buf[0] : (int16_t)(-2 - (int16_t)got); + if (key == _logged) return; + _logged = key; + + if (!buf) { + Serial.printf("Touch: short read (%u of %u bytes)\n", got, CHSC6X_READ_LEN); + return; + } + Serial.printf("Touch: raw %02X %02X %02X %02X %02X", buf[0], buf[1], buf[2], buf[3], buf[4]); + #ifdef PIN_TOUCH_INT + // Pulled up, so an unfitted R13 sits steady HIGH and a wired INT pulses LOW. + Serial.printf(" INT=%d", digitalRead(PIN_TOUCH_INT)); + #endif + Serial.println(); + } +#else + void logRaw(uint8_t, const uint8_t*) {} +#endif +}; diff --git a/src/helpers/ui/ObserverDashboard.h b/src/helpers/ui/ObserverDashboard.h new file mode 100644 index 00000000..c8bacd84 --- /dev/null +++ b/src/helpers/ui/ObserverDashboard.h @@ -0,0 +1,524 @@ +#pragma once + +#include +#include +#include + +#include "../RadioActivityWindow.h" +#include "DisplayDriver.h" +#include "DisplayFrameSignature.h" + +// Observer analytics dashboard for the Heltec V4 R8 TFT targets. +// +// Fork-owned and self-contained: the role UITasks only pick a layout, hand over +// a Context plus a snapshot, and ask for a full frame or a single changed row. +// Everything here is host-buildable against DisplayDriver, so the layout, the +// formatting and the redraw policy are all covered by test_observer_dashboard. + +namespace ObserverDashboard { + +// ---------------------------------------------------------------- palette --- +// Kept local to the dashboard. applyDarkPalette() retunes the shared UIColor +// slots at runtime, which is why no display driver needs editing for the theme. + +constexpr ColorVal rgb565(uint8_t r, uint8_t g, uint8_t b) { + return (ColorVal)(((r & 0xF8) << 8) | ((g & 0xFC) << 3) | (b >> 3)); +} + +constexpr ColorVal BG = rgb565(10, 12, 16); +constexpr ColorVal HEADER_BG = rgb565(18, 38, 74); +constexpr ColorVal HEADER_SUB = rgb565(130, 170, 214); +constexpr ColorVal TEXT = rgb565(232, 238, 246); +constexpr ColorVal MUTED = rgb565(122, 134, 150); +constexpr ColorVal ACCENT = rgb565(64, 176, 240); +constexpr ColorVal BAR = rgb565(38, 116, 168); +constexpr ColorVal BAR_NOW = rgb565(96, 208, 255); +constexpr ColorVal GRID = rgb565(38, 46, 60); +constexpr ColorVal GOOD = rgb565(72, 208, 136); +constexpr ColorVal WARN = rgb565(248, 176, 72); + +// Retunes the shared colour slots so every screen this UITask draws - boot, +// setup portal, reboot, power-off and the dashboard - is coherently dark. +inline void applyDarkPalette() { + UIColor::window_bkg = BG; + UIColor::title_bkg = HEADER_BG; + UIColor::title_txt = TEXT; + UIColor::primary_txt = TEXT; + UIColor::secondary_txt = MUTED; + UIColor::warning_txt = WARN; + UIColor::popup_bkg = HEADER_BG; + UIColor::popup_txt = TEXT; + UIColor::corp_blue = ACCENT; +} + +// ----------------------------------------------------------------- layout --- +// Logical 128x64 coordinates, as the shared DisplayDriver API expects. The two +// orientations need separate row pitches: a text row is a fixed 16 physical +// pixels, which is 3.2 logical units in portrait (y scale 5) but 4.27 in +// landscape (y scale 3.75), so one shared grid would overlap in landscape. + +struct Layout { + int16_t margin_x; + int16_t right_x; // right edge of the content column (exclusive) + int16_t header_h; + int16_t header_text_y; + int16_t header_sub_y; // second header row for the role label; -1 = same row + int16_t radio_y; + int16_t window_y; + int16_t headline_y; + int16_t headline_h; + int16_t rate_y; + int16_t graph_y; + int16_t graph_h; // includes the one-unit baseline at the bottom + int16_t rf_y; + int16_t status_y; + int16_t text_h; // logical height of one size-1 text row + int16_t max_chars; // size-1 characters that fit between the margins + int16_t max_chars_big; // size-2 characters that fit +}; + +// 240x320 panel: x scale 1.875, y scale 5, 12x16 glyphs (24x32 at size 2). +constexpr Layout portraitLayout() { + return Layout{ + /*margin_x*/ 4, /*right_x*/ 124, + /*header_h*/ 9, /*header_text_y*/ 1, /*header_sub_y*/ 5, + /*radio_y*/ 11, + /*window_y*/ 16, + /*headline_y*/ 21, /*headline_h*/ 7, + /*rate_y*/ 29, + /*graph_y*/ 34, /*graph_h*/ 16, + /*rf_y*/ 52, + /*status_y*/ 57, + /*text_h*/ 4, + /*max_chars*/ 18, /*max_chars_big*/ 9}; +} + +// 320x240 panel: x scale 2.5, y scale 3.75, 12x16 glyphs (30x40 at size 2). +constexpr Layout landscapeLayout() { + return Layout{ + /*margin_x*/ 4, /*right_x*/ 124, + /*header_h*/ 7, /*header_text_y*/ 1, /*header_sub_y*/ -1, + /*radio_y*/ 9, + /*window_y*/ 14, + /*headline_y*/ 19, /*headline_h*/ 12, + /*rate_y*/ 31, + /*graph_y*/ 36, /*graph_h*/ 12, + /*rf_y*/ 50, + /*status_y*/ 56, + /*text_h*/ 5, + /*max_chars*/ 25, /*max_chars_big*/ 10}; +} + +inline const Layout& activeLayout() { +#ifdef ST7789_PORTRAIT_PROFILE + static const Layout layout = portraitLayout(); +#else + static const Layout layout = landscapeLayout(); +#endif + return layout; +} + +// ------------------------------------------------------------- formatting --- +// Every value is formatted from integers. Arguments are widened explicitly so +// the same code is correct on a 32-bit target and on a 64-bit host. + +inline void formatCompactCount(char* out, size_t n, uint32_t v) { + if (v < 10000) { + snprintf(out, n, "%lu", (unsigned long)v); + } else if (v < 100000) { + snprintf(out, n, "%lu.%luk", (unsigned long)(v / 1000), (unsigned long)((v % 1000) / 100)); + } else if (v < 1000000) { + snprintf(out, n, "%luk", (unsigned long)(v / 1000)); + } else if (v < 100000000) { + snprintf(out, n, "%lu.%luM", (unsigned long)(v / 1000000), + (unsigned long)((v % 1000000) / 100000)); + } else { + snprintf(out, n, "%luM", (unsigned long)(v / 1000000)); + } +} + +inline void formatCompactBytes(char* out, size_t n, uint32_t v) { + if (v < 1024) { + snprintf(out, n, "%lu B", (unsigned long)v); + return; + } + if (v < 1048576UL) { + uint32_t tenths = (uint32_t)(((uint64_t)v * 10) / 1024); + if (tenths < 1000) { + snprintf(out, n, "%lu.%lu KB", (unsigned long)(tenths / 10), (unsigned long)(tenths % 10)); + } else { + snprintf(out, n, "%lu KB", (unsigned long)(tenths / 10)); + } + return; + } + uint32_t tenths = (uint32_t)(((uint64_t)v * 10) / 1048576UL); + if (tenths < 1000) { + snprintf(out, n, "%lu.%lu MB", (unsigned long)(tenths / 10), (unsigned long)(tenths % 10)); + } else { + snprintf(out, n, "%lu MB", (unsigned long)(tenths / 10)); + } +} + +// One decimal below 100, whole numbers above, so the field cannot grow wide. +inline void formatTenths(char* out, size_t n, uint32_t tenths) { + if (tenths < 1000) { + snprintf(out, n, "%lu.%lu", (unsigned long)(tenths / 10), (unsigned long)(tenths % 10)); + } else { + formatCompactCount(out, n, tenths / 10); + } +} + +inline void formatSignedTenths(char* out, size_t n, int32_t tenths) { + const char* sign = tenths < 0 ? "-" : "+"; + uint32_t mag = (uint32_t)(tenths < 0 ? -tenths : tenths); + snprintf(out, n, "%s%lu.%lu", sign, (unsigned long)(mag / 10), (unsigned long)(mag % 10)); +} + +// Quantised to the 5 s activity cadence so the string is stable within a tick +// and cannot make the status row repaint more often than the panel updates. +inline uint32_t quantizeAgeSecs(uint32_t age_ms) { return (age_ms / 5000) * 5; } + +inline void formatAge(char* out, size_t n, uint32_t age_ms, bool valid) { + if (!valid) { + snprintf(out, n, "--"); + return; + } + uint32_t secs = quantizeAgeSecs(age_ms); + if (secs < 5) { + snprintf(out, n, "now"); + } else if (secs < 60) { + snprintf(out, n, "%lus", (unsigned long)secs); + } else if (secs < 3600) { + snprintf(out, n, "%lum", (unsigned long)(secs / 60)); + } else if (secs < 86400) { + snprintf(out, n, "%luh", (unsigned long)(secs / 3600)); + } else { + snprintf(out, n, "%lud", (unsigned long)(secs / 86400)); + } +} + +// "910.525 SF7 BW62.5" / "869.618 SF8 BW250" - a trailing ".0" on the +// bandwidth would push the widest case past the content column. +inline void formatRadioStrip(char* out, size_t n, float freq, uint8_t sf, float bw) { + int32_t tenths = (int32_t)(bw * 10.0f + 0.5f); + char bw_str[12]; + if (tenths % 10 == 0) { + snprintf(bw_str, sizeof(bw_str), "%ld", (long)(tenths / 10)); + } else { + snprintf(bw_str, sizeof(bw_str), "%ld.%ld", (long)(tenths / 10), (long)(tenths % 10)); + } + snprintf(out, n, "%.3f SF%u BW%s", (double)freq, (unsigned)sf, bw_str); +} + +// Trims to a character budget rather than a measured width. The font is fixed +// width, so the budget is exact - and DisplayDriver::drawTextEllipsized() must +// not be used here: it trims against getTextWidth(), which reports an +// over-long string at the portrait driver's *fallback* scale and so stops +// trimming while the string is still too wide to draw at full size. +inline void fitToChars(char* out, size_t n, const char* src, int max_chars) { + if (max_chars < 0) max_chars = 0; + if ((size_t)max_chars > n - 1) max_chars = (int)(n - 1); + + size_t len = src ? strlen(src) : 0; + if (len <= (size_t)max_chars) { + memcpy(out, src ? src : "", len); + out[len] = 0; + return; + } + if (max_chars <= 3) { + memcpy(out, src, (size_t)max_chars); + out[max_chars] = 0; + return; + } + memcpy(out, src, (size_t)max_chars - 3); + memcpy(out + max_chars - 3, "...", 4); +} + +// ------------------------------------------------------------- row content -- + +enum Row { ROW_WINDOW = 0, ROW_HEADLINE, ROW_RATE, ROW_GRAPH, ROW_RF, ROW_STATUS, ROW_COUNT }; + +struct Context { + const char* node_name; + const char* role_label; // "REPEATER" / "ROOM SERVER" + float freq; + uint8_t sf; + float bw; + bool link_up; +}; + +struct RowText { + char left[24]; + char right[24]; + ColorVal left_color; + ColorVal right_color; +}; + +inline void composeRow(Row row, const Context& ctx, const RadioActivitySnapshot& s, RowText* out) { + out->left[0] = out->right[0] = 0; + out->left_color = TEXT; + out->right_color = MUTED; + + char scratch[24]; + switch (row) { + case ROW_WINDOW: + if (s.isWarmingUp()) { + snprintf(out->left, sizeof(out->left), "LIVE %lum", (unsigned long)s.warmupMinutes()); + } else { + snprintf(out->left, sizeof(out->left), "LAST 20m"); + } + out->left_color = MUTED; + if (s.peak_per_min > 0) { + formatCompactCount(scratch, sizeof(scratch), s.peak_per_min); + snprintf(out->right, sizeof(out->right), "max %s/m", scratch); + } + break; + + case ROW_HEADLINE: + if (s.isEmpty()) { + snprintf(out->left, sizeof(out->left), "No RF yet"); + out->left_color = MUTED; + } else { + formatCompactCount(scratch, sizeof(scratch), s.packets); + snprintf(out->left, sizeof(out->left), "%s pkt", scratch); + out->left_color = BAR_NOW; + } + break; + + case ROW_RATE: + formatCompactBytes(out->left, sizeof(out->left), s.wire_bytes); + formatTenths(scratch, sizeof(scratch), s.packetsPerMinuteX10()); + snprintf(out->right, sizeof(out->right), "%s/min", scratch); + if (s.isEmpty()) { + out->left_color = MUTED; + } + break; + + case ROW_RF: + if (s.isEmpty()) { + snprintf(out->left, sizeof(out->left), "SNR --"); + out->left_color = MUTED; + } else { + int32_t snr = s.avgSnrX10(); + formatSignedTenths(scratch, sizeof(scratch), snr); + snprintf(out->left, sizeof(out->left), "SNR %s", scratch); + out->left_color = snr >= 0 ? GOOD : (snr >= -70 ? TEXT : WARN); + } + { + uint32_t air = s.airtimePercentX10(); + formatTenths(scratch, sizeof(scratch), air); + snprintf(out->right, sizeof(out->right), "AIR %s%%", scratch); + out->right_color = air >= 100 ? WARN : MUTED; + } + break; + + case ROW_STATUS: + formatAge(scratch, sizeof(scratch), s.last_packet_age_ms, s.has_last_packet); + snprintf(out->left, sizeof(out->left), "RX %s", scratch); + out->left_color = s.has_last_packet ? TEXT : MUTED; + snprintf(out->right, sizeof(out->right), ctx.link_up ? "WiFi OK" : "WiFi --"); + out->right_color = ctx.link_up ? GOOD : WARN; + break; + + default: + break; + } +} + +inline int16_t rowY(const Layout& l, Row row) { + switch (row) { + case ROW_WINDOW: return l.window_y; + case ROW_HEADLINE: return l.headline_y; + case ROW_RATE: return l.rate_y; + case ROW_GRAPH: return l.graph_y; + case ROW_RF: return l.rf_y; + default: return l.status_y; + } +} + +inline int16_t rowH(const Layout& l, Row row) { + if (row == ROW_HEADLINE) return l.headline_h; + if (row == ROW_GRAPH) return l.graph_h; + return l.text_h; +} + +// --------------------------------------------------------------- the graph -- + +// Bar heights in logical units, oldest first. Any minute with traffic rounds up +// to at least one unit; empty minutes stay empty. +inline void barHeights(const Layout& l, const RadioActivitySnapshot& s, + uint8_t out[RADIO_ACTIVITY_BUCKETS]) { + uint16_t scale = s.peak_per_min > 0 ? s.peak_per_min : 1; + int16_t max_h = l.graph_h - 1; // the last unit is the baseline + for (int i = 0; i < RADIO_ACTIVITY_BUCKETS; i++) { + uint32_t v = s.buckets[i]; + out[i] = v == 0 ? 0 : (uint8_t)((v * max_h + scale - 1) / scale); + } +} + +inline int16_t barSlot(const Layout& l) { + return (int16_t)((l.right_x - l.margin_x) / RADIO_ACTIVITY_BUCKETS); +} + +// ------------------------------------------------------------- signatures --- +// One signature per row, computed from exactly what is drawn, so a repaint +// happens only where the pixels actually differ. + +inline uint32_t rowSignature(const Layout& l, Row row, const Context& ctx, + const RadioActivitySnapshot& s) { + uint32_t sig = DisplayFrameSignature::INITIAL; + if (row == ROW_GRAPH) { + uint8_t h[RADIO_ACTIVITY_BUCKETS]; + barHeights(l, s, h); + char buf[4 * RADIO_ACTIVITY_BUCKETS]; + size_t p = 0; + for (int i = 0; i < RADIO_ACTIVITY_BUCKETS && p + 4 < sizeof(buf); i++) { + p += (size_t)snprintf(buf + p, sizeof(buf) - p, "%u,", (unsigned)h[i]); + } + return DisplayFrameSignature::append(sig, buf); + } + + RowText t; + composeRow(row, ctx, s, &t); + sig = DisplayFrameSignature::append(sig, t.left); + return DisplayFrameSignature::append(sig, t.right); +} + +inline void allRowSignatures(const Layout& l, const Context& ctx, const RadioActivitySnapshot& s, + uint32_t out[ROW_COUNT]) { + for (int r = 0; r < ROW_COUNT; r++) out[r] = rowSignature(l, (Row)r, ctx, s); +} + +// ------------------------------------------------------------------ render -- + +// Right-hand text is measured and placed first, then the left text is +// ellipsized into whatever column is left, so the two can never collide. +// +// The right edge gets a one-unit gutter: getTextWidth() converts physical +// glyph widths back to logical units and rounds, so anchoring flush at right_x +// can land a pixel past it. +inline void drawPair(DisplayDriver& d, const Layout& l, int16_t y, const RowText& t, + int max_chars) { + int right_len = (int)strlen(t.right); + if (right_len > 0) { + int16_t rw = (int16_t)d.getTextWidth(t.right); + d.setColor(t.right_color); + d.setCursor((int16_t)(l.right_x - rw - 1), y); + d.print(t.right); + } + + int budget = max_chars - (right_len > 0 ? right_len + 1 : 0); + if (t.left[0] && budget > 0) { + char fitted[32]; + fitToChars(fitted, sizeof(fitted), t.left, budget); + d.setColor(t.left_color); + d.setCursor(l.margin_x, y); + d.print(fitted); + } +} + +inline void clearRow(DisplayDriver& d, const Layout& l, Row row) { + d.setColor(BG); + d.fillRect(l.margin_x, rowY(l, row), l.right_x - l.margin_x, rowH(l, row)); +} + +inline void drawGraph(DisplayDriver& d, const Layout& l, const RadioActivitySnapshot& s) { + int16_t base_y = (int16_t)(l.graph_y + l.graph_h - 1); + d.setColor(GRID); + d.fillRect(l.margin_x, base_y, l.right_x - l.margin_x, 1); + + uint8_t h[RADIO_ACTIVITY_BUCKETS]; + barHeights(l, s, h); + int16_t slot = barSlot(l); + for (int i = 0; i < RADIO_ACTIVITY_BUCKETS; i++) { + if (h[i] == 0) continue; + d.setColor(i == RADIO_ACTIVITY_BUCKETS - 1 ? BAR_NOW : BAR); + d.fillRect((int16_t)(l.margin_x + i * slot), (int16_t)(base_y - h[i]), (int16_t)(slot - 1), + h[i]); + } +} + +inline void drawRow(DisplayDriver& d, const Layout& l, Row row, const Context& ctx, + const RadioActivitySnapshot& s, bool clear_first) { + if (clear_first) clearRow(d, l, row); + + if (row == ROW_GRAPH) { + drawGraph(d, l, s); + return; + } + + RowText t; + composeRow(row, ctx, s, &t); + d.setTextSize(row == ROW_HEADLINE ? 2 : 1); + drawPair(d, l, rowY(l, row), t, row == ROW_HEADLINE ? l.max_chars_big : l.max_chars); + if (row == ROW_HEADLINE) d.setTextSize(1); +} + +inline void drawHeader(DisplayDriver& d, const Layout& l, const Context& ctx) { + d.setColor(HEADER_BG); + d.fillRect(0, 0, 128, l.header_h); + d.setTextSize(1); + + RowText t{}; + t.left_color = TEXT; + t.right_color = HEADER_SUB; + snprintf(t.left, sizeof(t.left), "%s", ctx.node_name ? ctx.node_name : ""); + snprintf(t.right, sizeof(t.right), "%s", ctx.role_label ? ctx.role_label : ""); + + if (l.header_sub_y < 0) { + drawPair(d, l, l.header_text_y, t, l.max_chars); // both fit on one line + return; + } + + // Narrow panel: the node name keeps the whole width and the role drops to a + // second line, rather than the name being ellipsized down to a few letters. + RowText name{}; + name.left_color = t.left_color; + memcpy(name.left, t.left, sizeof(name.left)); + drawPair(d, l, l.header_text_y, name, l.max_chars); + + RowText role{}; + role.left_color = t.right_color; + memcpy(role.left, t.right, sizeof(role.left)); + drawPair(d, l, l.header_sub_y, role, l.max_chars); +} + +inline void drawRadioStrip(DisplayDriver& d, const Layout& l, const Context& ctx) { + char tmp[32]; + formatRadioStrip(tmp, sizeof(tmp), ctx.freq, ctx.sf, ctx.bw); + char fitted[32]; + fitToChars(fitted, sizeof(fitted), tmp, l.max_chars); + d.setTextSize(1); + d.setColor(MUTED); + d.setCursor(l.margin_x, l.radio_y); + d.print(fitted); +} + +// Complete repaint. The caller has already run startFrame(), which clears to +// UIColor::window_bkg - the dark background applyDarkPalette() installed. +inline void drawFull(DisplayDriver& d, const Layout& l, const Context& ctx, + const RadioActivitySnapshot& s) { + drawHeader(d, l, ctx); + drawRadioStrip(d, l, ctx); + for (int r = 0; r < ROW_COUNT; r++) drawRow(d, l, (Row)r, ctx, s, false); +} + +// Repaints only the rows whose signature moved. Never touches the header, the +// radio strip, or anything outside the analytics rows, so no startFrame() and +// no whole-screen clear is involved. +inline bool drawChangedRows(DisplayDriver& d, const Layout& l, const Context& ctx, + const RadioActivitySnapshot& s, uint32_t signatures[ROW_COUNT]) { + uint32_t fresh[ROW_COUNT]; + allRowSignatures(l, ctx, s, fresh); + + bool drew = false; + for (int r = 0; r < ROW_COUNT; r++) { + if (fresh[r] == signatures[r]) continue; + drawRow(d, l, (Row)r, ctx, s, true); + signatures[r] = fresh[r]; + drew = true; + } + return drew; +} + +} // namespace ObserverDashboard diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index 1d00fd15..bcdaf81c 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -112,13 +112,18 @@ void ST7789LCDDisplay::turnOff() { digitalWrite(PIN_TFT_LEDA_CTL, HIGH); #endif } + #ifndef HELTEC_V4_R8_TFT if (PIN_TFT_RST != -1) { digitalWrite(PIN_TFT_RST, LOW); } - #ifndef HELTEC_V4_R8_TFT if (PIN_TFT_LEDA_CTL != -1) { digitalWrite(PIN_TFT_LEDA_CTL, LOW); } + #else + // On the V4 R8 Expansion Kit this reset line is shared with the touch + // panel's TP_RST, so parking it low would hold the touch controller in + // reset for as long as the display is off. Killing the backlight is what + // "off" means for this LCD anyway. #endif _isOn = false; diff --git a/src/helpers/ui/TouchTapDetector.h b/src/helpers/ui/TouchTapDetector.h new file mode 100644 index 00000000..b2a610f4 --- /dev/null +++ b/src/helpers/ui/TouchTapDetector.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +// Debounced rising-edge detector for a polled touch panel. +// +// Pure logic: no Arduino, no I2C. The caller polls the panel and hands over a +// raw "finger down" reading; this decides when that counts as a new tap. All +// elapsed-time comparisons are unsigned subtractions, so millis() rollover is +// a non-event. + +#ifndef TOUCH_TAP_DEBOUNCE_MS +#define TOUCH_TAP_DEBOUNCE_MS 40 +#endif + +// Ignores a second tap arriving this soon after an accepted one, so a bouncy +// panel or a slightly long press cannot toggle the display twice. +#ifndef TOUCH_TAP_MIN_GAP_MS +#define TOUCH_TAP_MIN_GAP_MS 400 +#endif + +class TouchTapDetector { +public: + TouchTapDetector() { reset(0); } + + void reset(uint32_t now_ms = 0) { + _raw = false; + _stable = false; + _changed_at = now_ms; + _last_tap = now_ms; + _tapped_before = false; + } + + // Returns true exactly once per accepted finger-down. + bool update(uint32_t now_ms, bool pressed) { + if (pressed != _raw) { // reading moved; restart the settling window + _raw = pressed; + _changed_at = now_ms; + return false; + } + if (now_ms - _changed_at < TOUCH_TAP_DEBOUNCE_MS) return false; // not settled + if (_raw == _stable) return false; // nothing new + + _stable = _raw; + if (!_stable) return false; // this is the release, not a tap + + if (_tapped_before && (now_ms - _last_tap) < TOUCH_TAP_MIN_GAP_MS) return false; + _last_tap = now_ms; + _tapped_before = true; + return true; + } + + bool isTouched() const { return _stable; } + +private: + uint32_t _changed_at; + uint32_t _last_tap; + bool _raw; + bool _stable; + bool _tapped_before; +}; diff --git a/test/README.md b/test/README.md index 901227ef..647b6daf 100644 --- a/test/README.md +++ b/test/README.md @@ -38,6 +38,9 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_prefs_serializer` | `src/helpers/MQTTPrefsSerializer.h`, `src/helpers/ConfigSerializer.*` | semantic nested `/mqtt.json` round trips; numeric slot keys; required/future version handling; strict length/overflow/duplicate rejection; safe semantic repair; scratch-before-live loading | | `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h`, `src/helpers/MQTTPrefsRecovery.h` | production JSON begin/write/checksum-finish/schema-verify/commit orchestration; first-migration and rename-boundary recovery; legacy `/node_prefs` handoff; failure cleanup and original-file preservation | | `test_mqtt_payload_builder` | `src/helpers/MQTTPayloadBuilder.cpp` | status/packet/raw JSON contracts; optional fields; escaping; RX metrics and path; score handling; exact buffer bounds; maximum representative payloads | +| `test_radio_activity_window` | `src/helpers/RadioActivityWindow.h` | 20-minute minute-bucketed RX window: totals and derived rates; bucket rotation and oldest-to-newest ordering; expiry at the boundary; ring clear after 20 minutes of silence; warm-up versus steady-state denominators; peak minute; last-packet age and staleness; counter saturation; `millis()` rollover, including the minute boundary a `now_ms / 60000` quotient would corrupt | +| `test_observer_dashboard` | `src/helpers/ui/ObserverDashboard.h` | R8 TFT observer dashboard against a recording `DisplayDriver` in both orientation profiles: compact number/byte/age formatting and the 5 s age quantisation; per-row character budgets; on-panel and inside-the-margin bounds; no silent portrait scale fallback; non-overlapping row rectangles and each row's repaint covering everything it draws; 20-bar graph scaling, ordering and empty/spike cases; per-row signatures and the partial-repaint policy | +| `test_touch_tap_detector` | `src/helpers/ui/TouchTapDetector.h` | debounced rising-edge detection for the polled Expansion Kit touch panel: idle quiet; one tap per touch; long presses do not repeat; sub-debounce blips ignored; contact bounce still counts once; minimum gap between accepted taps; `millis()` rollover; reset semantics | | `test_utils` | `src/Utils.cpp` | `Utils::toHex` (upstream) | ## Conventions (and how to add a suite) diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp index ce91b0c1..8c8bf32c 100644 --- a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -78,6 +78,7 @@ static MQTTPrefs defaults() { prefs.alert_mqtt_minutes = 240; prefs.alert_min_interval_min = 60; prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; + prefs.display_timeout_secs = DISPLAY_TIMEOUT_DEFAULT_SECS; strcpy(prefs.snmp_community, "public"); for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { strcpy(prefs.mqtt_slot_preset[i], "none"); @@ -375,6 +376,72 @@ TEST(MQTTPrefsSerializer, SaveNormalizationIsIdempotentAgainstKnownDefaults) { EXPECT_FALSE(repaired) << output.text(); } +TEST(MQTTPrefsSerializer, DisplayTimeoutRoundTrips) { + for (uint16_t secs : {(uint16_t)0, (uint16_t)45, DISPLAY_TIMEOUT_MAX_SECS}) { + MQTTPrefs source = defaults(); + source.display_timeout_secs = secs; + + OutputStream output; + MQTTPrefsSerializer writer(&source); + ASSERT_TRUE(writer.saveSerial(output)) << secs; + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)) << secs; + bool repaired = false; + ASSERT_TRUE(reader.apply(&repaired)) << secs; + EXPECT_FALSE(repaired) << secs; + EXPECT_EQ(secs, loaded.display_timeout_secs); + } +} + +TEST(MQTTPrefsSerializer, RepairsDisplayTimeoutOutOfRange) { + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,display:{timeout_s:99999}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(DISPLAY_TIMEOUT_DEFAULT_SECS, prefs.display_timeout_secs); + + prefs = defaults(); + InputStream negative("{version:1,display:{timeout_s:-5}}"); + MQTTPrefsSerializer negative_serializer(&prefs); + ASSERT_TRUE(negative_serializer.loadSerial(negative)); + repaired = false; + ASSERT_TRUE(negative_serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(DISPLAY_TIMEOUT_DEFAULT_SECS, prefs.display_timeout_secs); +} + +TEST(MQTTPrefsSerializer, PrefsWrittenBeforeTheDisplayGroupStillLoad) { + // Upgrade path: a /mqtt.json from firmware without the display group must + // load cleanly and keep the default rather than collapsing to 0 ("stay on"). + MQTTPrefs prefs = defaults(); + InputStream input("{version:1,radio:{watchdog_min:5}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_EQ(DISPLAY_TIMEOUT_DEFAULT_SECS, prefs.display_timeout_secs); +} + +TEST(MQTTPrefsSerializer, UnknownGroupsAreIgnoredSoAppendedKeysAreDowngradeSafe) { + // The mirror of the case above, and the reason appending `display` needed no + // MQTT_PREFS_JSON_FORMAT_VERSION bump: firmware that predates a group skips + // it rather than failing the load. + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,display:{timeout_s:45},future:{thing:1,nested:{x:2}}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_EQ(45, prefs.display_timeout_secs); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_observer_dashboard/MockDisplay.h b/test/test_observer_dashboard/MockDisplay.h new file mode 100644 index 00000000..23d243fe --- /dev/null +++ b/test/test_observer_dashboard/MockDisplay.h @@ -0,0 +1,122 @@ +#pragma once + +#include "helpers/ui/DisplayDriver.h" +#include "helpers/ui/DisplayViewport.h" + +#include +#include + +// A DisplayDriver that records physical-pixel draw calls instead of pushing +// them at a panel, reproducing the two real ST7789LCDDisplay coordinate and +// text-metric profiles: +// +// portrait (ST7789_PORTRAIT_PROFILE) 240x320, DisplayViewport mapping, +// physical text scale = logical * 2 +// landscape (default) 320x240, x * 2.5 / y * 3.75, +// physical text scale = (int)(logical * 2.5) +// +// Text extents are recorded at the size the layout *asked* for, so a row that +// only fits because the driver would silently shrink or clip it still shows up +// as an out-of-bounds op. +class MockDisplay : public DisplayDriver { +public: + enum Mode { PORTRAIT, LANDSCAPE }; + + struct Op { + enum Kind { FILL, RECT, TEXT } kind; + int x, y, w, h; // physical pixels + ColorVal color; + std::string text; + int logical_size; + bool scale_fallback; // portrait driver would have shrunk this string + }; + + explicit MockDisplay(Mode mode) + : DisplayDriver(128, 64), _mode(mode), _on(true), _color(0), _size(1), _cx(0), _cy(0) {} + + std::vector ops; + + int panelWidth() const { return _mode == PORTRAIT ? 240 : 320; } + int panelHeight() const { return _mode == PORTRAIT ? 320 : 240; } + + void reset() { ops.clear(); } + + // --- DisplayDriver --- + bool isOn() override { return _on; } + void turnOn() override { _on = true; } + void turnOff() override { _on = false; } + void clear() override { ops.clear(); } + void startFrame(ColorVal bkg = UIColor::window_bkg) override { + ops.clear(); + ops.push_back(Op{Op::FILL, 0, 0, panelWidth(), panelHeight(), bkg, "", 1, false}); + _size = 1; + } + void setTextSize(int sz) override { _size = sz > 0 ? sz : 1; } + void setColor(ColorVal c) override { _color = c; } + void setCursor(int x, int y) override { _cx = x; _cy = y; } + + void print(const char* str) override { + if (!str || !*str) return; + int n = (int)strlen(str); + int scale = physicalScale(_size); + int px = mapX(_cx), py = mapY(_cy); + bool fallback = false; + if (_mode == PORTRAIT) { + int available = panelWidth() - px; + if (n * 6 * scale > available) { + fallback = true; // the real driver drops to the minimum scale here + } + } + ops.push_back(Op{Op::TEXT, px, py, n * 6 * scale, 8 * scale, _color, std::string(str), _size, + fallback}); + _cx += (int)((n * 6 * scale) / xScale()); + } + + void fillRect(int x, int y, int w, int h) override { + ops.push_back(Op{Op::FILL, mapX(x), mapY(y), spanX(x, w), spanY(y, h), _color, "", _size, + false}); + } + void drawRect(int x, int y, int w, int h) override { + ops.push_back(Op{Op::RECT, mapX(x), mapY(y), spanX(x, w), spanY(y, h), _color, "", _size, + false}); + } + void drawXbm(int, int, const uint8_t*, int, int) override {} + void endFrame() override {} + + uint16_t getTextWidth(const char* str) override { + if (!str) return 0; + int n = (int)strlen(str); + int scale = physicalScale(_size); + if (_mode == PORTRAIT) { + // Mirrors ST7789LCDDisplay::getTextWidth(): measure at the scale the + // driver would pick, clamp to the panel, convert back to logical. + if (n * 6 * scale > panelWidth()) scale = _size; + int w = n * 6 * scale; + if (w > panelWidth()) w = panelWidth(); + DisplayViewport::Geometry g{128, 64, 240, 320}; + return g.logicalWidthForPhysical((uint16_t)w); + } + return (uint16_t)((n * 6 * scale) / 2.5f); + } + +private: + Mode _mode; + bool _on; + ColorVal _color; + int _size, _cx, _cy; + + float xScale() const { return _mode == PORTRAIT ? (240.0f / 128.0f) : 2.5f; } + + int physicalScale(int logical) const { + return _mode == PORTRAIT ? logical * 2 : (int)(uint8_t)(logical * 2.5f); + } + + int mapX(int x) const { + return _mode == PORTRAIT ? (int)((int32_t)x * 240 / 128) : (int)(x * 2.5f); + } + int mapY(int y) const { + return _mode == PORTRAIT ? (int)((int32_t)y * 320 / 64) : (int)(y * 3.75f); + } + int spanX(int x, int w) const { return mapX(x + w) - mapX(x); } + int spanY(int y, int h) const { return mapY(y + h) - mapY(y); } +}; diff --git a/test/test_observer_dashboard/test_observer_dashboard.cpp b/test/test_observer_dashboard/test_observer_dashboard.cpp new file mode 100644 index 00000000..7e036bba --- /dev/null +++ b/test/test_observer_dashboard/test_observer_dashboard.cpp @@ -0,0 +1,581 @@ +#include "MockDisplay.h" +#include "helpers/ui/ObserverDashboard.h" + +#include + +// UIColor's slots live in whichever display driver a firmware target links; the +// host build supplies its own. +ColorVal UIColor::window_bkg = 0; +ColorVal UIColor::title_bkg = 0; +ColorVal UIColor::title_txt = 0; +ColorVal UIColor::primary_txt = 0; +ColorVal UIColor::secondary_txt = 0; +ColorVal UIColor::warning_txt = 0; +ColorVal UIColor::popup_bkg = 0; +ColorVal UIColor::popup_txt = 0; +ColorVal UIColor::corp_blue = 0; + +using namespace ObserverDashboard; + +namespace { + +const int N = RADIO_ACTIVITY_BUCKETS; + +struct Profile { + MockDisplay::Mode mode; + Layout layout; + const char* name; + int panel_w, panel_h; + int margin_left, margin_right; // physical +}; + +Profile portrait() { return {MockDisplay::PORTRAIT, portraitLayout(), "portrait", 240, 320, 7, 232}; } +Profile landscape() { return {MockDisplay::LANDSCAPE, landscapeLayout(), "landscape", 320, 240, 10, 310}; } + +Context makeContext(const char* name = "Ridgeline North") { + Context c; + c.node_name = name; + c.role_label = "REPEATER"; + c.freq = 910.525f; + c.sf = 7; + c.bw = 62.5f; + c.link_up = true; + return c; +} + +// A busy but plausible 20 minutes: 1843 packets, a peak minute of 214. +RadioActivitySnapshot makeBusy() { + RadioActivityWindow w; + w.reset(0); + const uint16_t per_minute[RADIO_ACTIVITY_BUCKETS] = {12, 40, 8, 0, 97, 133, 71, 3, 214, 65, + 19, 88, 44, 27, 0, 150, 92, 61, 7, 35}; + for (int m = 0; m < N; m++) { + for (int i = 0; i < per_minute[m]; i++) { + w.recordPacket((uint32_t)m * RADIO_ACTIVITY_BUCKET_MS + 1000 + i, 48, 120, 26, -103); + } + } + RadioActivitySnapshot s; + w.snapshot((uint32_t)(N - 1) * RADIO_ACTIVITY_BUCKET_MS + 30000, &s); + return s; +} + +RadioActivitySnapshot makeEmpty() { + RadioActivityWindow w; + w.reset(0); + RadioActivitySnapshot s; + w.snapshot(7 * RADIO_ACTIVITY_BUCKET_MS, &s); + return s; +} + +bool insideRect(const MockDisplay::Op& op, int x, int y, int w, int h) { + return op.x >= x && op.y >= y && op.x + op.w <= x + w && op.y + op.h <= y + h; +} + +} // namespace + +// ------------------------------------------------------------- formatting --- + +TEST(ObserverDashboardFormat, CompactCountsStayShortAtEveryMagnitude) { + char b[24]; + struct { uint32_t v; const char* want; } cases[] = { + {0, "0"}, {7, "7"}, {9999, "9999"}, {10000, "10.0k"}, + {12345, "12.3k"}, {99999, "99.9k"}, {100000, "100k"}, {999999, "999k"}, + {1000000, "1.0M"},{12345678, "12.3M"},{100000000, "100M"}}; + for (auto& c : cases) { + formatCompactCount(b, sizeof(b), c.v); + EXPECT_STREQ(c.want, b) << "value " << c.v; + EXPECT_LE(strlen(b), 5u) << "value " << c.v; + } +} + +TEST(ObserverDashboardFormat, CompactBytesPickSensibleUnits) { + char b[24]; + struct { uint32_t v; const char* want; } cases[] = { + {0, "0 B"}, {1023, "1023 B"}, {1024, "1.0 KB"}, + {10240, "10.0 KB"}, {145408, "142 KB"}, {1048576, "1.0 MB"}, + {15728640, "15.0 MB"}}; + for (auto& c : cases) { + formatCompactBytes(b, sizeof(b), c.v); + EXPECT_STREQ(c.want, b) << "value " << c.v; + } +} + +TEST(ObserverDashboardFormat, TenthsAndSignedTenths) { + char b[24]; + formatTenths(b, sizeof(b), 0); EXPECT_STREQ("0.0", b); + formatTenths(b, sizeof(b), 34); EXPECT_STREQ("3.4", b); + formatTenths(b, sizeof(b), 999); EXPECT_STREQ("99.9", b); + formatTenths(b, sizeof(b), 1000); EXPECT_STREQ("100", b); + + formatSignedTenths(b, sizeof(b), 72); EXPECT_STREQ("+7.2", b); + formatSignedTenths(b, sizeof(b), 0); EXPECT_STREQ("+0.0", b); + formatSignedTenths(b, sizeof(b), -115); EXPECT_STREQ("-11.5", b); +} + +TEST(ObserverDashboardFormat, AgeIsQuantisedToTheFiveSecondCadence) { + EXPECT_EQ(0u, quantizeAgeSecs(0)); + EXPECT_EQ(0u, quantizeAgeSecs(4999)); + EXPECT_EQ(5u, quantizeAgeSecs(5000)); + EXPECT_EQ(5u, quantizeAgeSecs(9999)); + + char b[24]; + formatAge(b, sizeof(b), 0, false); EXPECT_STREQ("--", b); + formatAge(b, sizeof(b), 0, true); EXPECT_STREQ("now", b); + formatAge(b, sizeof(b), 4999, true); EXPECT_STREQ("now", b); + formatAge(b, sizeof(b), 12000, true); EXPECT_STREQ("10s", b); + formatAge(b, sizeof(b), 59999, true); EXPECT_STREQ("55s", b); + formatAge(b, sizeof(b), 60000, true); EXPECT_STREQ("1m", b); + formatAge(b, sizeof(b), 3599999, true); EXPECT_STREQ("59m", b); + formatAge(b, sizeof(b), 3600000, true); EXPECT_STREQ("1h", b); +} + +TEST(ObserverDashboardFormat, EmptyWindowNeverProducesNanOrInfinity) { + RadioActivitySnapshot s = makeEmpty(); + Context ctx = makeContext(); + + for (int r = 0; r < ROW_COUNT; r++) { + if (r == ROW_GRAPH) continue; + RowText t; + composeRow((Row)r, ctx, s, &t); + for (const char* p : {t.left, t.right}) { + EXPECT_EQ(nullptr, strstr(p, "nan")) << p; + EXPECT_EQ(nullptr, strstr(p, "inf")) << p; + } + } + + // Unmeasurable values read as "--"; measured zeroes read as real zeroes. + RowText headline, rate, rf, status; + composeRow(ROW_HEADLINE, ctx, s, &headline); + composeRow(ROW_RATE, ctx, s, &rate); + composeRow(ROW_RF, ctx, s, &rf); + composeRow(ROW_STATUS, ctx, s, &status); + EXPECT_STREQ("No RF yet", headline.left); + EXPECT_STREQ("0 B", rate.left); + EXPECT_STREQ("0.0/min", rate.right); + EXPECT_STREQ("SNR --", rf.left); + EXPECT_STREQ("AIR 0.0%", rf.right); + EXPECT_STREQ("RX --", status.left); +} + +TEST(ObserverDashboardFormat, EveryRowFitsTheCharacterBudget) { + for (const Profile& p : {portrait(), landscape()}) { + for (const RadioActivitySnapshot& s : {makeBusy(), makeEmpty()}) { + for (int r = 0; r < ROW_COUNT; r++) { + if (r == ROW_GRAPH) continue; + RowText t; + composeRow((Row)r, makeContext(), s, &t); + int budget = (r == ROW_HEADLINE) ? p.layout.max_chars_big : p.layout.max_chars; + int used = (int)strlen(t.left) + (int)strlen(t.right); + if (t.left[0] && t.right[0]) used += 1; // separating space + EXPECT_LE(used, budget) << p.name << " row " << r << ": '" << t.left << "' / '" << t.right << "'"; + } + } + } +} + +TEST(ObserverDashboardFormat, RadioStripDropsADeadBandwidthDecimal) { + char b[32]; + formatRadioStrip(b, sizeof(b), 910.525f, 7, 62.5f); + EXPECT_STREQ("910.525 SF7 BW62.5", b); + formatRadioStrip(b, sizeof(b), 869.618f, 8, 250.0f); + EXPECT_STREQ("869.618 SF8 BW250", b); + formatRadioStrip(b, sizeof(b), 433.125f, 12, 125.0f); + EXPECT_STREQ("433.125 SF12 BW125", b); +} + +TEST(ObserverDashboardFormat, RadioStripFitsEveryOrientationsBudget) { + // The widest realistic combination must not be ellipsized away. + const struct { float freq; uint8_t sf; float bw; } cases[] = { + {910.525f, 7, 62.5f}, {869.618f, 8, 250.0f}, {433.125f, 12, 125.0f}, + {915.000f, 11, 500.0f}, {868.000f, 9, 41.7f}}; + for (const Profile& p : {portrait(), landscape()}) { + for (const auto& c : cases) { + char b[32]; + formatRadioStrip(b, sizeof(b), c.freq, c.sf, c.bw); + EXPECT_LE((int)strlen(b), p.layout.max_chars) << p.name << " '" << b << "'"; + } + } +} + +TEST(ObserverDashboardLayout, HeaderTextStaysInsideTheHeaderBar) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + drawHeader(d, p.layout, makeContext()); + + ASSERT_FALSE(d.ops.empty()); + const auto& bar = d.ops.front(); + ASSERT_EQ(MockDisplay::Op::FILL, bar.kind) << p.name; + for (size_t i = 1; i < d.ops.size(); i++) { + EXPECT_TRUE(insideRect(d.ops[i], bar.x, bar.y, bar.w, bar.h)) + << p.name << " '" << d.ops[i].text << "'"; + } + } +} + +TEST(ObserverDashboardLayout, PortraitHeaderShowsTheWholeNodeName) { + // A 16-character name must survive intact: on a 240 px panel the role label + // moves to a second header line rather than eating the name. + MockDisplay d(MockDisplay::PORTRAIT); + Context ctx = makeContext("Ridgeline North"); + drawHeader(d, portraitLayout(), ctx); + + bool saw_name = false, saw_role = false; + for (const auto& op : d.ops) { + if (op.text == "Ridgeline North") saw_name = true; + if (op.text == "REPEATER") saw_role = true; + } + EXPECT_TRUE(saw_name) << "node name was ellipsized"; + EXPECT_TRUE(saw_role); +} + +TEST(ObserverDashboardFormat, TextIsAsciiOnly) { + // The driver's UTF-8 fallback collapses every non-ASCII byte to a full block, + // so any stray multi-byte character would render as a solid glyph. + for (const RadioActivitySnapshot& s : {makeBusy(), makeEmpty()}) { + for (int r = 0; r < ROW_COUNT; r++) { + if (r == ROW_GRAPH) continue; + RowText t; + composeRow((Row)r, makeContext(), s, &t); + for (const char* p : {t.left, t.right}) { + for (const char* c = p; *c; c++) { + EXPECT_GE((unsigned char)*c, 32u) << "row " << r; + EXPECT_LE((unsigned char)*c, 126u) << "row " << r; + } + } + } + } +} + +// ----------------------------------------------------------------- layout --- + +TEST(ObserverDashboardLayout, EveryDrawnPixelStaysOnThePanel) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + drawFull(d, p.layout, makeContext(), makeBusy()); + ASSERT_FALSE(d.ops.empty()); + for (const auto& op : d.ops) { + EXPECT_GE(op.x, 0) << p.name; + EXPECT_GE(op.y, 0) << p.name; + EXPECT_LE(op.x + op.w, p.panel_w) << p.name << " '" << op.text << "'"; + EXPECT_LE(op.y + op.h, p.panel_h) << p.name << " '" << op.text << "'"; + } + } +} + +TEST(ObserverDashboardLayout, AllTextIsPaddedInsideTheMargins) { + for (const Profile& p : {portrait(), landscape()}) { + for (const RadioActivitySnapshot& s : {makeBusy(), makeEmpty()}) { + MockDisplay d(p.mode); + drawFull(d, p.layout, makeContext(), s); + for (const auto& op : d.ops) { + if (op.kind != MockDisplay::Op::TEXT) continue; + EXPECT_GE(op.x, p.margin_left) << p.name << " '" << op.text << "'"; + EXPECT_LE(op.x + op.w, p.margin_right) << p.name << " '" << op.text << "'"; + } + } + } +} + +TEST(ObserverDashboardLayout, NoTextSilentlyShrinksToTheFallbackScale) { + // The portrait driver halves the glyph size rather than clipping. A row that + // only fits because of that would break the grid, so it must never happen. + for (const RadioActivitySnapshot& s : {makeBusy(), makeEmpty()}) { + MockDisplay d(MockDisplay::PORTRAIT); + drawFull(d, portraitLayout(), makeContext(), s); + for (const auto& op : d.ops) { + EXPECT_FALSE(op.scale_fallback) << "'" << op.text << "'"; + } + } +} + +TEST(ObserverDashboardLayout, ContentClearsTheTopAndBottomEdges) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + drawFull(d, p.layout, makeContext(), makeBusy()); + int lowest = 0; + for (const auto& op : d.ops) lowest = std::max(lowest, op.y + op.h); + EXPECT_GE(p.panel_h - lowest, 8) << p.name << ": bottom margin too small"; + } +} + +TEST(ObserverDashboardLayout, RowRectanglesDoNotOverlap) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + for (int a = 0; a < ROW_COUNT; a++) { + for (int b = a + 1; b < ROW_COUNT; b++) { + int ay = p.layout.margin_x, unused = ay; + (void)unused; + int a_top = rowY(p.layout, (Row)a), a_bot = a_top + rowH(p.layout, (Row)a); + int b_top = rowY(p.layout, (Row)b), b_bot = b_top + rowH(p.layout, (Row)b); + bool overlap = a_top < b_bot && b_top < a_bot; + EXPECT_FALSE(overlap) << p.name << ": rows " << a << " and " << b; + } + } + // ...and the header and radio strip sit above the first row. + EXPECT_LT(p.layout.header_h, p.layout.radio_y) << p.name; + EXPECT_LT(p.layout.radio_y + p.layout.text_h, rowY(p.layout, ROW_WINDOW) + 1) << p.name; + } +} + +TEST(ObserverDashboardLayout, EachRowRepaintCoversEverythingThatRowDraws) { + // The no-flash invariant: a partial repaint clears one row rectangle and then + // redraws inside it. Anything drawn outside that rectangle would leave stale + // pixels behind or scribble on a neighbouring row. + for (const Profile& p : {portrait(), landscape()}) { + for (const RadioActivitySnapshot& s : {makeBusy(), makeEmpty()}) { + for (int r = 0; r < ROW_COUNT; r++) { + MockDisplay d(p.mode); + drawRow(d, p.layout, (Row)r, makeContext(), s, true); + ASSERT_FALSE(d.ops.empty()) << p.name << " row " << r; + + const auto& clear = d.ops.front(); + ASSERT_EQ(MockDisplay::Op::FILL, clear.kind) << p.name << " row " << r; + EXPECT_EQ(BG, clear.color) << p.name << " row " << r; + + for (size_t i = 1; i < d.ops.size(); i++) { + EXPECT_TRUE(insideRect(d.ops[i], clear.x, clear.y, clear.w, clear.h)) + << p.name << " row " << r << " op " << i << " '" << d.ops[i].text << "'"; + } + } + } + } +} + +// ------------------------------------------------------------------ graph --- + +TEST(ObserverDashboardGraph, DrawsExactlyTwentyNonOverlappingBars) { + for (const Profile& p : {portrait(), landscape()}) { + RadioActivitySnapshot s = makeBusy(); + for (int i = 0; i < N; i++) s.buckets[i] = (uint16_t)(i + 1); + s.peak_per_min = N; + + MockDisplay d(p.mode); + drawGraph(d, p.layout, s); + + // First op is the baseline, then one bar per non-empty bucket. + ASSERT_GE(d.ops.size(), 1u); + std::vector bars(d.ops.begin() + 1, d.ops.end()); + ASSERT_EQ((size_t)N, bars.size()) << p.name; + + for (size_t i = 1; i < bars.size(); i++) { + EXPECT_GE(bars[i].x, bars[i - 1].x + bars[i - 1].w) << p.name << " bar " << i << " overlaps"; + EXPECT_GE(bars[i].h, bars[i - 1].h) << p.name << " bar " << i << " not monotonic"; + } + EXPECT_EQ(BAR_NOW, bars.back().color) << p.name << ": current minute must stand out"; + EXPECT_EQ(BAR, bars.front().color) << p.name; + } +} + +TEST(ObserverDashboardGraph, BarsStayInsideTheGraphRectangle) { + for (const Profile& p : {portrait(), landscape()}) { + RadioActivitySnapshot s = makeBusy(); + MockDisplay probe(p.mode); + probe.setColor(0); + probe.fillRect(p.layout.margin_x, p.layout.graph_y, p.layout.right_x - p.layout.margin_x, + p.layout.graph_h); + MockDisplay::Op rect = probe.ops.front(); + + MockDisplay d(p.mode); + drawGraph(d, p.layout, s); + for (const auto& op : d.ops) { + EXPECT_TRUE(insideRect(op, rect.x, rect.y, rect.w, rect.h)) << p.name; + } + } +} + +TEST(ObserverDashboardGraph, AllZeroWindowDrawsOnlyTheBaseline) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + drawGraph(d, p.layout, makeEmpty()); + ASSERT_EQ(1u, d.ops.size()) << p.name << ": empty minutes must not draw one-pixel activity"; + EXPECT_EQ(GRID, d.ops.front().color) << p.name; + } +} + +TEST(ObserverDashboardGraph, SingleSpikeFillsTheGraphAndLeavesTheRestEmpty) { + for (const Profile& p : {portrait(), landscape()}) { + RadioActivitySnapshot s = makeEmpty(); + s.buckets[5] = 400; + s.peak_per_min = 400; + s.packets = 400; + + uint8_t h[RADIO_ACTIVITY_BUCKETS]; + barHeights(p.layout, s, h); + EXPECT_EQ(p.layout.graph_h - 1, h[5]) << p.name; + for (int i = 0; i < N; i++) { + if (i != 5) EXPECT_EQ(0, h[i]) << p.name << " bucket " << i; + } + + MockDisplay d(p.mode); + drawGraph(d, p.layout, s); + EXPECT_EQ(2u, d.ops.size()) << p.name; // baseline + one bar + } +} + +TEST(ObserverDashboardGraph, AnyTrafficRoundsUpToAVisibleBar) { + for (const Profile& p : {portrait(), landscape()}) { + RadioActivitySnapshot s = makeEmpty(); + s.buckets[0] = 1; + s.buckets[19] = 5000; + s.peak_per_min = 5000; + + uint8_t h[RADIO_ACTIVITY_BUCKETS]; + barHeights(p.layout, s, h); + EXPECT_EQ(1, h[0]) << p.name << ": a single packet must still be visible"; + EXPECT_EQ(p.layout.graph_h - 1, h[19]) << p.name; + } +} + +// ------------------------------------------------------------- signatures --- + +TEST(ObserverDashboardSignature, IdenticallyFormattedDataDoesNotRepaint) { + Layout l = portraitLayout(); + Context ctx = makeContext(); + + RadioActivitySnapshot a = makeEmpty(); + a.packets = 100000; + RadioActivitySnapshot b = a; + b.packets = 100999; // both render as "100k pkt" + + EXPECT_EQ(rowSignature(l, ROW_HEADLINE, ctx, a), rowSignature(l, ROW_HEADLINE, ctx, b)); + + b.packets = 101500; // renders as "101k pkt" + EXPECT_NE(rowSignature(l, ROW_HEADLINE, ctx, a), rowSignature(l, ROW_HEADLINE, ctx, b)); +} + +TEST(ObserverDashboardSignature, AGraphChangeTouchesOnlyTheGraphRow) { + Layout l = portraitLayout(); + Context ctx = makeContext(); + + RadioActivitySnapshot a = makeBusy(); + RadioActivitySnapshot b = a; + b.buckets[N - 1] = (uint16_t)(a.buckets[N - 1] + 40); // the current minute grows + + uint32_t sa[ROW_COUNT], sb[ROW_COUNT]; + allRowSignatures(l, ctx, a, sa); + allRowSignatures(l, ctx, b, sb); + + for (int r = 0; r < ROW_COUNT; r++) { + if (r == ROW_GRAPH) { + EXPECT_NE(sa[r], sb[r]) << "graph row must notice the new bar height"; + } else { + EXPECT_EQ(sa[r], sb[r]) << "row " << r << " must not repaint"; + } + } +} + +TEST(ObserverDashboardSignature, DataChangesBelowTheGraphResolutionDoNotRepaint) { + // Signatures are computed from bar heights, not from the packet counts behind + // them, so a busy minute ticking up by one costs nothing on screen. + Layout l = portraitLayout(); + Context ctx = makeContext(); + + RadioActivitySnapshot a = makeBusy(); + RadioActivitySnapshot b = a; + b.buckets[N - 1] = (uint16_t)(a.buckets[N - 1] + 1); + + uint8_t ha[RADIO_ACTIVITY_BUCKETS], hb[RADIO_ACTIVITY_BUCKETS]; + barHeights(l, a, ha); + barHeights(l, b, hb); + ASSERT_EQ(ha[N - 1], hb[N - 1]) << "test needs a change smaller than one bar unit"; + + EXPECT_EQ(rowSignature(l, ROW_GRAPH, ctx, a), rowSignature(l, ROW_GRAPH, ctx, b)); +} + +TEST(ObserverDashboardSignature, LinkStateOnlyTouchesTheStatusRow) { + Layout l = portraitLayout(); + RadioActivitySnapshot s = makeBusy(); + + Context up = makeContext(); + Context down = makeContext(); + down.link_up = false; + + uint32_t sa[ROW_COUNT], sb[ROW_COUNT]; + allRowSignatures(l, up, s, sa); + allRowSignatures(l, down, s, sb); + + for (int r = 0; r < ROW_COUNT; r++) { + if (r == ROW_STATUS) { + EXPECT_NE(sa[r], sb[r]); + } else { + EXPECT_EQ(sa[r], sb[r]) << "row " << r; + } + } +} + +TEST(ObserverDashboardSignature, PartialRepaintDrawsOnlyTheChangedRow) { + Profile p = portrait(); + Context ctx = makeContext(); + RadioActivitySnapshot a = makeBusy(); + + uint32_t sigs[ROW_COUNT]; + allRowSignatures(p.layout, ctx, a, sigs); + + RadioActivitySnapshot b = a; + b.buckets[N - 1] = (uint16_t)(a.buckets[N - 1] + 40); + + MockDisplay d(p.mode); + EXPECT_TRUE(drawChangedRows(d, p.layout, ctx, b, sigs)); + + // One clear plus the graph contents, all inside the graph rectangle. + ASSERT_FALSE(d.ops.empty()); + const auto& clear = d.ops.front(); + EXPECT_EQ(BG, clear.color); + for (const auto& op : d.ops) { + EXPECT_TRUE(insideRect(op, clear.x, clear.y, clear.w, clear.h)); + } + + // Nothing left to do on a second pass with the same data. + MockDisplay d2(p.mode); + EXPECT_FALSE(drawChangedRows(d2, p.layout, ctx, b, sigs)); + EXPECT_TRUE(d2.ops.empty()); +} + +TEST(ObserverDashboardSignature, LongNodeNameIsTrimmedInsideTheHeader) { + for (const Profile& p : {portrait(), landscape()}) { + MockDisplay d(p.mode); + Context ctx = makeContext("A Very Long Repeater Node Name That Cannot Possibly Fit"); + drawHeader(d, p.layout, ctx); + + bool saw_role = false, saw_ellipsis = false; + for (const auto& op : d.ops) { + if (op.kind != MockDisplay::Op::TEXT) continue; + EXPECT_GE(op.x, p.margin_left) << p.name; + EXPECT_LE(op.x + op.w, p.margin_right) << p.name << " '" << op.text << "'"; + EXPECT_FALSE(op.scale_fallback) << p.name << " '" << op.text << "'"; + if (op.text == "REPEATER") saw_role = true; + if (op.text.size() >= 3 && op.text.compare(op.text.size() - 3, 3, "...") == 0) + saw_ellipsis = true; + } + EXPECT_TRUE(saw_role) << p.name << ": the role label must survive a long node name"; + EXPECT_TRUE(saw_ellipsis) << p.name << ": the name must be visibly truncated"; + } +} + +TEST(ObserverDashboardSignature, FitToCharsRespectsItsBudget) { + char b[32]; + fitToChars(b, sizeof(b), "short", 18); EXPECT_STREQ("short", b); + fitToChars(b, sizeof(b), "exactly-18-chars!", 17); EXPECT_STREQ("exactly-18-chars!", b); + fitToChars(b, sizeof(b), "Ridgeline North Ridge", 18); + EXPECT_STREQ("Ridgeline North...", b); + EXPECT_EQ(18u, strlen(b)); + fitToChars(b, sizeof(b), "abcdef", 3); EXPECT_STREQ("abc", b); + fitToChars(b, sizeof(b), "abcdef", 0); EXPECT_STREQ("", b); + fitToChars(b, sizeof(b), "", 18); EXPECT_STREQ("", b); +} + +TEST(ObserverDashboardSignature, DarkPaletteRetunesTheSharedColourSlots) { + applyDarkPalette(); + EXPECT_EQ(BG, UIColor::window_bkg); + EXPECT_EQ(TEXT, UIColor::primary_txt); + EXPECT_EQ(HEADER_BG, UIColor::title_bkg); + EXPECT_EQ(ACCENT, UIColor::corp_blue); + // The setup portal's highlight must stay legible on the dark background. + EXPECT_NE(UIColor::window_bkg, UIColor::warning_txt); + EXPECT_NE(UIColor::window_bkg, UIColor::primary_txt); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_radio_activity_window/test_radio_activity_window.cpp b/test/test_radio_activity_window/test_radio_activity_window.cpp new file mode 100644 index 00000000..65f953e4 --- /dev/null +++ b/test/test_radio_activity_window/test_radio_activity_window.cpp @@ -0,0 +1,376 @@ +#include "helpers/RadioActivityWindow.h" + +#include + +namespace { + +const uint32_t MINUTE = RADIO_ACTIVITY_BUCKET_MS; +const int N = RADIO_ACTIVITY_BUCKETS; + +// Representative packet: 32 wire bytes, 100 ms airtime, +7.0 dB SNR, -95 dBm. +void recordTypical(RadioActivityWindow& w, uint32_t at_ms, uint16_t bytes = 32) { + w.recordPacket(at_ms, bytes, 100, 28, -95); +} + +RadioActivitySnapshot snapshotAt(RadioActivityWindow& w, uint32_t at_ms) { + RadioActivitySnapshot s; + w.snapshot(at_ms, &s); + return s; +} + +} // namespace + +TEST(RadioActivityWindow, EmptySnapshotHasNoTotalsAndNoDivisionByZero) { + RadioActivityWindow w; + w.reset(0); + + RadioActivitySnapshot s = snapshotAt(w, 0); + + EXPECT_TRUE(s.isEmpty()); + EXPECT_EQ(0u, s.packets); + EXPECT_EQ(0u, s.wire_bytes); + EXPECT_EQ(0u, s.window_ms); + EXPECT_FALSE(s.has_last_packet); + EXPECT_EQ(0u, s.peak_per_min); + + // Every derived value must be defined with a zero denominator. + EXPECT_EQ(0u, s.packetsPerMinuteX10()); + EXPECT_EQ(0u, s.bytesPerSecondX10()); + EXPECT_EQ(0u, s.avgBytesPerPacket()); + EXPECT_EQ(0u, s.airtimePercentX10()); + EXPECT_EQ(0, s.avgSnrX10()); + EXPECT_EQ(0, s.avgRssi()); + + for (int i = 0; i < N; i++) EXPECT_EQ(0u, s.buckets[i]); +} + +TEST(RadioActivityWindow, SingleEventProducesExactTotalsAndRates) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + + RadioActivitySnapshot s = snapshotAt(w, 2000); + + EXPECT_EQ(1u, s.packets); + EXPECT_EQ(32u, s.wire_bytes); + EXPECT_EQ(100u, s.airtime_ms); + EXPECT_EQ(2000u, s.window_ms); + EXPECT_EQ(2000u, s.tracking_ms); + + EXPECT_EQ(300u, s.packetsPerMinuteX10()); // 30.0 packets/min + EXPECT_EQ(160u, s.bytesPerSecondX10()); // 16.0 B/s + EXPECT_EQ(32u, s.avgBytesPerPacket()); + EXPECT_EQ(50u, s.airtimePercentX10()); // 5.0 % + EXPECT_EQ(70, s.avgSnrX10()); // +7.0 dB + EXPECT_EQ(-95, s.avgRssi()); + + EXPECT_TRUE(s.has_last_packet); + EXPECT_EQ(1000u, s.last_packet_age_ms); + + // The current minute is the rightmost bucket. + EXPECT_EQ(1u, s.buckets[N - 1]); + for (int i = 0; i < N - 1; i++) EXPECT_EQ(0u, s.buckets[i]); +} + +TEST(RadioActivityWindow, MultipleEventsInOneMinuteAccumulate) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000, 10); + recordTypical(w, 2000, 20); + recordTypical(w, 3000, 30); + + RadioActivitySnapshot s = snapshotAt(w, 4000); + + EXPECT_EQ(3u, s.packets); + EXPECT_EQ(60u, s.wire_bytes); + EXPECT_EQ(300u, s.airtime_ms); + EXPECT_EQ(20u, s.avgBytesPerPacket()); + EXPECT_EQ(3u, s.buckets[N - 1]); + EXPECT_EQ(3u, s.peak_per_min); + EXPECT_EQ(1000u, s.last_packet_age_ms); +} + +TEST(RadioActivityWindow, EventsRotateIntoTheNextBucketAtTheMinuteBoundary) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 30000); // minute 0 + recordTypical(w, MINUTE); // exactly on the boundary: minute 1 + recordTypical(w, MINUTE + 5000); // minute 1 + + RadioActivitySnapshot s = snapshotAt(w, MINUTE + 10000); + + EXPECT_EQ(3u, s.packets); + EXPECT_EQ(2u, s.buckets[N - 1]); // current minute + EXPECT_EQ(1u, s.buckets[N - 2]); // previous minute + EXPECT_EQ(2u, s.peak_per_min); +} + +TEST(RadioActivityWindow, BucketsAreOrderedOldestToNewest) { + RadioActivityWindow w; + w.reset(0); + + // Minute m gets (m + 1) packets. + for (int m = 0; m < N; m++) { + for (int i = 0; i <= m; i++) recordTypical(w, m * MINUTE + 1000 + i); + } + + RadioActivitySnapshot s = snapshotAt(w, (N - 1) * MINUTE + 30000); + + for (int i = 0; i < N; i++) { + EXPECT_EQ((uint16_t)(i + 1), s.buckets[i]) << "bucket " << i; + } + EXPECT_EQ((uint16_t)N, s.peak_per_min); + EXPECT_EQ((uint32_t)(N * (N + 1) / 2), s.packets); +} + +TEST(RadioActivityWindow, OldestBucketExpiresOnceItLeavesTheWindow) { + RadioActivityWindow w; + w.reset(0); + + for (int m = 0; m < N; m++) recordTypical(w, m * MINUTE + 1000); + + // Still inside the window: all 20 minutes are represented. + RadioActivitySnapshot before = snapshotAt(w, (N - 1) * MINUTE + 59999); + EXPECT_EQ((uint32_t)N, before.packets); + EXPECT_EQ(1u, before.buckets[0]); + + // One tick past the boundary: the oldest minute is gone, and the new current + // minute is empty. + RadioActivitySnapshot after = snapshotAt(w, N * MINUTE); + EXPECT_EQ((uint32_t)(N - 1), after.packets); + EXPECT_EQ(1u, after.buckets[0]); // what was minute 1 + EXPECT_EQ(0u, after.buckets[N - 1]); // the fresh current minute +} + +TEST(RadioActivityWindow, MoreThanTwentyMinutesOfSilenceClearsTheRing) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + + uint32_t now = 21 * MINUTE; + RadioActivitySnapshot s = snapshotAt(w, now); + + EXPECT_TRUE(s.isEmpty()); + for (int i = 0; i < N; i++) EXPECT_EQ(0u, s.buckets[i]); + + // Tracking restarts at the current minute, so the window reports itself as + // warming up again rather than claiming 20 minutes of empty coverage. + EXPECT_EQ(0u, s.tracking_ms); + EXPECT_EQ(0u, s.window_ms); + EXPECT_TRUE(s.isWarmingUp()); + + // The last-packet age survives the ring clear: it is still the most useful + // thing to show when nothing is arriving. + EXPECT_TRUE(s.has_last_packet); + EXPECT_EQ(now - 1000, s.last_packet_age_ms); +} + +TEST(RadioActivityWindow, LastPacketAgeIsDroppedOnceItGoesStale) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + + RadioActivitySnapshot fresh = snapshotAt(w, 1000 + RADIO_ACTIVITY_MAX_AGE_MS); + EXPECT_TRUE(fresh.has_last_packet); + + RadioActivitySnapshot stale = snapshotAt(w, 1000 + RADIO_ACTIVITY_MAX_AGE_MS + 1); + EXPECT_FALSE(stale.has_last_packet); +} + +TEST(RadioActivityWindow, WarmupUsesObservedDurationNotAFixedTwentyMinutes) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 30000); + + // Five minutes in, rates are computed against five minutes, not twenty. + RadioActivitySnapshot warm = snapshotAt(w, 5 * MINUTE); + EXPECT_TRUE(warm.isWarmingUp()); + EXPECT_EQ(5u, warm.warmupMinutes()); + EXPECT_EQ(5 * MINUTE, warm.window_ms); + // 1 packet over 5 minutes is 0.2/min. Against a fixed 1200 s denominator the + // same data would round away to 0.0/min. + EXPECT_EQ(2u, warm.packetsPerMinuteX10()); +} + +TEST(RadioActivityWindow, SteadyStateWindowNeverClaimsMoreCoverageThanTheRingHas) { + RadioActivityWindow w; + w.reset(0); + for (int m = 0; m < 25; m++) recordTypical(w, m * MINUTE + 1000); + + // 19 whole minutes plus the elapsed part of the current one - never 20:00. + RadioActivitySnapshot at_start = snapshotAt(w, 25 * MINUTE); + EXPECT_FALSE(at_start.isWarmingUp()); + EXPECT_EQ(19 * MINUTE, at_start.window_ms); + + RadioActivitySnapshot mid = snapshotAt(w, 25 * MINUTE + 30000); + EXPECT_EQ(19 * MINUTE + 30000, mid.window_ms); + + RadioActivitySnapshot late = snapshotAt(w, 25 * MINUTE + 59999); + EXPECT_EQ(19 * MINUTE + 59999, late.window_ms); + EXPECT_LT(late.window_ms, (uint32_t)N * MINUTE); +} + +TEST(RadioActivityWindow, PeakIsTheBusiestVisibleMinute) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + for (int i = 0; i < 7; i++) recordTypical(w, MINUTE + 1000 + i); + recordTypical(w, 2 * MINUTE + 1000); + + EXPECT_EQ(7u, snapshotAt(w, 2 * MINUTE + 30000).peak_per_min); + + // Once the busy minute ages out of the ring, so does the peak. + EXPECT_EQ(1u, snapshotAt(w, 21 * MINUTE).peak_per_min); +} + +TEST(RadioActivityWindow, SurvivesMillisRollover) { + const uint32_t base = 0xFFFFF000u; // ~4 s before the 32-bit wrap + RadioActivityWindow w; + w.reset(base); + + recordTypical(w, base + 1000); + + // 65 s later, which is 60904 in wrapped millis(). + uint32_t after_wrap = (uint32_t)(base + 65000); + ASSERT_LT(after_wrap, base) << "test setup must actually cross the wrap"; + recordTypical(w, after_wrap); + + RadioActivitySnapshot s = snapshotAt(w, after_wrap + 1000); + + EXPECT_EQ(2u, s.packets); + EXPECT_EQ(1u, s.buckets[N - 1]); // the post-wrap minute + EXPECT_EQ(1u, s.buckets[N - 2]); // the pre-wrap minute + EXPECT_EQ(66000u, s.window_ms); + EXPECT_EQ(1000u, s.last_packet_age_ms); +} + +TEST(RadioActivityWindow, RolloverDoesNotCorruptTheMinuteBoundary) { + // A boundary derived from now_ms / BUCKET_MS would misplace a minute here, + // because 2^32 is not a whole number of 60000 ms buckets. + const uint32_t base = 0xFFFFFFFFu - 30000u; + RadioActivityWindow w; + w.reset(base); + + for (int m = 0; m < 5; m++) recordTypical(w, (uint32_t)(base + m * MINUTE + 1000)); + + RadioActivitySnapshot s = snapshotAt(w, (uint32_t)(base + 4 * MINUTE + 30000)); + + EXPECT_EQ(5u, s.packets); + for (int i = 0; i < 5; i++) { + EXPECT_EQ(1u, s.buckets[N - 1 - i]) << "minute -" << i; + } + EXPECT_EQ(1u, s.peak_per_min); +} + +TEST(RadioActivityWindow, SurvivesAFullMillisCycleOfContinuousUptime) { + // The always-on dashboard services the tracker every few seconds forever. Past + // 2^32 ms (~49.7 days) a 32-bit tracker age wraps back to a small value, which + // would drop the window into warm-up and divide 20 minutes of traffic by + // seconds - inflating every rate on screen. + RadioActivityWindow w; + w.reset(0); + + const uint32_t STEP = 30000; // two packets per minute bucket + uint32_t now = 0; + for (uint64_t elapsed = 0; elapsed < 0x100000000ull + 10 * MINUTE; elapsed += STEP) { + recordTypical(w, now); + RadioActivitySnapshot tick; + w.snapshot(now, &tick); + now += STEP; + } + + RadioActivitySnapshot s = snapshotAt(w, now); + + EXPECT_FALSE(s.isWarmingUp()) << "must not fall back into warm-up after the wrap"; + EXPECT_GE(s.window_ms, 19 * MINUTE); + EXPECT_LE(s.window_ms, (uint32_t)N * MINUTE); + + // 19 whole minutes at two packets each, plus however much of the current + // minute has elapsed. + EXPECT_GE(s.packets, 38u); + EXPECT_LE(s.packets, 41u); + // Two packets a minute, and it must still read as two. + EXPECT_GE(s.packetsPerMinuteX10(), 15u); + EXPECT_LE(s.packetsPerMinuteX10(), 25u); + + EXPECT_TRUE(s.has_last_packet); + EXPECT_EQ(STEP, s.last_packet_age_ms); +} + +TEST(RadioActivityWindow, StaleLastPacketDoesNotComeBackAfterTheWrap) { + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + + // Serviced continuously, but silent, for more than one full 32-bit cycle. + uint32_t now = 0; + const uint32_t STEP = 60000; + for (uint64_t elapsed = 0; elapsed < 0x100000000ull + 10 * MINUTE; elapsed += STEP) { + RadioActivitySnapshot tick; + w.snapshot(now, &tick); + if (elapsed > RADIO_ACTIVITY_MAX_AGE_MS) { + ASSERT_FALSE(tick.has_last_packet) << "a stale age must never look fresh again"; + } + now += STEP; + } + + RadioActivitySnapshot s = snapshotAt(w, now); + EXPECT_TRUE(s.isEmpty()); + EXPECT_FALSE(s.has_last_packet); +} + +TEST(RadioActivityWindow, SaturatedMinuteDropsFurtherEventsWhole) { + RadioActivityWindow w; + w.reset(0); + + for (uint32_t i = 0; i < 65535; i++) w.recordPacket(1000, 10, 1, 4, -100); + + RadioActivitySnapshot full = snapshotAt(w, 2000); + EXPECT_EQ(65535u, full.packets); + EXPECT_EQ(655350u, full.wire_bytes); + + // Past saturation nothing is counted, so bytes-per-packet stays truthful. + w.recordPacket(1500, 10, 1, 4, -100); + RadioActivitySnapshot after = snapshotAt(w, 2000); + EXPECT_EQ(65535u, after.packets); + EXPECT_EQ(655350u, after.wire_bytes); + EXPECT_EQ(10u, after.avgBytesPerPacket()); +} + +TEST(RadioActivityWindow, AnOlderTimestampDoesNotExpireTheWindow) { + // recordPacket() and snapshot() read millis() at slightly different moments; + // a reading that arrives out of order must cost nothing. + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 5000); + + RadioActivitySnapshot ahead = snapshotAt(w, 10000); + ASSERT_EQ(1u, ahead.packets); + + recordTypical(w, 9000); // stale reading, 1 s behind the last snapshot + RadioActivitySnapshot s = snapshotAt(w, 10000); + + EXPECT_EQ(2u, s.packets) << "the ring must not have been cleared"; + EXPECT_EQ(2u, s.buckets[N - 1]); + EXPECT_EQ(10000u, s.window_ms); +} + +TEST(RadioActivityWindow, AveragesHandleNegativeSnrAndMixedSigns) { + RadioActivityWindow w; + w.reset(0); + w.recordPacket(1000, 40, 50, 28, -80); // +7.0 dB + w.recordPacket(1100, 40, 50, -28, -120); // -7.0 dB + + RadioActivitySnapshot s = snapshotAt(w, 2000); + EXPECT_EQ(0, s.avgSnrX10()); + EXPECT_EQ(-100, s.avgRssi()); +} + +TEST(RadioActivityWindow, StaysWithinItsMemoryBudget) { + EXPECT_LE(sizeof(RadioActivityWindow), 1024u); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_touch_tap_detector/test_touch_tap_detector.cpp b/test/test_touch_tap_detector/test_touch_tap_detector.cpp new file mode 100644 index 00000000..656b5b42 --- /dev/null +++ b/test/test_touch_tap_detector/test_touch_tap_detector.cpp @@ -0,0 +1,123 @@ +#include "helpers/ui/TouchTapDetector.h" + +#include + +namespace { + +// Drives the detector at the firmware's 50 ms poll cadence. +const uint32_t POLL = 50; + +// Holds `pressed` for `ms`, returning how many taps were accepted. +int hold(TouchTapDetector& d, uint32_t& now, bool pressed, uint32_t ms) { + int taps = 0; + for (uint32_t t = 0; t < ms; t += POLL) { + if (d.update(now, pressed)) taps++; + now += POLL; + } + return taps; +} + +} // namespace + +TEST(TouchTapDetector, IdlePanelNeverTaps) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + EXPECT_EQ(0, hold(d, now, false, 5000)); + EXPECT_FALSE(d.isTouched()); +} + +TEST(TouchTapDetector, OneTouchProducesExactlyOneTap) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + EXPECT_EQ(1, hold(d, now, true, 300)); + EXPECT_TRUE(d.isTouched()); + EXPECT_EQ(0, hold(d, now, false, 300)); + EXPECT_FALSE(d.isTouched()); +} + +TEST(TouchTapDetector, HoldingAFingerDownDoesNotRepeat) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(0, hold(d, now, true, 10000)) << "a long press must not toggle repeatedly"; +} + +TEST(TouchTapDetector, ContactShorterThanTheDebounceWindowIsIgnored) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + // A single 30 ms blip, below TOUCH_TAP_DEBOUNCE_MS. + EXPECT_FALSE(d.update(now, true)); + now += 30; + EXPECT_FALSE(d.update(now, false)); + now += 30; + EXPECT_EQ(0, hold(d, now, false, 500)); +} + +TEST(TouchTapDetector, BounceOnContactStillCountsAsOneTap) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + int taps = 0; + for (int i = 0; i < 6; i++) { // chattering edge + if (d.update(now, i % 2 == 0)) taps++; + now += 10; + } + taps += hold(d, now, true, 200); // then settles down + EXPECT_EQ(1, taps); +} + +TEST(TouchTapDetector, SecondTapTooSoonIsSuppressed) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(0, hold(d, now, false, 100)); + EXPECT_EQ(0, hold(d, now, true, 100)) << "inside TOUCH_TAP_MIN_GAP_MS"; +} + +TEST(TouchTapDetector, DeliberateSecondTapIsAccepted) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + + EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(0, hold(d, now, false, 600)); // past the min gap + EXPECT_EQ(1, hold(d, now, true, 100)); +} + +TEST(TouchTapDetector, SurvivesMillisRollover) { + TouchTapDetector d; + const uint32_t start = 0xFFFFFF9Bu; // ~100 ms short of the 32-bit wrap + uint32_t now = start; + d.reset(now); + + EXPECT_EQ(1, hold(d, now, true, 200)); // the touch itself crosses the wrap + ASSERT_LT(now, start) << "test setup must actually wrap"; + EXPECT_EQ(0, hold(d, now, false, 600)); + EXPECT_EQ(1, hold(d, now, true, 200)); +} + +TEST(TouchTapDetector, ResetClearsPendingState) { + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + EXPECT_EQ(1, hold(d, now, true, 200)); + + d.reset(now); + EXPECT_FALSE(d.isTouched()); + EXPECT_EQ(1, hold(d, now, true, 200)) << "a fresh probe starts clean"; +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.cpp b/variants/heltec_v4_r8/HeltecV4R8Board.cpp index 34d6b992..9a07db92 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.cpp +++ b/variants/heltec_v4_r8/HeltecV4R8Board.cpp @@ -12,16 +12,15 @@ void HeltecV4R8Board::begin() { loRaFEMControl.init(); - // GPIO 21 is shared by LCD_RST and TP_RST. Let ST7789LCDDisplay own the - // reset sequence; no separate touch reset is needed. -#ifdef PIN_TOUCH_RST - pinMode(PIN_TOUCH_RST, OUTPUT); - digitalWrite(PIN_TOUCH_RST, HIGH); - delay(10); - digitalWrite(PIN_TOUCH_RST, LOW); - delay(100); - digitalWrite(PIN_TOUCH_RST, HIGH); -#endif + // Expansion Kit V2 display/touch pins, verified against Heltec's + // Expansion_board_V2.03 schematic and the V4-R8 datasheet pinout: + // GPIO 17/18 TP_SDA / TP_SCL - the module's OLED_SDA/OLED_SCL I2C bus + // GPIO 21 LCD_RST *and* TP_RST on one net (the module's OLED_RST) + // GPIO 43 TP_INT, optional via R13, and also U0TXD + // GPIO 44 LCD_LEDK backlight, also U0RXD + // ST7789LCDDisplay owns GPIO 21, so there is no separate touch reset to do + // here - and because that net is shared, it must not be parked low while the + // display is off or the touch controller is held in reset with it. esp_reset_reason_t reason = esp_reset_reason(); if (reason == ESP_RST_DEEPSLEEP) { diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index 9025d422..ec509b0a 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -335,6 +335,14 @@ build_flags = -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm -D WITH_SNMP=1 -D DISPLAY_REDRAW_ON_CHANGE=1 + -D DISPLAY_ACTIVITY_DASHBOARD=1 +; Blanking is a runtime setting now: `set display.timeout `, 0 = stay on, +; 60 s default. Tap the Expansion Kit panel or press USER to toggle by hand. + -D DISPLAY_TOUCH_TOGGLE=1 +; Diagnostic: logs the raw CHSC6X frame (and TP_INT) when it changes. Drop this +; and PIN_TOUCH_INT once touch is confirmed working. + -D DISPLAY_TOUCH_DEBUG=1 + -D PIN_TOUCH_INT=43 build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + @@ -402,6 +410,14 @@ build_flags = -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm -D WITH_SNMP=1 -D DISPLAY_REDRAW_ON_CHANGE=1 + -D DISPLAY_ACTIVITY_DASHBOARD=1 +; Blanking is a runtime setting now: `set display.timeout `, 0 = stay on, +; 60 s default. Tap the Expansion Kit panel or press USER to toggle by hand. + -D DISPLAY_TOUCH_TOGGLE=1 +; Diagnostic: logs the raw CHSC6X frame (and TP_INT) when it changes. Drop this +; and PIN_TOUCH_INT once touch is confirmed working. + -D DISPLAY_TOUCH_DEBUG=1 + -D PIN_TOUCH_INT=43 build_src_filter = ${heltec_v4_r8_tft.build_src_filter} + + From 86c4849e55c11af0130adc188d574b67c5dd4a16 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 14:26:52 -0700 Subject: [PATCH 43/93] fix(display): correct R8 touch decode, display wake and power-off Four defects found by hardware testing of the Expansion Kit V2. Touch never registered. The panel's controller does not use the point-count encoding the reference CHSC6X drivers document: byte 0 reads 0x00 idle and 0x1F while a finger is down, so testing for a count of 1 never fired. A partly-failed read leaves 0xFF, which must not count as a press either, so the test is now != 0x00 && != 0xFF. Touch polling could stall the UI loop for ~1 s at a time. The controller NACKs its address whenever it has nothing to report, and calling requestFrom() unconditionally logged a bus error on every 50 ms poll and, once the bus wedged, burned a full ESP_ERR_TIMEOUT inside loop(). Probe the address first, which reports the same NACK quietly, and bound the read with setTimeOut(). The display could not be woken once it blanked; only RST brought it back. turnOn() re-ran the whole display.init(), which re-enters SPI setup, spends ~500 ms in Adafruit's reset delays and pulses GPIO 21 - the line shared with TP_RST, so it reset the touch controller on every wake. Since turnOff() no longer parks that line low, the panel stays configured while dark and waking is just the backlight. Toggling also clears the refresh deadline so the current frame is drawn immediately instead of the stale one. Power-off rebooted instead of staying off. powerOff() went through enterDeepSleep(), which always arms an ext1 wake on P_LORA_DIO_1; a deep-sleep wake is a full reboot, so a node in live traffic restarted within seconds of showing "Turning OFF". It now disables every wake source, so the node stays down until RST or a power cycle. Note that the display off/on cycle had never been exercised on this board before: observer builds pinned AUTO_OFF_MILLIS=0, so the panel never blanked until display.timeout made it a runtime setting. --- examples/simple_repeater/UITask.cpp | 1 + examples/simple_room_server/UITask.cpp | 1 + src/helpers/ui/CHSC6XTouch.h | 46 ++++++++++++++++------- src/helpers/ui/ST7789LCDDisplay.cpp | 17 +++++++++ src/helpers/ui/ST7789LCDDisplay.h | 3 ++ variants/heltec_v4_r8/HeltecV4R8Board.cpp | 14 ++++++- 6 files changed, 68 insertions(+), 14 deletions(-) diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index ce46b9c7..70fa7891 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -320,6 +320,7 @@ void UITask::toggleDisplay() { #ifdef DISPLAY_ACTIVITY_DASHBOARD _rows_valid = false; #endif + _next_refresh = 0; // redraw at once rather than showing the stale frame _auto_off = millis() + displayTimeoutMillis(); } #endif diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 3f124cb1..2e9818e5 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -297,6 +297,7 @@ void UITask::toggleDisplay() { #ifdef DISPLAY_ACTIVITY_DASHBOARD _rows_valid = false; #endif + _next_refresh = 0; // redraw at once rather than showing the stale frame _auto_off = millis() + displayTimeoutMillis(); } #endif diff --git a/src/helpers/ui/CHSC6XTouch.h b/src/helpers/ui/CHSC6XTouch.h index c4c5cd2e..fc8f119f 100644 --- a/src/helpers/ui/CHSC6XTouch.h +++ b/src/helpers/ui/CHSC6XTouch.h @@ -18,7 +18,12 @@ #endif #define CHSC6X_READ_LEN 5 -#define CHSC6X_MAX_POINTS 1 + +// Bounds a single read. The shared bus can wedge, and the ESP32 default lets a +// failed transfer burn ~1 s inside the UI loop. +#ifndef CHSC6X_I2C_TIMEOUT_MS +#define CHSC6X_I2C_TIMEOUT_MS 10 +#endif class CHSC6XTouch { public: @@ -63,32 +68,47 @@ private: TouchTapDetector _detector; bool readPressed() { + // Probe the address first. This controller NACKs while it has nothing to + // report, and going straight to requestFrom() turns that into a logged bus + // error every poll plus, once the bus wedges, a ~1 s timeout stall in the + // UI loop. endTransmission() reports the same NACK quietly and cheaply. + _wire->beginTransmission((uint8_t)CHSC6X_I2C_ADDR); + if (_wire->endTransmission() != 0) return false; + + const uint16_t prev_timeout = _wire->getTimeOut(); + _wire->setTimeOut(CHSC6X_I2C_TIMEOUT_MS); uint8_t got = _wire->requestFrom((uint8_t)CHSC6X_I2C_ADDR, (uint8_t)CHSC6X_READ_LEN); + uint8_t buf[CHSC6X_READ_LEN]; + for (uint8_t i = 0; i < CHSC6X_READ_LEN; i++) { + buf[i] = i < got ? (uint8_t)_wire->read() : 0xFF; + } + while (_wire->available()) _wire->read(); // drain a short read + _wire->setTimeOut(prev_timeout); + if (got != CHSC6X_READ_LEN) { - while (_wire->available()) _wire->read(); // drain a short read logRaw(got, NULL); return false; } - - uint8_t buf[CHSC6X_READ_LEN]; - for (uint8_t i = 0; i < CHSC6X_READ_LEN; i++) buf[i] = (uint8_t)_wire->read(); logRaw(got, buf); - // buf[0] is the reported touch-point count (buf[2]/buf[4] are x/y). It must - // be tested against a *valid* count, not merely against zero: an idle or - // NACKed read can come back as 0xFF, which "non-zero" reads as a finger - // held down forever - the tap detector then fires once and, seeing no - // release, never fires again. - return buf[0] >= 1 && buf[0] <= CHSC6X_MAX_POINTS; + // Measured on the Expansion Kit V2 panel: byte 0 reads 0x00 while idle and + // 0x1F while a finger is down - not the 0x01 point count the reference + // CHSC6X drivers document, so testing for a count of 1 never fires. A + // partly-failed read leaves 0xFF, which must not register as a press. + return buf[0] != 0x00 && buf[0] != 0xFF; } #ifdef DISPLAY_TOUCH_DEBUG int16_t _logged = -1; + uint8_t _log_budget = 5; // always show the first few frames, then on change - // Logs on change only, so a normal boot stays quiet. + // Mostly logs on change, so a normal boot stays quiet - but the opening + // frames are unconditional so an idle read that never changes is still + // visible in the log. void logRaw(uint8_t got, const uint8_t* buf) { int16_t key = buf ? (int16_t)buf[0] : (int16_t)(-2 - (int16_t)got); - if (key == _logged) return; + if (key == _logged && _log_budget == 0) return; + if (_log_budget > 0) _log_budget--; _logged = key; if (!buf) { diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index bcdaf81c..6ccdd24b 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -50,6 +50,22 @@ ColorVal UIColor::corp_blue = 0x001A; bool ST7789LCDDisplay::begin() { if (!_isOn) { + #ifdef HELTEC_V4_R8_TFT + // turnOff() leaves this panel configured and powered - its reset line is + // shared with the touch controller, so it is never parked low - which makes + // waking just a backlight switch. Re-running the init below would re-enter + // SPI setup and pulse GPIO 21, resetting the touch controller on every wake + // and stalling the UI loop for ~500 ms of reset delays. + if (_panel_ready) { + if (_peripher_power) _peripher_power->claim(); + if (PIN_TFT_LEDA_CTL != -1) { + digitalWrite(PIN_TFT_LEDA_CTL, PIN_TFT_LEDA_CTL_ACTIVE); + } + _isOn = true; + return true; + } + #endif + if (_peripher_power) { _peripher_power->claim(); #ifdef HELTEC_V4_R8_TFT @@ -91,6 +107,7 @@ bool ST7789LCDDisplay::begin() { if (PIN_TFT_LEDA_CTL != -1) { digitalWrite(PIN_TFT_LEDA_CTL, PIN_TFT_LEDA_CTL_ACTIVE); } + _panel_ready = true; #endif _isOn = true; diff --git a/src/helpers/ui/ST7789LCDDisplay.h b/src/helpers/ui/ST7789LCDDisplay.h index f6d7446b..baa72bf0 100644 --- a/src/helpers/ui/ST7789LCDDisplay.h +++ b/src/helpers/ui/ST7789LCDDisplay.h @@ -13,6 +13,9 @@ class ST7789LCDDisplay : public DisplayDriver { #endif Adafruit_ST7789 display; bool _isOn; +#ifdef HELTEC_V4_R8_TFT + bool _panel_ready = false; // panel configured once; wake is backlight-only +#endif uint16_t _color; RefCountedDigitalPin* _peripher_power; diff --git a/variants/heltec_v4_r8/HeltecV4R8Board.cpp b/variants/heltec_v4_r8/HeltecV4R8Board.cpp index 9a07db92..434a83ca 100644 --- a/variants/heltec_v4_r8/HeltecV4R8Board.cpp +++ b/variants/heltec_v4_r8/HeltecV4R8Board.cpp @@ -71,9 +71,21 @@ void HeltecV4R8Board::enterDeepSleep(uint32_t secs, int pin_wake_btn) { } void HeltecV4R8Board::powerOff() { - enterDeepSleep(0); +#if defined(HELTEC_V4_R8_TFT) && defined(DISPLAY_CLASS) + display.turnOff(); +#endif + + // Deliberately NOT enterDeepSleep(): that always arms an ext1 wake on + // P_LORA_DIO_1, so a node sitting in live traffic woke - and a deep-sleep + // wake is a full reboot - within seconds of showing "Turning OFF". With every + // wake source disabled the node stays down until RST or a power cycle, which + // is what asking for power-off means. + esp_sleep_disable_wakeup_source(ESP_SLEEP_WAKEUP_ALL); + loRaFEMControl.setRxModeEnableWhenMCUSleep(); + esp_deep_sleep_start(); } + uint16_t HeltecV4R8Board::getBattMilliVolts() { analogReadResolution(12); From f3c81b559b3c8ab994e7eb8d5093d22cf43f510f Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 14:37:03 -0700 Subject: [PATCH 44/93] feat(display): add runtime display.flip for panel orientation `set display.flip 0|1` (also off/on) turns the panel 180 degrees from its compiled DISPLAY_ROTATION, persisted in MQTTPrefs alongside display.timeout and applied live without a reboot. Adding 2 to the compiled rotation rather than setting an absolute value keeps portrait portrait and landscape landscape, so the DisplayViewport geometry never changes with it and the setting cannot produce a nonsensical mix. DisplayDriver gains a defaulted no-op setFlipped(), so no other display driver is affected. The compiled rotation was verified identical across `pio run` and `build.sh` on two machines (movi a11, 2 at the setRotation call site), yet the panel read upside down for one tester and upright for another - which is what a board mounted either way up looks like. No single compiled constant satisfies both, so orientation becomes a setting rather than another rebuild. Runtime-only, like display_timeout_secs: LegacyV1MQTTPrefs and the frozen binary payload sizes are untouched, and the JSON group is an append that older firmware skips. --- examples/simple_repeater/UITask.cpp | 19 ++++++++++++ examples/simple_repeater/UITask.h | 2 ++ examples/simple_room_server/UITask.cpp | 19 ++++++++++++ examples/simple_room_server/UITask.h | 2 ++ src/helpers/CommonCLI_Observer.cpp | 14 +++++++++ src/helpers/MQTTDefaults.h | 1 + src/helpers/MQTTPrefsSerializer.h | 13 ++++++--- src/helpers/MQTTPrefsStorage.h | 4 +++ src/helpers/ui/DisplayDriver.h | 2 ++ src/helpers/ui/ST7789LCDDisplay.cpp | 17 +++++++++-- src/helpers/ui/ST7789LCDDisplay.h | 7 +++-- .../test_mqtt_prefs_serializer.cpp | 29 +++++++++++++++++++ 12 files changed, 120 insertions(+), 9 deletions(-) diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index 70fa7891..f845577c 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -32,6 +32,23 @@ static inline bool millisReached(unsigned long now, unsigned long deadline) { return (int32_t)((uint32_t)now - (uint32_t)deadline) >= 0; } +// Applies `display.flip` when it changes, forcing a complete repaint because +// the panel's existing contents are now the wrong way up. +void UITask::applyDisplayFlip() { +#ifdef WITH_MQTT_BRIDGE + if (_observer_prefs == NULL || _observer_prefs->display_flip == _flip_seen) return; + _flip_seen = _observer_prefs->display_flip; + _display->setFlipped(_flip_seen != 0); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif + _next_refresh = 0; +#endif +} + // `display.timeout` when the observer prefs are available, otherwise the // compiled-in default. Read on every use so a `set display.timeout` takes // effect immediately. @@ -72,6 +89,7 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi ObserverDashboard::applyDarkPalette(); // retunes UIColor for this target only #endif _display->turnOn(); + applyDisplayFlip(); #ifdef DISPLAY_TOUCH_TOGGLE _touch.begin(); #endif @@ -407,6 +425,7 @@ void UITask::loop() { // `_auto_off` is only armed on activity, so a timeout changed at runtime has // to restart the countdown here - otherwise 0 -> 60 blanks instantly off a // boot-time deadline, and 60 -> 3600 still blanks at the old 60 s mark. + applyDisplayFlip(); unsigned long timeout = displayTimeoutMillis(); if (timeout != _timeout_seen) { _timeout_seen = timeout; diff --git a/examples/simple_repeater/UITask.h b/examples/simple_repeater/UITask.h index ad6c5de1..65ca1674 100644 --- a/examples/simple_repeater/UITask.h +++ b/examples/simple_repeater/UITask.h @@ -55,6 +55,8 @@ class UITask { MQTTPrefs* _observer_prefs = NULL; #endif unsigned long _timeout_seen = 0; // to notice a live `display.timeout` change + uint8_t _flip_seen = 0xFF; // 0xFF forces the first apply + void applyDisplayFlip(); unsigned long displayTimeoutMillis() const; diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 2e9818e5..998cc7e2 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -31,6 +31,23 @@ static inline bool millisReached(unsigned long now, unsigned long deadline) { return (int32_t)((uint32_t)now - (uint32_t)deadline) >= 0; } +// Applies `display.flip` when it changes, forcing a complete repaint because +// the panel's existing contents are now the wrong way up. +void UITask::applyDisplayFlip() { +#ifdef WITH_MQTT_BRIDGE + if (_observer_prefs == NULL || _observer_prefs->display_flip == _flip_seen) return; + _flip_seen = _observer_prefs->display_flip; + _display->setFlipped(_flip_seen != 0); +#ifdef DISPLAY_REDRAW_ON_CHANGE + _frame_valid = false; +#endif +#ifdef DISPLAY_ACTIVITY_DASHBOARD + _rows_valid = false; +#endif + _next_refresh = 0; +#endif +} + // `display.timeout` when the observer prefs are available, otherwise the // compiled-in default. Read on every use so a `set display.timeout` takes // effect immediately. @@ -68,6 +85,7 @@ void UITask::begin(NodePrefs* node_prefs, const char* build_date, const char* fi ObserverDashboard::applyDarkPalette(); // retunes UIColor for this target only #endif _display->turnOn(); + applyDisplayFlip(); #ifdef DISPLAY_TOUCH_TOGGLE _touch.begin(); #endif @@ -386,6 +404,7 @@ void UITask::loop() { // `_auto_off` is only armed on activity, so a timeout changed at runtime has // to restart the countdown here - otherwise 0 -> 60 blanks instantly off a // boot-time deadline, and 60 -> 3600 still blanks at the old 60 s mark. + applyDisplayFlip(); unsigned long timeout = displayTimeoutMillis(); if (timeout != _timeout_seen) { _timeout_seen = timeout; diff --git a/examples/simple_room_server/UITask.h b/examples/simple_room_server/UITask.h index 37005e18..96f63ac4 100644 --- a/examples/simple_room_server/UITask.h +++ b/examples/simple_room_server/UITask.h @@ -52,6 +52,8 @@ class UITask { MQTTPrefs* _observer_prefs = NULL; #endif unsigned long _timeout_seen = 0; // to notice a live `display.timeout` change + uint8_t _flip_seen = 0xFF; // 0xFF forces the first apply + void applyDisplayFlip(); unsigned long displayTimeoutMillis() const; diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 4cf66845..9b2b2d30 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -266,6 +266,18 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } } } + } else if (memcmp(config, "display.flip ", 13) == 0) { + const char* val = &config[13]; + if (strcmp(val, "0") == 0 || strcmp(val, "off") == 0) { + _mqtt_prefs.display_flip = 0; + } else if (strcmp(val, "1") == 0 || strcmp(val, "on") == 0) { + _mqtt_prefs.display_flip = 1; + } else { + strcpy(reply, "Error: display.flip must be 0/1 (or off/on)"); + return true; + } + if (!persistObserverPrefs(reply)) return true; + strcpy(reply, _mqtt_prefs.display_flip ? "OK - display rotated 180" : "OK - display upright"); } else if (memcmp(config, "display.timeout ", 16) == 0) { const char* val = &config[16]; bool all_digits = (*val != '\0'); @@ -887,6 +899,8 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf sprintf(reply, "> %d", (uint32_t)_mqtt_prefs.radio_watchdog_minutes); } else if (memcmp(config, "display.timeout", 15) == 0) { sprintf(reply, "> %d", (uint32_t)_mqtt_prefs.display_timeout_secs); + } else if (memcmp(config, "display.flip", 12) == 0) { + strcpy(reply, _mqtt_prefs.display_flip ? "> on" : "> off"); #ifdef WITH_MQTT_BRIDGE } else if (memcmp(config, "mqtt.origin", 11) == 0) { char effective_origin[32]; diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h index 70140133..c390da0b 100644 --- a/src/helpers/MQTTDefaults.h +++ b/src/helpers/MQTTDefaults.h @@ -111,6 +111,7 @@ static inline void applyMQTTDefaults(MQTTPrefs* prefs) { prefs->mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; prefs->display_timeout_secs = DISPLAY_TIMEOUT_DEFAULT_SECS; + prefs->display_flip = 0; } #endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h index 42e56477..f783b52c 100644 --- a/src/helpers/MQTTPrefsSerializer.h +++ b/src/helpers/MQTTPrefsSerializer.h @@ -368,19 +368,24 @@ class MQTTPrefsSerializer : public ConfigSerializer { class DisplayPrefs : public ConfigSerializer { MQTTPrefs* _prefs; - int32_t _timeout_s; - bool _seen_timeout = false; + int32_t _timeout_s, _flip; + bool _seen_timeout = false, _seen_flip = false; protected: - void structure() override { defStrict("timeout_s", _timeout_s, _seen_timeout); } + void structure() override { + defStrict("timeout_s", _timeout_s, _seen_timeout); + defStrict("flip", _flip, _seen_flip); + } public: explicit DisplayPrefs(MQTTPrefs* prefs) - : _prefs(prefs), _timeout_s(prefs->display_timeout_secs) {} + : _prefs(prefs), _timeout_s(prefs->display_timeout_secs), _flip(prefs->display_flip) {} void apply(bool* repaired) { if (_timeout_s < 0 || _timeout_s > DISPLAY_TIMEOUT_MAX_SECS) { _timeout_s = DISPLAY_TIMEOUT_DEFAULT_SECS; *repaired = true; } + if (_flip < 0 || _flip > 1) { _flip = 0; *repaired = true; } _prefs->display_timeout_secs = static_cast(_timeout_s); + _prefs->display_flip = static_cast(_flip); } }; diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h index 1ee4a99f..7401825b 100644 --- a/src/helpers/MQTTPrefsStorage.h +++ b/src/helpers/MQTTPrefsStorage.h @@ -122,6 +122,10 @@ struct MQTTPrefs { // only - deliberately absent from LegacyV1MQTTPrefs, so the frozen binary // layout and its four payload sizes are unchanged. uint16_t display_timeout_secs; + + // Rotate the panel 180 degrees from its compiled DISPLAY_ROTATION, for boards + // mounted the other way up. Runtime only, like display_timeout_secs. + uint8_t display_flip; }; static const uint16_t DISPLAY_TIMEOUT_DEFAULT_SECS = 60; diff --git a/src/helpers/ui/DisplayDriver.h b/src/helpers/ui/DisplayDriver.h index b76a1b6c..74d3f2f6 100644 --- a/src/helpers/ui/DisplayDriver.h +++ b/src/helpers/ui/DisplayDriver.h @@ -27,6 +27,8 @@ public: virtual void turnOff() = 0; virtual void clear() = 0; virtual void startFrame(ColorVal bkg = UIColor::window_bkg) = 0; + // Rotate 180 degrees from the compiled orientation. No-op where unsupported. + virtual void setFlipped(bool flipped) { } virtual void setTextSize(int sz) = 0; virtual void setColor(ColorVal c) = 0; virtual void setCursor(int x, int y) = 0; diff --git a/src/helpers/ui/ST7789LCDDisplay.cpp b/src/helpers/ui/ST7789LCDDisplay.cpp index 6ccdd24b..4d37ebd3 100644 --- a/src/helpers/ui/ST7789LCDDisplay.cpp +++ b/src/helpers/ui/ST7789LCDDisplay.cpp @@ -33,6 +33,19 @@ static DisplayViewport::Geometry portraitViewport(int16_t physical_width, int16_ } #endif +// The compiled orientation, optionally turned 180 degrees by `display.flip`. +// Adding 2 keeps portrait portrait and landscape landscape, so the viewport +// geometry below never has to change with it. +uint8_t ST7789LCDDisplay::effectiveRotation() const { + return (uint8_t)((DISPLAY_ROTATION + (_flipped ? 2 : 0)) & 3); +} + +void ST7789LCDDisplay::setFlipped(bool flipped) { + if (_flipped == flipped) return; + _flipped = flipped; + if (_panel_ready) display.setRotation(effectiveRotation()); +} + bool ST7789LCDDisplay::i2c_probe(TwoWire& wire, uint8_t addr) { return true; } @@ -88,7 +101,7 @@ bool ST7789LCDDisplay::begin() { #endif display.init(DISPLAY_WIDTH, DISPLAY_HEIGHT); - display.setRotation(DISPLAY_ROTATION); + display.setRotation(effectiveRotation()); display.setSPISpeed(40e6); @@ -107,9 +120,9 @@ bool ST7789LCDDisplay::begin() { if (PIN_TFT_LEDA_CTL != -1) { digitalWrite(PIN_TFT_LEDA_CTL, PIN_TFT_LEDA_CTL_ACTIVE); } - _panel_ready = true; #endif + _panel_ready = true; _isOn = true; } diff --git a/src/helpers/ui/ST7789LCDDisplay.h b/src/helpers/ui/ST7789LCDDisplay.h index baa72bf0..24c5a04b 100644 --- a/src/helpers/ui/ST7789LCDDisplay.h +++ b/src/helpers/ui/ST7789LCDDisplay.h @@ -13,9 +13,8 @@ class ST7789LCDDisplay : public DisplayDriver { #endif Adafruit_ST7789 display; bool _isOn; -#ifdef HELTEC_V4_R8_TFT - bool _panel_ready = false; // panel configured once; wake is backlight-only -#endif + bool _panel_ready = false; // panel has been configured at least once + bool _flipped = false; uint16_t _color; RefCountedDigitalPin* _peripher_power; @@ -27,6 +26,7 @@ class ST7789LCDDisplay : public DisplayDriver { void printFitted(const char* str, uint16_t available_width); #endif + uint8_t effectiveRotation() const; bool i2c_probe(TwoWire& wire, uint8_t addr); public: #ifdef USE_PIN_TFT @@ -59,6 +59,7 @@ public: void turnOff() override; void clear() override; void startFrame(ColorVal bkg = UIColor::window_bkg) override; + void setFlipped(bool flipped) override; void setTextSize(int sz) override; void setColor(ColorVal c) override; void setCursor(int x, int y) override; diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp index 8c8bf32c..421ea52b 100644 --- a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -79,6 +79,7 @@ static MQTTPrefs defaults() { prefs.alert_min_interval_min = 60; prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; prefs.display_timeout_secs = DISPLAY_TIMEOUT_DEFAULT_SECS; + prefs.display_flip = 0; strcpy(prefs.snmp_community, "public"); for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { strcpy(prefs.mqtt_slot_preset[i], "none"); @@ -396,6 +397,34 @@ TEST(MQTTPrefsSerializer, DisplayTimeoutRoundTrips) { } } +TEST(MQTTPrefsSerializer, DisplayFlipRoundTripsAndRepairs) { + MQTTPrefs source = defaults(); + EXPECT_EQ(0, source.display_flip); + source.display_flip = 1; + + OutputStream output; + MQTTPrefsSerializer writer(&source); + ASSERT_TRUE(writer.saveSerial(output)); + + MQTTPrefs loaded = defaults(); + InputStream input(output.text()); + MQTTPrefsSerializer reader(&loaded); + ASSERT_TRUE(reader.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(reader.apply(&repaired)); + EXPECT_FALSE(repaired); + EXPECT_EQ(1, loaded.display_flip); + + MQTTPrefs prefs = defaults(); + InputStream bogus("{version:1,display:{flip:7}}"); + MQTTPrefsSerializer bogus_serializer(&prefs); + ASSERT_TRUE(bogus_serializer.loadSerial(bogus)); + repaired = false; + ASSERT_TRUE(bogus_serializer.apply(&repaired)); + EXPECT_TRUE(repaired); + EXPECT_EQ(0, prefs.display_flip); +} + TEST(MQTTPrefsSerializer, RepairsDisplayTimeoutOutOfRange) { MQTTPrefs prefs = defaults(); InputStream input("{version:1,display:{timeout_s:99999}}"); From 018d68063f7300bbe810e9cb8d6cb2a4859a85f3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 14:55:44 -0700 Subject: [PATCH 45/93] fix(display): harden CHSC6X polling and long-gap activity accounting Addresses a Codex review of the preceding three commits. Touch could disable itself for the whole session. begin() latched _present from a single address probe, but this controller NACKs whenever it has nothing to report, so an idle probe at boot was indistinguishable from absent hardware. The probe is now diagnostics only; checkTap() polls regardless, and a NACK costs one quiet bus cycle. Touch no longer mutates shared bus state. Wrapping the read in setTimeOut()/restore was an unsynchronised global write, and the MQTT task drives the same Wire through AutoDiscoverRTCClock in its NTP fallback, so it could inherit the short timeout. The address probe alone removes the ESP_ERR_TIMEOUT stalls that motivated it, and it also ran before the timeout was installed, so the transaction most exposed to a wedged bus was unprotected anyway. A single failed read could fake a release. The debounce window was 40 ms against a 50 ms poll, so a state change was confirmed by the very next sample and one NACK mid-touch produced a release followed by a second tap. It is now 80 ms - two consecutive consistent reads. RadioActivityWindow froze after a gap longer than ~24.8 days. tick() treated any delta past the signed halfway mark as an out-of-order timestamp, so a node left unserviced that long kept a month-old packet in the 20-minute window and reported it as recently received. A backwards step is now only believed when it is small, which is what an out-of-order reading between two call sites actually looks like. Not changed: a downgrade that then saves prefs drops the display group, since /mqtt.json is rewritten from the older serializer's known schema. That is inherent to every appended field in this format, and bumping the JSON version would be worse - older firmware would reject the file rather than ignore one group. Documented instead. --- src/helpers/RadioActivityWindow.h | 28 ++++++++++------ src/helpers/ui/CHSC6XTouch.h | 23 ++++++------- src/helpers/ui/TouchTapDetector.h | 6 +++- .../test_radio_activity_window.cpp | 17 ++++++++++ .../test_touch_tap_detector.cpp | 32 ++++++++++++++----- 5 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/helpers/RadioActivityWindow.h b/src/helpers/RadioActivityWindow.h index 36032b75..ddd8547c 100644 --- a/src/helpers/RadioActivityWindow.h +++ b/src/helpers/RadioActivityWindow.h @@ -22,6 +22,10 @@ // as a stale or (after a 49-day rollover) nonsensical age. #define RADIO_ACTIVITY_MAX_AGE_MS (100UL * RADIO_ACTIVITY_BUCKET_MS) +// How far a timestamp may run behind the previous one and still count as an +// out-of-order reading rather than a very long forward gap. +#define RADIO_ACTIVITY_BACKSTEP_TOLERANCE_MS 5000UL + struct RadioActivitySnapshot { uint32_t packets; uint32_t wire_bytes; @@ -166,16 +170,22 @@ private: uint8_t _head; // ring index of the current minute bool _ever_received; - // Accumulates the delta since the previous call, which is correct across one - // millis() wrap. Read as signed so a caller handing back a slightly older - // reading counts as no time passing, rather than as a ~49-day leap forward - // that would expire the whole ring. Successive calls must therefore be less - // than 2^31 ms (~24.8 days) apart - guaranteed while the tracker is being - // serviced, and when it is not the ring is empty anyway. + // Accumulates the unsigned delta since the previous call, which is correct + // across one millis() wrap. + // + // A delta past the halfway mark is ambiguous: it is either a slightly stale + // reading or a genuine gap of more than ~24.8 days. recordPacket() and + // snapshot() sample millis() microseconds apart, so a real stale reading is + // tiny - anything larger is treated as the long gap it is, which matters + // because rejecting it outright would freeze the ring and leave a + // month-old packet looking recently received. void tick(uint32_t now_ms) { - int32_t delta = (int32_t)(now_ms - _last_input_ms); - if (delta <= 0) return; // stale or repeated reading: no time has passed - _now_ms += (uint32_t)delta; + uint32_t delta = now_ms - _last_input_ms; + if (delta > 0x80000000u && + (uint32_t)(_last_input_ms - now_ms) <= RADIO_ACTIVITY_BACKSTEP_TOLERANCE_MS) { + return; // out-of-order reading: no time has passed + } + _now_ms += delta; _last_input_ms = now_ms; } diff --git a/src/helpers/ui/CHSC6XTouch.h b/src/helpers/ui/CHSC6XTouch.h index fc8f119f..a34e9100 100644 --- a/src/helpers/ui/CHSC6XTouch.h +++ b/src/helpers/ui/CHSC6XTouch.h @@ -19,16 +19,12 @@ #define CHSC6X_READ_LEN 5 -// Bounds a single read. The shared bus can wedge, and the ESP32 default lets a -// failed transfer burn ~1 s inside the UI loop. -#ifndef CHSC6X_I2C_TIMEOUT_MS -#define CHSC6X_I2C_TIMEOUT_MS 10 -#endif - class CHSC6XTouch { public: - // Probes the bus. Returns false (and disables itself) when nothing answers, - // so a board without the touch panel simply carries on without it. + // Probes the bus for diagnostics only. The result must NOT gate polling: this + // controller NACKs its address whenever it has nothing to report, so a single + // idle probe at boot is indistinguishable from absent hardware. checkTap() + // keeps polling either way, and a NACK there costs one quiet, fast bus cycle. bool begin(TwoWire& wire = Wire) { _wire = &wire; _wire->beginTransmission((uint8_t)CHSC6X_I2C_ADDR); @@ -58,8 +54,12 @@ public: // True exactly once per new touch. bool checkTap(uint32_t now_ms) { - if (!_present) return false; - return _detector.update(now_ms, readPressed()); + bool pressed = readPressed(); + if (pressed && !_present) { + _present = true; // answered late; the boot probe caught it mid-idle + Serial.println("Touch: CHSC6X responding"); + } + return _detector.update(now_ms, pressed); } private: @@ -75,15 +75,12 @@ private: _wire->beginTransmission((uint8_t)CHSC6X_I2C_ADDR); if (_wire->endTransmission() != 0) return false; - const uint16_t prev_timeout = _wire->getTimeOut(); - _wire->setTimeOut(CHSC6X_I2C_TIMEOUT_MS); uint8_t got = _wire->requestFrom((uint8_t)CHSC6X_I2C_ADDR, (uint8_t)CHSC6X_READ_LEN); uint8_t buf[CHSC6X_READ_LEN]; for (uint8_t i = 0; i < CHSC6X_READ_LEN; i++) { buf[i] = i < got ? (uint8_t)_wire->read() : 0xFF; } while (_wire->available()) _wire->read(); // drain a short read - _wire->setTimeOut(prev_timeout); if (got != CHSC6X_READ_LEN) { logRaw(got, NULL); diff --git a/src/helpers/ui/TouchTapDetector.h b/src/helpers/ui/TouchTapDetector.h index b2a610f4..b7de0976 100644 --- a/src/helpers/ui/TouchTapDetector.h +++ b/src/helpers/ui/TouchTapDetector.h @@ -9,8 +9,12 @@ // elapsed-time comparisons are unsigned subtractions, so millis() rollover is // a non-event. +// Must exceed the caller's poll interval, or a state change is confirmed by the +// very next sample and the debounce does nothing. At the 50 ms touch poll this +// requires two consecutive consistent reads, so one NACK or short read during a +// continuous touch cannot fake a release (and therefore a second tap). #ifndef TOUCH_TAP_DEBOUNCE_MS -#define TOUCH_TAP_DEBOUNCE_MS 40 +#define TOUCH_TAP_DEBOUNCE_MS 80 #endif // Ignores a second tap arriving this soon after an accepted one, so a bouncy diff --git a/test/test_radio_activity_window/test_radio_activity_window.cpp b/test/test_radio_activity_window/test_radio_activity_window.cpp index 65f953e4..2628e81c 100644 --- a/test/test_radio_activity_window/test_radio_activity_window.cpp +++ b/test/test_radio_activity_window/test_radio_activity_window.cpp @@ -355,6 +355,23 @@ TEST(RadioActivityWindow, AnOlderTimestampDoesNotExpireTheWindow) { EXPECT_EQ(10000u, s.window_ms); } +TEST(RadioActivityWindow, AGapLongerThanHalfTheMillisRangeStillExpiresTheRing) { + // Display off and no traffic for ~25 days: the elapsed time passes the signed + // halfway mark, which must not be mistaken for an out-of-order reading, or a + // month-old packet would still be sitting in the "last 20 minutes". + RadioActivityWindow w; + w.reset(0); + recordTypical(w, 1000); + ASSERT_EQ(1u, snapshotAt(w, 2000).packets); + + const uint32_t twenty_five_days = 25UL * 24 * 3600 * 1000; + ASSERT_GT(twenty_five_days, 0x80000000u) << "gap must cross the halfway mark"; + + RadioActivitySnapshot s = snapshotAt(w, twenty_five_days); + EXPECT_TRUE(s.isEmpty()); + EXPECT_FALSE(s.has_last_packet); +} + TEST(RadioActivityWindow, AveragesHandleNegativeSnrAndMixedSigns) { RadioActivityWindow w; w.reset(0); diff --git a/test/test_touch_tap_detector/test_touch_tap_detector.cpp b/test/test_touch_tap_detector/test_touch_tap_detector.cpp index 656b5b42..538bdf66 100644 --- a/test/test_touch_tap_detector/test_touch_tap_detector.cpp +++ b/test/test_touch_tap_detector/test_touch_tap_detector.cpp @@ -43,7 +43,7 @@ TEST(TouchTapDetector, HoldingAFingerDownDoesNotRepeat) { uint32_t now = 1000; d.reset(now); - EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(1, hold(d, now, true, 200)); EXPECT_EQ(0, hold(d, now, true, 10000)) << "a long press must not toggle repeatedly"; } @@ -79,9 +79,9 @@ TEST(TouchTapDetector, SecondTapTooSoonIsSuppressed) { uint32_t now = 1000; d.reset(now); - EXPECT_EQ(1, hold(d, now, true, 100)); - EXPECT_EQ(0, hold(d, now, false, 100)); - EXPECT_EQ(0, hold(d, now, true, 100)) << "inside TOUCH_TAP_MIN_GAP_MS"; + EXPECT_EQ(1, hold(d, now, true, 200)); + EXPECT_EQ(0, hold(d, now, false, 150)); // release long enough to be confirmed + EXPECT_EQ(0, hold(d, now, true, 200)) << "inside TOUCH_TAP_MIN_GAP_MS"; } TEST(TouchTapDetector, DeliberateSecondTapIsAccepted) { @@ -89,9 +89,25 @@ TEST(TouchTapDetector, DeliberateSecondTapIsAccepted) { uint32_t now = 1000; d.reset(now); - EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(1, hold(d, now, true, 200)); EXPECT_EQ(0, hold(d, now, false, 600)); // past the min gap - EXPECT_EQ(1, hold(d, now, true, 100)); + EXPECT_EQ(1, hold(d, now, true, 200)); +} + +TEST(TouchTapDetector, ASingleBadSampleDuringATouchDoesNotFakeARelease) { + // A NACK or short read reports "not pressed" for that poll. One of those must + // not confirm a release, or recovery reads as a second tap and the display + // toggles twice on one touch. + TouchTapDetector d; + uint32_t now = 1000; + d.reset(now); + EXPECT_EQ(1, hold(d, now, true, 200)); + + EXPECT_FALSE(d.update(now, false)); + now += POLL; + EXPECT_TRUE(d.isTouched()) << "one bad sample must not confirm a release"; + + EXPECT_EQ(0, hold(d, now, true, 2000)) << "and must not produce a second tap"; } TEST(TouchTapDetector, SurvivesMillisRollover) { @@ -100,10 +116,10 @@ TEST(TouchTapDetector, SurvivesMillisRollover) { uint32_t now = start; d.reset(now); - EXPECT_EQ(1, hold(d, now, true, 200)); // the touch itself crosses the wrap + EXPECT_EQ(1, hold(d, now, true, 250)); // the touch itself crosses the wrap ASSERT_LT(now, start) << "test setup must actually wrap"; EXPECT_EQ(0, hold(d, now, false, 600)); - EXPECT_EQ(1, hold(d, now, true, 200)); + EXPECT_EQ(1, hold(d, now, true, 250)); } TEST(TouchTapDetector, ResetClearsPendingState) { From 3e7e0322e59e2cded25d39f69ce7b7723084362f Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 15:37:09 -0700 Subject: [PATCH 46/93] fix(display): make the USER button click immediate, 3 s hold to power off Power-off now needs a 3 second hold; any shorter press toggles the display. MomentaryButton reports a CLICK for any release short of its threshold, so that single value defines both. The button also felt unreliable - "a brief press doesn't wake it, more often than not". MomentaryButton's multi-click detection withholds a CLICK for MULTI_CLICK_WINDOW_MS (280 ms) after release, and folds a second press arriving inside that window into a DOUBLE_CLICK. Since the handler only acts on CLICK, an impatient second press produced nothing at all: press, see nothing, press again, still nothing. Multi-click is now off for these targets, so CLICK fires on release. Both settings are build flags defaulted in variants/heltec_v4_r8/target.cpp and overridden only on the two TFT observer bases, because the companion builds share this user_btn and do use double/triple click. DISPLAY_TOUCH_DEBUG additionally logs which input caused a toggle ("Display: button -> on"), so any remaining flake can be attributed to the button or to a spurious touch read rather than guessed at. --- examples/simple_repeater/UITask.cpp | 11 ++++++++--- examples/simple_repeater/UITask.h | 2 +- examples/simple_room_server/UITask.cpp | 11 ++++++++--- examples/simple_room_server/UITask.h | 2 +- variants/heltec_v4_r8/platformio.ini | 8 ++++++++ variants/heltec_v4_r8/target.cpp | 14 +++++++++++++- 6 files changed, 39 insertions(+), 9 deletions(-) diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index f845577c..61a7fedd 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -326,12 +326,17 @@ void UITask::updateActivityRows() { #endif #ifdef DISPLAY_TOUCH_TOGGLE -void UITask::toggleDisplay() { +void UITask::toggleDisplay(const char* source) { if (_display->isOn()) { _display->turnOff(); } else { _display->turnOn(); } +#ifdef DISPLAY_TOUCH_DEBUG + Serial.printf("Display: %s -> %s\n", source, _display->isOn() ? "on" : "off"); +#else + (void)source; +#endif #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; // wake draws one complete current frame #endif @@ -348,7 +353,7 @@ void UITask::loop() { int ev = user_btn.check(); if (ev == BUTTON_EVENT_CLICK) { #ifdef DISPLAY_TOUCH_TOGGLE - toggleDisplay(); // same action as tapping the panel + toggleDisplay("button"); // same action as tapping the panel #else if (_display->isOn()) { // TODO: any action ? @@ -392,7 +397,7 @@ void UITask::loop() { unsigned long now = millis(); if (millisReached(now, _next_touch)) { _next_touch = now + TOUCH_POLL_MILLIS; - if (_touch.checkTap(now)) toggleDisplay(); + if (_touch.checkTap(now)) toggleDisplay("touch"); } } #endif diff --git a/examples/simple_repeater/UITask.h b/examples/simple_repeater/UITask.h index 65ca1674..9c7a05ef 100644 --- a/examples/simple_repeater/UITask.h +++ b/examples/simple_repeater/UITask.h @@ -48,7 +48,7 @@ class UITask { CHSC6XTouch _touch; unsigned long _next_touch = 0; - void toggleDisplay(); + void toggleDisplay(const char* source); #endif #ifdef WITH_MQTT_BRIDGE diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 998cc7e2..7b4470d2 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -303,12 +303,17 @@ void UITask::updateActivityRows() { #endif #ifdef DISPLAY_TOUCH_TOGGLE -void UITask::toggleDisplay() { +void UITask::toggleDisplay(const char* source) { if (_display->isOn()) { _display->turnOff(); } else { _display->turnOn(); } +#ifdef DISPLAY_TOUCH_DEBUG + Serial.printf("Display: %s -> %s\n", source, _display->isOn() ? "on" : "off"); +#else + (void)source; +#endif #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; // wake draws one complete current frame #endif @@ -327,7 +332,7 @@ void UITask::loop() { if (btnState != _prevBtnState) { if (btnState == USER_BTN_PRESSED) { // pressed? #ifdef DISPLAY_TOUCH_TOGGLE - toggleDisplay(); // same action as tapping the panel + toggleDisplay("button"); // same action as tapping the panel #else if (_display->isOn()) { // TODO: any action ? @@ -371,7 +376,7 @@ void UITask::loop() { unsigned long now = millis(); if (millisReached(now, _next_touch)) { _next_touch = now + TOUCH_POLL_MILLIS; - if (_touch.checkTap(now)) toggleDisplay(); + if (_touch.checkTap(now)) toggleDisplay("touch"); } } #endif diff --git a/examples/simple_room_server/UITask.h b/examples/simple_room_server/UITask.h index 96f63ac4..04704679 100644 --- a/examples/simple_room_server/UITask.h +++ b/examples/simple_room_server/UITask.h @@ -45,7 +45,7 @@ class UITask { CHSC6XTouch _touch; unsigned long _next_touch = 0; - void toggleDisplay(); + void toggleDisplay(const char* source); #endif #ifdef WITH_MQTT_BRIDGE diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index ec509b0a..cc6d5cc3 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -339,6 +339,10 @@ build_flags = ; Blanking is a runtime setting now: `set display.timeout `, 0 = stay on, ; 60 s default. Tap the Expansion Kit panel or press USER to toggle by hand. -D DISPLAY_TOUCH_TOGGLE=1 +; Hold 3 s to power off; any shorter press toggles the display, acted on the +; moment the button is released rather than 280 ms later. + -D USER_BTN_LONG_PRESS_MILLIS=3000 + -D USER_BTN_MULTICLICK=0 ; Diagnostic: logs the raw CHSC6X frame (and TP_INT) when it changes. Drop this ; and PIN_TOUCH_INT once touch is confirmed working. -D DISPLAY_TOUCH_DEBUG=1 @@ -414,6 +418,10 @@ build_flags = ; Blanking is a runtime setting now: `set display.timeout `, 0 = stay on, ; 60 s default. Tap the Expansion Kit panel or press USER to toggle by hand. -D DISPLAY_TOUCH_TOGGLE=1 +; Hold 3 s to power off; any shorter press toggles the display, acted on the +; moment the button is released rather than 280 ms later. + -D USER_BTN_LONG_PRESS_MILLIS=3000 + -D USER_BTN_MULTICLICK=0 ; Diagnostic: logs the raw CHSC6X frame (and TP_INT) when it changes. Drop this ; and PIN_TOUCH_INT once touch is confirmed working. -D DISPLAY_TOUCH_DEBUG=1 diff --git a/variants/heltec_v4_r8/target.cpp b/variants/heltec_v4_r8/target.cpp index 0b38531e..b1912d40 100644 --- a/variants/heltec_v4_r8/target.cpp +++ b/variants/heltec_v4_r8/target.cpp @@ -25,7 +25,19 @@ AutoDiscoverRTCClock rtc_clock(fallback_clock); #ifdef DISPLAY_CLASS DISPLAY_CLASS display(&board.periph_power); - MomentaryButton user_btn(PIN_USER_BTN, 1000, true); + #ifndef USER_BTN_LONG_PRESS_MILLIS + #define USER_BTN_LONG_PRESS_MILLIS 1000 + #endif + // Multi-click detection holds a CLICK back for MULTI_CLICK_WINDOW_MS (280 ms) + // after release, and folds a second press inside that window into a + // DOUBLE_CLICK. Targets that only want a plain click set this to 0 so the + // event fires on release instead of being delayed - or swallowed when an + // impatient second press arrives. + #ifndef USER_BTN_MULTICLICK + #define USER_BTN_MULTICLICK 1 + #endif + MomentaryButton user_btn(PIN_USER_BTN, USER_BTN_LONG_PRESS_MILLIS, true, false, + USER_BTN_MULTICLICK); #endif bool radio_init() { From 06656d430829667d277f2d384bed9a3c5d015dcd Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 15:45:19 -0700 Subject: [PATCH 47/93] fix(display): default the R8 portrait panels to DISPLAY_ROTATION=0 Rotation 2 was consistently upside down on the Expansion Kit V2 panel, so 0 becomes the compiled default for both portrait observer targets. display.flip is unchanged and still defaults to off: the compiled constant should be the correct orientation, with the setting reserved for a board mounted the other way up. A node already carrying `display.flip 1` from testing needs `set display.flip 0` after this. Landscape targets are untouched - they take the driver's own DISPLAY_ROTATION default of 3. --- variants/heltec_v4_r8/platformio.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/variants/heltec_v4_r8/platformio.ini b/variants/heltec_v4_r8/platformio.ini index cc6d5cc3..24b834c7 100644 --- a/variants/heltec_v4_r8/platformio.ini +++ b/variants/heltec_v4_r8/platformio.ini @@ -369,7 +369,9 @@ extends = env:heltec_v4_r8_tft_repeater_observer_mqtt build_flags = ${env:heltec_v4_r8_tft_repeater_observer_mqtt.build_flags} -D ST7789_PORTRAIT_PROFILE=1 - -D DISPLAY_ROTATION=2 +; Verified upright on the Expansion Kit V2 panel. `set display.flip 1` turns it +; 180 degrees for a board mounted the other way up. + -D DISPLAY_ROTATION=0 [env:heltec_v4_r8_tft_room_server] extends = heltec_v4_r8_tft @@ -448,7 +450,9 @@ extends = env:heltec_v4_r8_tft_room_server_observer_mqtt build_flags = ${env:heltec_v4_r8_tft_room_server_observer_mqtt.build_flags} -D ST7789_PORTRAIT_PROFILE=1 - -D DISPLAY_ROTATION=2 +; Verified upright on the Expansion Kit V2 panel. `set display.flip 1` turns it +; 180 degrees for a board mounted the other way up. + -D DISPLAY_ROTATION=0 [env:heltec_v4_r8_tft_terminal_chat] extends = heltec_v4_r8_tft From 445eaf8342a111c2236ce0ced26aa0bd653a426e Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 15:54:16 -0700 Subject: [PATCH 48/93] chore(display): log the persisted display.flip state at boot display.flip lives in /mqtt.json, so it survives a reflash and is invisible while someone is chasing a wrong orientation - a node still carrying flip=1 from testing looks exactly like a firmware that was never fixed. Boot now reports "Display: flip off" or "flip on (rotated 180)". --- examples/simple_repeater/UITask.cpp | 3 +++ examples/simple_room_server/UITask.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index 61a7fedd..ee66520f 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -39,6 +39,9 @@ void UITask::applyDisplayFlip() { if (_observer_prefs == NULL || _observer_prefs->display_flip == _flip_seen) return; _flip_seen = _observer_prefs->display_flip; _display->setFlipped(_flip_seen != 0); + // Logged unconditionally: this is persisted config, so it survives a reflash + // and is otherwise invisible when someone is chasing a wrong orientation. + Serial.printf("Display: flip %s\n", _flip_seen ? "on (rotated 180)" : "off"); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; #endif diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 7b4470d2..8dab1cdc 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -38,6 +38,9 @@ void UITask::applyDisplayFlip() { if (_observer_prefs == NULL || _observer_prefs->display_flip == _flip_seen) return; _flip_seen = _observer_prefs->display_flip; _display->setFlipped(_flip_seen != 0); + // Logged unconditionally: this is persisted config, so it survives a reflash + // and is otherwise invisible when someone is chasing a wrong orientation. + Serial.printf("Display: flip %s\n", _flip_seen ? "on (rotated 180)" : "off"); #ifdef DISPLAY_REDRAW_ON_CHANGE _frame_valid = false; #endif From 7ea615dabd9e4670b90daff0e193c12a897a203b Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 28 Aug 2026 16:06:16 -0700 Subject: [PATCH 49/93] docs: document display.timeout and display.flip Adds both settings to docs/cli_commands.md in the existing format, and records the R8 dashboard, display controls and the two hardware fixes in the changelog. The display.flip entry calls out that it is persisted config which survives a firmware update, since a node still carrying flip=1 from testing looks exactly like a firmware whose orientation was never fixed. --- CHANGELOG.md | 5 +++++ docs/cli_commands.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 816ae1e5..d949d468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ upstream MeshCore `dev` branch, which generally pulls in a new MeshCore software ### August 2026 +- **New** — Analytics dashboard on the Heltec V4 R8 Expansion Kit V2 TFT: 20-minute rolling RF activity with a packets-per-minute graph, on a dark padded layout that repaints only the rows that changed 2026-08-28 +- **New** — `display.timeout` (blank after N seconds, 0 = always on) and `display.flip` (rotate 180 degrees), both applied live 2026-08-28 +- **New** — Tap the Expansion Kit V2 panel or press USER to blank/wake the display; hold USER 3 s to power off 2026-08-28 +- **Fix** — Power-off on the V4 R8 armed an RF wake, so a node in live traffic rebooted seconds after showing "Turning OFF" 2026-08-28 +- **Fix** — The V4 R8 TFT could not be woken once blanked, because waking re-ran a full panel init that also reset the touch controller 2026-08-28 - **New** · `platformio` — MQTT observer repeater and room-server configs for Heltec V4 R8 (OLED and TFT) 2026-08-20 ### June 2026 diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 057fc0e2..0c4ddf6a 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -662,6 +662,37 @@ Elsewhere it replies `Err - neighbors not enabled in this build`. If a --- +#### View or change the display timeout +**Usage:** +- `get display.timeout` +- `set display.timeout ` + +**Parameters:** +- `seconds`: `0` to keep the display on permanently, or `1-3600` seconds of inactivity before it blanks + +**Default:** `60` + +**Note:** Observer builds with a display only. The change applies immediately and restarts the +countdown. Tap the panel or press the USER button to wake or blank it by hand. + +--- + +#### Rotate the display 180 degrees +**Usage:** +- `get display.flip` +- `set display.flip ` + +**Parameters:** +- `state`: `0`/`off` (as built) or `1`/`on` (rotated 180 degrees) + +**Default:** `0` + +**Note:** Observer builds with a display only, for a board mounted the other way up. This is +persisted config, so it survives a firmware update - a node that looks upside down after an +update may simply still be carrying `display.flip 1`. The boot log reports the current state. + +--- + #### Enable or disable Multi-Acks support **Usage:** - `get multi.acks` From 3975bc0b01c5727ea93c23e84d958c52099664c3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 1 Sep 2026 16:31:28 -0700 Subject: [PATCH 50/93] feat: abstract MQTT network transport --- examples/simple_repeater/MyMesh.cpp | 26 +- examples/simple_room_server/MyMesh.cpp | 26 +- src/helpers/AlertFaultPolicy.h | 53 +++- src/helpers/AlertReporter.cpp | 13 +- src/helpers/CommonCLI_Observer.cpp | 85 +++-- src/helpers/ESP32Board.cpp | 17 +- src/helpers/NetworkInterface.cpp | 296 ++++++++++++++++++ src/helpers/NetworkInterface.h | 44 +++ src/helpers/NetworkPolicy.h | 32 ++ src/helpers/SNMPAgent.cpp | 9 +- src/helpers/SNMPAgent.h | 3 +- src/helpers/bridges/MQTTBridge.cpp | 250 ++++----------- src/helpers/bridges/MQTTBridge.h | 39 +-- src/helpers/ethernet/ch390/CH390Config.h | 31 ++ .../ethernet/ch390/CH390EthernetInterface.cpp | 19 +- test/README.md | 1 + .../test_alert_fault_policy.cpp | 11 + .../test_network_policy.cpp | 36 +++ variants/thinknode_m7/platformio.ini | 46 ++- 19 files changed, 731 insertions(+), 306 deletions(-) create mode 100644 src/helpers/NetworkInterface.cpp create mode 100644 src/helpers/NetworkInterface.h create mode 100644 src/helpers/NetworkPolicy.h create mode 100644 src/helpers/ethernet/ch390/CH390Config.h create mode 100644 test/test_network_policy/test_network_policy.cpp diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 75d9fb4e..48322c6a 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1,7 +1,12 @@ #include "MyMesh.h" #include +#include #include // for qsort() #include +#include +#if defined(ESP_PLATFORM) && !defined(NETWORK_USE_ETHERNET) +#include +#endif #if defined(WITH_MQTT_NEIGHBORS) #include // kSyncedClockEpoch #endif @@ -1218,11 +1223,13 @@ void MyMesh::begin(FILESYSTEM *fs) { #if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) // First-boot setup portal: raised only when no WiFi has ever been configured, // so an OTA onto a deployed (configured) node can never open an AP. + #if !defined(NETWORK_USE_ETHERNET) if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { char wc_reply[160]; startWebConfig(false, wc_reply); Serial.println(wc_reply); } + #endif #endif radio_driver.setParams(_prefs.freq, _prefs.bw, _prefs.sf, _prefs.cr); @@ -1467,6 +1474,11 @@ void MyMesh::clearStats() { #ifdef WITH_WEBCONFIG bool MyMesh::startWebConfig(bool force_ap, char* reply) { +#if defined(NETWORK_USE_ETHERNET) + (void)force_ap; + strcpy(reply, "Err: webconfig unavailable on Ethernet observer; use serial CLI"); + return true; +#else if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); @@ -1490,6 +1502,7 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { _webconfig->startLanMode(reply); // reports "WiFi not connected" if down } return true; +#endif } bool MyMesh::stopWebConfig(char* reply) { @@ -1523,12 +1536,17 @@ void MyMesh::onConfigBatchEnd() { void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; int wifi_rssi = 0; - if (WiFi.status() == WL_CONNECTED) { - strncpy(ip, WiFi.localIP().toString().c_str(), sizeof(ip) - 1); - wifi_rssi = WiFi.RSSI(); - } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { + NetworkInterface& network = activeNetworkInterface(); + if (network.isConnected()) { + strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); + const int signal = network.rssi(); + wifi_rssi = signal == INT_MIN ? 0 : signal; + } +#if !defined(NETWORK_USE_ETHERNET) + else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); } +#endif int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index e6bed7d6..90d5ea4b 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1,6 +1,11 @@ #include "MyMesh.h" #include +#include #include +#include +#if defined(ESP_PLATFORM) && !defined(NETWORK_USE_ETHERNET) +#include +#endif #if defined(WITH_MQTT_NEIGHBORS) #include // kSyncedClockEpoch #endif @@ -1020,11 +1025,13 @@ void MyMesh::begin(FILESYSTEM *fs) { #if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) // First-boot setup portal: raised only when no WiFi has ever been configured, // so an OTA onto a deployed (configured) node can never open an AP. + #if !defined(NETWORK_USE_ETHERNET) if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { char wc_reply[160]; startWebConfig(false, wc_reply); Serial.println(wc_reply); } + #endif #endif } @@ -1269,6 +1276,11 @@ void MyMesh::formatPacketStatsReply(char *reply) { #ifdef WITH_WEBCONFIG bool MyMesh::startWebConfig(bool force_ap, char* reply) { +#if defined(NETWORK_USE_ETHERNET) + (void)force_ap; + strcpy(reply, "Err: webconfig unavailable on Ethernet observer; use serial CLI"); + return true; +#else if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); @@ -1292,6 +1304,7 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { _webconfig->startLanMode(reply); // reports "WiFi not connected" if down } return true; +#endif } bool MyMesh::stopWebConfig(char* reply) { @@ -1325,12 +1338,17 @@ void MyMesh::onConfigBatchEnd() { void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; int wifi_rssi = 0; - if (WiFi.status() == WL_CONNECTED) { - strncpy(ip, WiFi.localIP().toString().c_str(), sizeof(ip) - 1); - wifi_rssi = WiFi.RSSI(); - } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { + NetworkInterface& network = activeNetworkInterface(); + if (network.isConnected()) { + strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); + const int signal = network.rssi(); + wifi_rssi = signal == INT_MIN ? 0 : signal; + } +#if !defined(NETWORK_USE_ETHERNET) + else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); } +#endif int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," diff --git a/src/helpers/AlertFaultPolicy.h b/src/helpers/AlertFaultPolicy.h index d15aaf1f..1dbbac49 100644 --- a/src/helpers/AlertFaultPolicy.h +++ b/src/helpers/AlertFaultPolicy.h @@ -220,39 +220,58 @@ static inline void formatAge(uint32_t age_ms, char* out, size_t out_size) { } } -static inline void formatWifiDown(char* out, size_t out_size, uint32_t duration_ms, - uint8_t reason) { +static inline void formatNetworkDown(char* out, size_t out_size, const char* medium, + uint32_t duration_ms, uint8_t reason) { if (!out || out_size == 0) return; char age[16]; formatAge(duration_ms, age, sizeof(age)); if (reason != 0) { - snprintf(out, out_size, "WiFi down %s (reason %u)", age, (unsigned)reason); + snprintf(out, out_size, "%s down %s (reason %u)", medium, age, (unsigned)reason); } else { - snprintf(out, out_size, "WiFi down %s", age); + snprintf(out, out_size, "%s down %s", medium, age); } } +static inline void formatNetworkRecovered(char* out, size_t out_size, + const char* medium, + uint32_t duration_ms) { + if (!out || out_size == 0) return; + char age[16]; + formatAge(duration_ms, age, sizeof(age)); + snprintf(out, out_size, "%s recovered after %s", medium, age); +} + +// Compatibility helpers retained for existing callers and native tests. +static inline void formatWifiDown(char* out, size_t out_size, + uint32_t duration_ms, uint8_t reason) { + formatNetworkDown(out, out_size, "WiFi", duration_ms, reason); +} + static inline void formatWifiRecovered(char* out, size_t out_size, uint32_t duration_ms) { - if (!out || out_size == 0) return; - char age[16]; - formatAge(duration_ms, age, sizeof(age)); - snprintf(out, out_size, "WiFi recovered after %s", age); + formatNetworkRecovered(out, out_size, "WiFi", duration_ms); +} + +static inline bool formatNetworkAlert(char* out, size_t out_size, + const char* medium, const TickResult& r, + const OutageSnapshot& snap) { + const char* label = (medium && *medium) ? medium : "Network"; + if (r.action == Action::FireDown) { + formatNetworkDown(out, out_size, label, r.duration_ms, snap.reason); + return true; + } + if (r.action == Action::FireRecovered) { + formatNetworkRecovered(out, out_size, label, r.duration_ms); + return true; + } + return false; } // Production formatting entry: the same (TickResult, OutageSnapshot) pair // AlertReporter feeds after tick(). Returns false when there is no message. static inline bool formatWifiAlert(char* out, size_t out_size, const TickResult& r, const OutageSnapshot& snap) { - if (r.action == Action::FireDown) { - formatWifiDown(out, out_size, r.duration_ms, snap.reason); - return true; - } - if (r.action == Action::FireRecovered) { - formatWifiRecovered(out, out_size, r.duration_ms); - return true; - } - return false; + return formatNetworkAlert(out, out_size, "WiFi", r, snap); } static inline void formatMqttDown(char* out, size_t out_size, int slot_1based, diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index b55ac90c..171d81c8 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -6,6 +6,7 @@ #include #ifdef WITH_MQTT_BRIDGE #include "AlertFaultPolicy.h" +#include "NetworkInterface.h" #endif // Header layout for PAYLOAD_TYPE_GRP_TXT before encryption: @@ -217,13 +218,21 @@ void AlertReporter::onLoop(unsigned long now_ms) { min_interval_ms); if (r.action == AlertFaultPolicy::Action::FireDown) { char text[80]; - AlertFaultPolicy::formatWifiAlert(text, sizeof(text), r, snap); + AlertFaultPolicy::formatNetworkAlert( + text, sizeof(text), + strcmp(activeNetworkInterface().mediumName(), "ethernet") == 0 + ? "Ethernet" : "WiFi", + r, snap); if (sendChannel(text)) { AlertFaultPolicy::commitDown(_wifi, now, snap.started_ms); } } else if (r.action == AlertFaultPolicy::Action::FireRecovered) { char text[80]; - AlertFaultPolicy::formatWifiAlert(text, sizeof(text), r, snap); + AlertFaultPolicy::formatNetworkAlert( + text, sizeof(text), + strcmp(activeNetworkInterface().mediumName(), "ethernet") == 0 + ? "Ethernet" : "WiFi", + r, snap); sendChannel(text); AlertFaultPolicy::commitRecovered(_wifi); } diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 9b2b2d30..28ac768b 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -15,7 +15,9 @@ #include "TxtDataHelpers.h" #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) +#include "NetworkInterface.h" #include +#include #include #ifdef ESP_PLATFORM #include @@ -414,8 +416,9 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // runs on the Arduino loop task, shared with mesh/radio processing and the // web config batch, so a synchronous wait of up to 30 s would stall the // node. The sync runs in the background; verify with `get mqtt.ntp.diag`. - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "OK - saved (WiFi not connected; NTP sync pending)"); + if (!activeNetworkInterface().isConnected()) { + snprintf(reply, 160, "OK - saved (%s not connected; NTP sync pending)", + activeNetworkInterface().mediumName()); } else if (!_callbacks->isMqttBridgeRunning()) { strcpy(reply, "OK - saved (MQTT bridge not running)"); } else if (_callbacks->syncMqttNtp()) { @@ -457,7 +460,8 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf _mqtt_prefs.wifi_power_save = ps_value; if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM - if (WiFi.status() == WL_CONNECTED) { + if (strcmp(activeNetworkInterface().mediumName(), "wifi") == 0 && + activeNetworkInterface().isConnected()) { wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : (ps_value == 2) ? WIFI_PS_MAX_MODEM : WIFI_PS_MIN_MODEM; esp_err_t ps_result = esp_wifi_set_ps(ps_mode); @@ -958,8 +962,9 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf #ifdef ESP_PLATFORM // Connectivity probe across all configured NTP servers; never updates the clock. // Serial console (sender_timestamp == 0) gets a detailed table; LoRa gets a compact list. - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "Error: WiFi not connected"); + if (!activeNetworkInterface().isConnected()) { + snprintf(reply, 160, "Error: %s not connected", + activeNetworkInterface().mediumName()); } else if (!_callbacks->isMqttBridgeRunning()) { strcpy(reply, "Error: MQTT bridge not running"); } else if (!_callbacks->runMqttNtpDiag(reply, 160, sender_timestamp == 0)) { @@ -1033,20 +1038,28 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf } else { strcpy(reply, _mqtt_prefs.wifi_password[0] ? "> ******** (serial only)" : "> (not set)"); } - } else if (memcmp(config, "wifi.status", 11) == 0) { - wl_status_t status = WiFi.status(); - const char* status_str; - switch (status) { - case WL_CONNECTED: status_str = "connected"; break; - case WL_NO_SSID_AVAIL: status_str = "no_ssid"; break; - case WL_CONNECT_FAILED: status_str = "connect_failed"; break; - case WL_CONNECTION_LOST: status_str = "connection_lost"; break; - case WL_DISCONNECTED: status_str = "disconnected"; break; - case 255: status_str = "not_started"; break; - default: status_str = "unknown"; break; + } else if (memcmp(config, "link.status", 11) == 0 || + memcmp(config, "wifi.status", 11) == 0) { + NetworkInterface& network = activeNetworkInterface(); + const bool wifi_alias = config[0] == 'w'; + if (wifi_alias && strcmp(network.mediumName(), "wifi") != 0) { + snprintf(reply, 160, "> n/a (%s selected; use get link.status)", + network.mediumName()); + return true; } - if (status == WL_CONNECTED) { - sprintf(reply, "> %s, IP: %s, RSSI: %d dBm", status_str, WiFi.localIP().toString().c_str(), WiFi.RSSI()); + const bool connected = network.isConnected(); + if (connected) { + const int signal = network.rssi(); + if (wifi_alias) { + snprintf(reply, 160, "> %s, IP: %s, RSSI: %d dBm", + network.statusName(), network.localIP().toString().c_str(), signal); + } else if (signal == INT_MIN) { + snprintf(reply, 160, "> %s: connected, IP: %s", network.mediumName(), + network.localIP().toString().c_str()); + } else { + snprintf(reply, 160, "> %s: connected, IP: %s, RSSI: %d dBm", + network.mediumName(), network.localIP().toString().c_str(), signal); + } #ifdef WITH_MQTT_BRIDGE unsigned long connect_at = MQTTBridge::getWifiConnectedAtMillis(); if (connect_at != 0) { @@ -1074,19 +1087,35 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf #endif } else { #ifdef WITH_MQTT_BRIDGE - uint8_t reason = MQTTBridge::getLastWifiDisconnectReason(); + uint8_t reason = network.lastDisconnectReason(); if (reason != 0) { const char* desc = MQTTBridge::wifiReasonStr(reason); if (desc) { - sprintf(reply, "> %s: %s (reason: %d)", status_str, desc, reason); + if (wifi_alias) { + sprintf(reply, "> %s: %s (reason: %d)", network.statusName(), desc, reason); + } else { + sprintf(reply, "> %s: %s (reason: %d)", network.mediumName(), desc, reason); + } } else { - sprintf(reply, "> %s: reason %d", status_str, reason); + if (wifi_alias) { + sprintf(reply, "> %s: reason %d", network.statusName(), reason); + } else { + sprintf(reply, "> %s: reason %d", network.mediumName(), reason); + } } } else { - sprintf(reply, "> %s (code: %d)", status_str, status); + if (wifi_alias) { + sprintf(reply, "> %s (code: %d)", network.statusName(), network.statusCode()); + } else { + sprintf(reply, "> %s: %s", network.mediumName(), network.statusName()); + } } #else - sprintf(reply, "> %s (code: %d)", status_str, status); + if (wifi_alias) { + sprintf(reply, "> %s (code: %d)", network.statusName(), network.statusCode()); + } else { + sprintf(reply, "> %s: %s", network.mediumName(), network.statusName()); + } #endif } } else if (memcmp(config, "wifi.powersave", 14) == 0) { @@ -1150,8 +1179,9 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, #ifdef WITH_MQTT_BRIDGE if (memcmp(command, "tls.bundletest ", 15) == 0) { #ifdef ESP_PLATFORM - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "ERR: WiFi not connected"); + if (!activeNetworkInterface().isConnected()) { + snprintf(reply, 160, "ERR: %s not connected", + activeNetworkInterface().mediumName()); } else { size_t bundle_len = 0; if (rootca_crt_bundle_start != nullptr && @@ -1196,8 +1226,9 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, // ota check -> report available build, do not flash // ota update -> download and flash, then reboot #if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "ERR: WiFi not connected"); + if (!activeNetworkInterface().isConnected()) { + snprintf(reply, 160, "ERR: %s not connected", + activeNetworkInterface().mediumName()); } else if (memcmp(command, "ota check", 9) == 0) { // Check is synchronous so its result lands in this reply, and runs with the // MQTT bridge UP: the slim per-variant manifest is tiny, so the fetch only diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 120203ab..5452c50c 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -1,6 +1,7 @@ #ifdef ESP_PLATFORM #include "ESP32Board.h" +#include "NetworkInterface.h" #include #if defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA) // Repeater or Room Server only @@ -14,15 +15,16 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { inhibit_sleep = true; // prevent sleep during OTA - // If the device is already on a WiFi network (e.g. an observer joined in STA - // mode), serve ElegantOTA on the station IP so it's reachable from the LAN - // without joining a separate AP. Otherwise raise the MeshCore-OTA SoftAP. + // If the device is already on its selected network, serve ElegantOTA on that + // address so it is reachable without joining a separate AP. Otherwise raise + // the MeshCore-OTA SoftAP. // force_ap ("start ota ap") always raises the SoftAP, so the OTA UI stays // reachable even when the joined network applies client isolation and the // station IP can't be reached. IPAddress ip; - if (!force_ap && WiFi.status() == WL_CONNECTED) { - ip = WiFi.localIP(); + if (NetworkPolicy::startOtaUsesSelectedNetwork( + force_ap, activeNetworkInterface().isConnected())) { + ip = activeNetworkInterface().localIP(); } else { WiFi.softAP("MeshCore-OTA", NULL); ip = WiFi.softAPIP(); @@ -200,8 +202,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char strcpy(reply, "ERR: OTA not configured (build via build.sh)"); return false; #else - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "ERR: WiFi not connected"); + if (!activeNetworkInterface().isConnected()) { + snprintf(reply, 160, "ERR: %s not connected", + activeNetworkInterface().mediumName()); return false; } diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp new file mode 100644 index 00000000..1d4d2d39 --- /dev/null +++ b/src/helpers/NetworkInterface.cpp @@ -0,0 +1,296 @@ +#include "NetworkInterface.h" + +#if defined(ESP_PLATFORM) + +#include "MQTTConnectionPolicy.h" + +#include +#include +#include + +#include +#include + +#if defined(NETWORK_USE_ETHERNET) +#include "ethernet/ch390/CH390Config.h" +#endif + +namespace { + +class NetworkInterfaceBase : public NetworkInterface { + protected: + std::atomic _outage_bits{AlertFaultPolicy::packOutageSnapshot({false, 0, 0})}; + std::atomic _connected_at{0}; + std::atomic _last_disconnect_time{0}; + std::atomic _last_disconnect_reason{0}; + bool _status_initialized = false; + bool _last_connected = false; + unsigned long _last_status_check = 0; + + AlertFaultPolicy::OutageSnapshot outage() const { + return AlertFaultPolicy::unpackOutageSnapshot( + _outage_bits.load(std::memory_order_acquire)); + } + + void setOutage(AlertFaultPolicy::OutageSnapshot snapshot) { + _outage_bits.store(AlertFaultPolicy::packOutageSnapshot(snapshot), + std::memory_order_release); + } + + void noteConnected(unsigned long now_ms) { + if (_connected_at.load(std::memory_order_relaxed) == 0) { + _connected_at.store(now_ms, std::memory_order_relaxed); + } + setOutage(AlertFaultPolicy::applyWifiGotIp(outage())); + } + + void noteDisconnected(unsigned long now_ms, uint8_t reason) { + _last_disconnect_reason.store(reason, std::memory_order_relaxed); + _last_disconnect_time.store(now_ms, std::memory_order_relaxed); + setOutage(AlertFaultPolicy::applyWifiDisconnectEvent( + (uint32_t)now_ms, reason, outage())); + } + + public: + unsigned long connectedAtMillis() const override { + return _connected_at.load(std::memory_order_relaxed); + } + + uint8_t lastDisconnectReason() const override { + return _last_disconnect_reason.load(std::memory_order_relaxed); + } + + unsigned long lastDisconnectTime() const override { + return _last_disconnect_time.load(std::memory_order_relaxed); + } + + AlertFaultPolicy::OutageSnapshot outageSnapshot() const override { + return outage(); + } +}; + +class WiFiNetworkInterface final : public NetworkInterfaceBase { + bool _event_registered = false; + char _ssid[33] = {}; + char _password[65] = {}; + unsigned long _last_reconnect_attempt = 0; + uint8_t _reconnect_backoff_attempt = 0; + + void applyPowerPrefs(uint8_t wifi_power_save) { + wifi_ps_type_t ps_mode = wifi_power_save == 2 ? WIFI_PS_MAX_MODEM : WIFI_PS_NONE; + esp_wifi_set_ps(ps_mode); +#ifdef MQTT_WIFI_TX_POWER + WiFi.setTxPower(MQTT_WIFI_TX_POWER); +#else + WiFi.setTxPower(WIFI_POWER_11dBm); +#endif + } + + public: + const char* mediumName() const override { return "wifi"; } + const char* statusName() const override { + switch (WiFi.status()) { + case WL_CONNECTED: return "connected"; + case WL_NO_SSID_AVAIL: return "no_ssid"; + case WL_CONNECT_FAILED: return "connect_failed"; + case WL_CONNECTION_LOST: return "connection_lost"; + case WL_DISCONNECTED: return "disconnected"; + case 255: return "not_started"; + default: return "unknown"; + } + } + int statusCode() const override { return (int)WiFi.status(); } + + bool configValid(const char* wifi_ssid) const override { + return wifi_ssid && wifi_ssid[0] != '\0'; + } + + bool begin(const char* wifi_ssid, const char* wifi_password) override { + if (!configValid(wifi_ssid)) return false; + strncpy(_ssid, wifi_ssid, sizeof(_ssid) - 1); + _ssid[sizeof(_ssid) - 1] = '\0'; + strncpy(_password, wifi_password ? wifi_password : "", sizeof(_password) - 1); + _password[sizeof(_password) - 1] = '\0'; + + WiFi.mode(WIFI_STA); + WiFi.setAutoReconnect(true); + WiFi.setAutoConnect(true); + + if (!_event_registered) { + WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { + switch (event) { + case ARDUINO_EVENT_WIFI_STA_GOT_IP: + noteConnected(millis()); + _reconnect_backoff_attempt = 0; + break; + case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: + noteDisconnected(millis(), info.wifi_sta_disconnected.reason); + break; + default: + break; + } + }); + _event_registered = true; + } + + // Preserve the existing restart behavior: MQTT stop leaves the station up, + // and begin() must not force a disconnect that races the first DNS lookup. + if (!isConnected()) { + WiFi.begin(_ssid, _password); + } else { + noteConnected(millis()); + } + return true; + } + + NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) override { + const bool connected = isConnected(); + if (connected && connectedAtMillis() == 0) noteConnected(now_ms); + + if (!_status_initialized) { + _last_connected = connected; + _status_initialized = true; + setOutage(AlertFaultPolicy::applyWifiStatus( + now_ms, connected, outage(), false)); + } + + if ((uint32_t)(now_ms - _last_status_check) <= 10000) { + if (connected && outage().down) noteConnected(now_ms); + return NetworkTransition::None; + } + _last_status_check = now_ms; + + if (connected) { + const bool transitioned = !_last_connected; + if (transitioned) { + setOutage(AlertFaultPolicy::applyWifiStatus( + now_ms, true, outage(), true)); + _connected_at.store(now_ms, std::memory_order_relaxed); + _reconnect_backoff_attempt = 0; + applyPowerPrefs(wifi_power_save); + } + _last_connected = true; + return transitioned ? NetworkTransition::Up : NetworkTransition::None; + } + + AlertFaultPolicy::OutageSnapshot snapshot = AlertFaultPolicy::applyWifiStatus( + now_ms, false, outage(), true); + setOutage(snapshot); + const bool transitioned = _last_connected; + if (transitioned) { + _connected_at.store(0, std::memory_order_relaxed); + } else if (snapshot.down && MQTTConnectionPolicy::wifiReconnectDue( + now_ms, snapshot.started_ms, (uint32_t)_last_reconnect_attempt, + _reconnect_backoff_attempt)) { + _last_reconnect_attempt = now_ms; + _reconnect_backoff_attempt = + MQTTConnectionPolicy::nextWifiBackoffAttempt(_reconnect_backoff_attempt); + WiFi.disconnect(); + WiFi.begin(_ssid, _password); + } + _last_connected = false; + return transitioned ? NetworkTransition::Down : NetworkTransition::None; + } + + bool isConnected() const override { return WiFi.status() == WL_CONNECTED; } + IPAddress localIP() const override { return WiFi.localIP(); } + int rssi() const override { return isConnected() ? WiFi.RSSI() : INT_MIN; } + bool resolveHost(const char* hostname, IPAddress& address) const override { + return WiFi.hostByName(hostname, address); + } +}; + +#if defined(NETWORK_USE_ETHERNET) +class EthernetNetworkInterface final : public NetworkInterfaceBase { + bool _started = false; + bool _event_registered = false; + + public: + const char* mediumName() const override { return "ethernet"; } + const char* statusName() const override { + return isConnected() ? "connected" : "disconnected"; + } + int statusCode() const override { return isConnected() ? 1 : 0; } + bool configValid(const char*) const override { return true; } + + bool begin(const char*, const char*) override { + if (_started) return true; + if (!_event_registered) { + WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t) { + switch (event) { + case ARDUINO_EVENT_ETH_GOT_IP: + noteConnected(millis()); + break; + case ARDUINO_EVENT_ETH_DISCONNECTED: + // Ethernet has no 802.11 reason code; zero means unavailable. + noteDisconnected(millis(), 0); + _connected_at.store(0, std::memory_order_relaxed); + break; + default: + break; + } + }); + _event_registered = true; + } + _started = beginConfiguredCH390(); + if (_started && isConnected()) noteConnected(millis()); + return _started; + } + + NetworkTransition maintain(uint32_t now_ms, uint8_t) override { + const bool connected = isConnected(); + if (!_status_initialized) { + _last_connected = connected; + _status_initialized = true; + setOutage(AlertFaultPolicy::applyWifiStatus( + now_ms, connected, outage(), false)); + if (connected) noteConnected(now_ms); + return NetworkTransition::None; + } + + if (connected == _last_connected) { + if (connected && outage().down) noteConnected(now_ms); + return NetworkTransition::None; + } + + _last_connected = connected; + if (connected) { + _connected_at.store(now_ms, std::memory_order_relaxed); + setOutage(AlertFaultPolicy::applyWifiStatus( + now_ms, true, outage(), true)); + return NetworkTransition::Up; + } + + const bool outage_was_down = outage().down; + _connected_at.store(0, std::memory_order_relaxed); + AlertFaultPolicy::OutageSnapshot snapshot = AlertFaultPolicy::applyWifiStatus( + now_ms, false, outage(), true); + setOutage(snapshot); + if (!outage_was_down) { + _last_disconnect_time.store(now_ms, std::memory_order_relaxed); + } + return NetworkTransition::Down; + } + + bool isConnected() const override { return _started && CH390.isConnected(); } + IPAddress localIP() const override { return CH390.localIP(); } + int rssi() const override { return INT_MIN; } + bool resolveHost(const char* hostname, IPAddress& address) const override { + // Arduino's hostByName is a thin wrapper over the process-wide lwIP resolver; + // DNS follows the selected esp_netif even though this entry point is named WiFi. + return WiFi.hostByName(hostname, address); + } +}; +#endif + +} // namespace + +NetworkInterface& activeNetworkInterface() { +#if defined(NETWORK_USE_ETHERNET) + static EthernetNetworkInterface network; +#else + static WiFiNetworkInterface network; +#endif + return network; +} +#endif diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkInterface.h new file mode 100644 index 00000000..a4e631a1 --- /dev/null +++ b/src/helpers/NetworkInterface.h @@ -0,0 +1,44 @@ +#pragma once + +#include "NetworkPolicy.h" + +#if defined(ESP_PLATFORM) + +#include +#include +#include "AlertFaultPolicy.h" + +/** + * Physical network selected for IP-based services. + * + * The interface owns link bring-up and medium-specific maintenance. MQTT, NTP, + * OTA, and other socket users only consume connectivity and addressing. Normal + * MQTT shutdown deliberately does not stop this interface because an OTA + * download runs after the broker clients have been released. + */ +class NetworkInterface { + public: + virtual ~NetworkInterface() = default; + + virtual const char* mediumName() const = 0; + virtual const char* statusName() const = 0; + virtual int statusCode() const = 0; + virtual bool configValid(const char* wifi_ssid) const = 0; + virtual bool begin(const char* wifi_ssid, const char* wifi_password) = 0; + virtual NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) = 0; + + virtual bool isConnected() const = 0; + virtual IPAddress localIP() const = 0; + virtual int rssi() const = 0; // INT_MIN when the selected medium has no RSSI. + virtual bool resolveHost(const char* hostname, IPAddress& address) const = 0; + + virtual unsigned long connectedAtMillis() const = 0; + virtual uint8_t lastDisconnectReason() const = 0; + virtual unsigned long lastDisconnectTime() const = 0; + virtual AlertFaultPolicy::OutageSnapshot outageSnapshot() const = 0; +}; + +/** Build-selected singleton. Wi-Fi is the compatibility default. */ +NetworkInterface& activeNetworkInterface(); + +#endif diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h new file mode 100644 index 00000000..731ed349 --- /dev/null +++ b/src/helpers/NetworkPolicy.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +enum class NetworkTransition : uint8_t { + None, + Up, + Down, +}; + +namespace NetworkPolicy { + +struct MQTTTransitionActions { + bool disconnect_slots; + bool retry_disconnected_slots_now; +}; + +static constexpr MQTTTransitionActions mqttActions(NetworkTransition transition) { + return { + transition == NetworkTransition::Down, + transition == NetworkTransition::Up, + }; +} + +// `start ota` uses the selected LAN only when reachable and not explicitly +// forced to SoftAP. Manifest OTA has no fallback and checks connectivity itself. +static constexpr bool startOtaUsesSelectedNetwork(bool force_ap, + bool network_connected) { + return !force_ap && network_connected; +} + +} // namespace NetworkPolicy diff --git a/src/helpers/SNMPAgent.cpp b/src/helpers/SNMPAgent.cpp index a70716f5..ddfb2fe7 100644 --- a/src/helpers/SNMPAgent.cpp +++ b/src/helpers/SNMPAgent.cpp @@ -1,7 +1,9 @@ #ifdef WITH_SNMP #include "SNMPAgent.h" +#include "NetworkInterface.h" #include +#include #define SNMP_PORT 161 @@ -67,7 +69,7 @@ void MeshSNMPAgent::begin(const char* community) { void MeshSNMPAgent::loop() { if (!_running) return; - // Update memory and network stats locally (we're on Core 0 with WiFi) + // Update memory and selected-network stats locally on Core 0. _free_heap = (int)ESP.getFreeHeap(); _max_alloc = (int)ESP.getMaxAllocHeap(); _internal_free = (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL); @@ -77,9 +79,8 @@ void MeshSNMPAgent::loop() { _psram_free = 0; #endif - if (WiFi.isConnected()) { - _wifi_rssi = (int)WiFi.RSSI(); - } + const int signal = activeNetworkInterface().rssi(); + _wifi_rssi = signal == INT_MIN ? 0 : signal; _snmp.loop(); } diff --git a/src/helpers/SNMPAgent.h b/src/helpers/SNMPAgent.h index c1bfe0f5..2f792634 100644 --- a/src/helpers/SNMPAgent.h +++ b/src/helpers/SNMPAgent.h @@ -2,7 +2,6 @@ #ifdef WITH_SNMP -#include #include #include @@ -15,7 +14,7 @@ // .2.x.0 = radio (packets, RSSI, SNR, noise floor, air time) // .3.x.0 = mqtt (connected slots, queue depth, skipped publishes) // .4.x.0 = memory (free heap, max alloc, internal free, PSRAM free) -// .5.x.0 = network (WiFi RSSI) +// .5.x.0 = network (RSSI, or 0 when the selected medium has no RSSI) class MeshSNMPAgent { public: diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 39c8f05b..8b48401f 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -100,16 +100,8 @@ void MQTTBridge::getEffectiveMqttOrigin(const NodePrefs* np, const MQTTPrefs* ob applyEffectiveOrigin(np, obs, buf, buf_size); } -// Helper function to check if WiFi credentials are valid -static bool isWiFiConfigValid(const MQTTPrefs* obs) { - // Check if WiFi SSID is configured (not empty) - if (!obs || strlen(obs->wifi_ssid) == 0) { - return false; - } - - // WiFi password can be empty for open networks, so we don't check it - - return true; +static bool isNetworkConfigValid(const MQTTPrefs* obs) { + return obs && activeNetworkInterface().configValid(obs->wifi_ssid); } #ifdef WITH_MQTT_BRIDGE @@ -122,7 +114,7 @@ static bool customEndpointComplete(const char* host, uint16_t port) { } bool MQTTBridge::isConfigValid(const MQTTPrefs* obs) { - if (!obs || !isWiFiConfigValid(obs)) return false; + if (!obs || !isNetworkConfigValid(obs)) return false; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { const char* preset_name = obs->mqtt_slot_preset[i]; if (preset_name[0] == '\0' || strcmp(preset_name, MQTT_PRESET_NONE) == 0) continue; @@ -210,13 +202,6 @@ void* MQTTBridge::JsonScratchAllocator::reallocate(void* ptr, size_t new_size) { return psram_realloc(ptr, new_size); } -// Time (millis()) when WiFi was last seen connected; 0 when disconnected. Used for get wifi.status uptime. -static unsigned long s_wifi_connected_at = 0; - -// Last WiFi disconnect reason (from ESP-IDF event). Used for get wifi.status diagnostics. -static uint8_t s_wifi_disconnect_reason = 0; -static unsigned long s_wifi_disconnect_time = 0; - #ifdef MQTT_MEMORY_DEBUG // #region agent log static void agentLogHeap(const char* location, const char* message, const char* hypothesisId, @@ -236,7 +221,7 @@ static void agentLogHeap(const char* location, const char* message, const char* static MQTTBridge* s_mqtt_bridge_instance = nullptr; unsigned long MQTTBridge::getWifiConnectedAtMillis() { - return s_wifi_connected_at; + return activeNetworkInterface().connectedAtMillis(); } #if defined(WITH_MQTT_NEIGHBORS) @@ -427,8 +412,12 @@ int MQTTBridge::getMaxActiveSlots() { #endif } -uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } -unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } +uint8_t MQTTBridge::getLastWifiDisconnectReason() { + return activeNetworkInterface().lastDisconnectReason(); +} +unsigned long MQTTBridge::getLastWifiDisconnectTime() { + return activeNetworkInterface().lastDisconnectTime(); +} unsigned long MQTTBridge::getSlotCurrentOutageStartMs(int slot_index) const { if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return 0; @@ -635,6 +624,7 @@ static inline uint32_t mqttStopTimeoutForSlots(int slots) { MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mgr, mesh::RTCClock *rtc, mesh::LocalIdentity *identity) : BridgeBase(prefs, mgr, rtc), _obs(obs), + _network(&activeNetworkInterface()), _queue_count(0), _last_status_publish(0), _last_status_retry(0), _status_interval(300000), _ntp_client(_ntp_udp, effectiveNtpPrimary(obs), 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), @@ -661,8 +651,6 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg #ifdef WITH_SNMP _snmp_agent(nullptr), #endif - _last_wifi_check(0), _last_wifi_status(WL_DISCONNECTED), _wifi_status_initialized(false), - _wifi_outage_bits{0}, _last_wifi_reconnect_attempt(0), _wifi_reconnect_backoff_attempt(0), _last_slot_reconnect_ms(0) #ifdef ESP_PLATFORM , _packet_queue_handle(nullptr), _mqtt_task_handle(nullptr), @@ -857,9 +845,10 @@ void MQTTBridge::begin() { _max_active_slots = getMaxActiveSlots(); MQTT_DEBUG_PRINTLN("Max active slots: %d", _max_active_slots); - // Check if WiFi credentials are configured first - if (!isWiFiConfigValid(_obs)) { - MQTT_DEBUG_PRINTLN("MQTT Bridge initialization skipped - WiFi credentials not configured"); + // Ethernet needs no credentials; Wi-Fi preserves the existing SSID gate. + if (!isNetworkConfigValid(_obs)) { + MQTT_DEBUG_PRINTLN("MQTT Bridge initialization skipped - %s is not configured", + _network->mediumName()); return; } @@ -1043,11 +1032,8 @@ void MQTTBridge::begin() { MQTT_DEBUG_PRINTLN("MQTT task created on Core %d", MQTT_TASK_CORE); #else - // Non-ESP32: Initialize WiFi directly (no task) - WiFi.mode(WIFI_STA); - WiFi.setAutoReconnect(true); - WiFi.setAutoConnect(true); - WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); + // Non-ESP32: initialize the selected network directly (no task). + _network->begin(_obs->wifi_ssid, _obs->wifi_password); // NOTE: Slot setup deferred until after NTP sync in loop() #endif @@ -1237,75 +1223,31 @@ void MQTTBridge::mqttTask(void* parameter) { vTaskDelete(nullptr); } -void MQTTBridge::initializeWiFiInTask() { - MQTT_DEBUG_PRINTLN("Initializing WiFi in MQTT task..."); +void MQTTBridge::initializeNetworkInTask() { + MQTT_DEBUG_PRINTLN("Initializing %s network in MQTT task...", _network->mediumName()); - // Initialize WiFi - WiFi.mode(WIFI_STA); - - // Enable automatic reconnection - ESP32 will handle reconnection automatically - WiFi.setAutoReconnect(true); - WiFi.setAutoConnect(true); - - // Set up WiFi event handlers for better diagnostics and immediate disconnection - // detection. Register ONCE — the bridge is reused across restarts (e.g. stopped - // for `ota check`/`ota update`, or `set mqtt…` reconfigure) and WiFi.onEvent() - // never removes prior callbacks, so re-registering leaks handlers and duplicates - // every log line. - if (!_wifi_event_registered) { - WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { - switch(event) { - case ARDUINO_EVENT_WIFI_STA_GOT_IP: - MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); - setWifiOutage(AlertFaultPolicy::applyWifiGotIp(wifiOutage())); - _wifi_reconnect_backoff_attempt = 0; - // Set flag to trigger NTP sync from loop() instead of doing it here - if (!_ntp_synced && !_ntp_sync_pending) { - _ntp_sync_pending = true; - } - break; - case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: { - const uint8_t reason = info.wifi_sta_disconnected.reason; - const unsigned long t = millis(); - s_wifi_disconnect_reason = reason; - s_wifi_disconnect_time = t; - setWifiOutage(AlertFaultPolicy::applyWifiDisconnectEvent( - (uint32_t)t, reason, wifiOutage())); - MQTT_DEBUG_PRINTLN("WiFi disconnected: reason %d", s_wifi_disconnect_reason); - break; - } - default: - break; - } - }); - _wifi_event_registered = true; - } - - // Only (re)start the WiFi association if it isn't already up. end() leaves the - // STA link connected, so on a restart (e.g. after `ota check`) calling - // WiFi.begin() again forces a needless disconnect/reconnect — which also races - // the MQTT task's first DNS lookup (getaddrinfo fails until WiFi/DNS recovers). - // When already connected, the deferred slot setup still fires in mqttTaskLoop() - // because _ntp_synced persists across end() (only _slots_setup_done is reset). - if (WiFi.status() != WL_CONNECTED) { - WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); - } else if (!_ntp_synced && !_ntp_sync_pending) { - _ntp_sync_pending = true; // already connected but never synced — kick NTP now + // begin() is idempotent and deliberately leaves an already-up link alone. + // MQTT end()/begin() cycles therefore keep the transport alive for OTA and do + // not race the first DNS lookup after a bridge restart. + if (!_network->begin(_obs->wifi_ssid, _obs->wifi_password)) { + MQTT_DEBUG_PRINTLN("%s network initialization failed", _network->mediumName()); + } else if (_network->isConnected() && !_ntp_synced && !_ntp_sync_pending) { + _ntp_sync_pending = true; } // NOTE: Slot setup is deferred until after NTP sync in mqttTaskLoop(). // JWT-auth slots need valid timestamps for token creation, and connecting // before NTP sync just wastes heap on TLS handshakes that will be rejected. - MQTT_DEBUG_PRINTLN("WiFi initialization started in task"); + MQTT_DEBUG_PRINTLN("%s network initialization started in task", _network->mediumName()); } // --------------------------------------------------------------------------- // mqttTaskLoop() - main loop running on Core 0 // --------------------------------------------------------------------------- void MQTTBridge::mqttTaskLoop() { - // Initialize WiFi first - initializeWiFiInTask(); + // Initialize the selected physical network first. + initializeNetworkInTask(); // Wait a bit for WiFi to start connecting vTaskDelay(pdMS_TO_TICKS(1000)); @@ -1362,9 +1304,9 @@ void MQTTBridge::mqttTaskLoop() { } #endif - bool wifi_just_connected = handleWiFiConnection(now); - if (wifi_just_connected) { - // WiFi recovered — reset last_reconnect_attempt for disconnected slots so they + bool network_just_connected = handleNetworkConnection(now); + if (network_just_connected) { + // The uplink recovered — reset last_reconnect_attempt for disconnected slots so they // retry immediately rather than waiting up to 5 min for backoff timers to expire. for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].enabled && _slots[i].initial_connect_done && !_slots[i].connected) { @@ -1373,14 +1315,18 @@ void MQTTBridge::mqttTaskLoop() { } } - // Check for pending NTP sync (triggered from WiFi event handler) - if (_ntp_sync_pending && WiFi.status() == WL_CONNECTED) { + // A connected observation is enough to schedule NTP; physical event callbacks + // stay encapsulated in the selected network adapter. + if (!_ntp_synced && _network->isConnected() && !_ntp_sync_pending) { + _ntp_sync_pending = true; + } + if (_ntp_sync_pending && _network->isConnected()) { _ntp_sync_pending = false; syncTimeWithNTP(); } // Retry NTP every 30s if initial sync failed (slots can't start without valid time) - if (!_ntp_synced && WiFi.status() == WL_CONNECTED) { + if (!_ntp_synced && _network->isConnected()) { static unsigned long last_ntp_retry = 0; if (now - last_ntp_retry >= 30000) { last_ntp_retry = now; @@ -1495,7 +1441,7 @@ void MQTTBridge::mqttTaskLoop() { #ifdef WITH_SNMP // SNMP agent loop — process incoming UDP requests if (_snmp_agent) { - if (!_snmp_agent->isRunning() && WiFi.isConnected() && _obs->snmp_enabled) { + if (!_snmp_agent->isRunning() && _network->isConnected() && _obs->snmp_enabled) { _snmp_agent->begin(_obs->snmp_community); MQTT_DEBUG_PRINTLN("SNMP agent started on port 161 (community: %s)", _obs->snmp_community); } @@ -1517,7 +1463,7 @@ void MQTTBridge::mqttTaskLoop() { // Periodic NTP refresh (every hour) — lightweight, non-blocking. // Uses async SNTP instead of the heavy syncTimeWithNTP() which blocks Core 0 // for up to 20+ seconds with DNS lookups, UDP sockets, and retry loops. - if (WiFi.status() == WL_CONNECTED && now - _last_ntp_sync > 3600000) { + if (_network->isConnected() && now - _last_ntp_sync > 3600000) { refreshNTP(); } @@ -2062,7 +2008,7 @@ void MQTTBridge::maintainSlotConnections() { if (!_identity) return; // Check WiFi status first - if (WiFi.status() != WL_CONNECTED) return; + if (!_network->isConnected()) return; unsigned long now_millis = millis(); unsigned long current_time = time(nullptr); @@ -2792,98 +2738,26 @@ void MQTTBridge::checkConfigurationMismatch() { } } -bool MQTTBridge::handleWiFiConnection(unsigned long now) { - wl_status_t current_wifi_status = WiFi.status(); - bool transitioned_to_connected = false; - - if (current_wifi_status == WL_CONNECTED && s_wifi_connected_at == 0) { - s_wifi_connected_at = now; - } - if (!_wifi_status_initialized) { - _last_wifi_status = current_wifi_status; - _wifi_status_initialized = true; - setWifiOutage(AlertFaultPolicy::applyWifiStatus( - (uint32_t)now, current_wifi_status == WL_CONNECTED, wifiOutage(), false)); - } - if (now - _last_wifi_check <= 10000) { - // Events own the snapshot between 10 s polls. If STA is associated again - // and GOT_IP was missed, still close the outage so a flap contained - // between polls does not look like one continuous downtime. - if (current_wifi_status == WL_CONNECTED) { - AlertFaultPolicy::OutageSnapshot snap = wifiOutage(); - if (snap.down) { - setWifiOutage(AlertFaultPolicy::applyWifiGotIp(snap)); +bool MQTTBridge::handleNetworkConnection(unsigned long now) { + const NetworkTransition transition = + _network->maintain((uint32_t)now, _obs->wifi_power_save); + const NetworkPolicy::MQTTTransitionActions actions = + NetworkPolicy::mqttActions(transition); + if (actions.disconnect_slots) { + // Broker ownership stays in the bridge. The physical adapter reports the + // edge; the bridge explicitly closes every slot instead of waiting for + // eventual socket timeouts. + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].client && _slots[i].connected) { + _slots[i].client->disconnect(); } } - return false; } - _last_wifi_check = now; - - if (current_wifi_status == WL_CONNECTED) { - if (_last_wifi_status != WL_CONNECTED) { - transitioned_to_connected = true; - setWifiOutage(AlertFaultPolicy::applyWifiStatus( - (uint32_t)now, true, wifiOutage(), true)); - s_wifi_connected_at = now; - _wifi_reconnect_backoff_attempt = 0; - #ifdef ESP_PLATFORM - wifi_ps_type_t ps_mode; - uint8_t ps_pref = _obs->wifi_power_save; - if (ps_pref == 1) { - ps_mode = WIFI_PS_NONE; - } else if (ps_pref == 2) { - ps_mode = WIFI_PS_MAX_MODEM; - } else { - ps_mode = WIFI_PS_NONE; // default: no power save; eliminates DTIM wake latency on mains-powered bridges - } - esp_wifi_set_ps(ps_mode); - #ifdef MQTT_WIFI_TX_POWER - WiFi.setTxPower(MQTT_WIFI_TX_POWER); - #else - WiFi.setTxPower(WIFI_POWER_11dBm); - #endif - #endif - } - if (s_wifi_connected_at == 0) { - s_wifi_connected_at = now; - } - _last_wifi_status = WL_CONNECTED; - } else { - const bool last_connected = (_last_wifi_status == WL_CONNECTED); - AlertFaultPolicy::OutageSnapshot snap = AlertFaultPolicy::applyWifiStatus( - (uint32_t)now, false, wifiOutage(), true); - setWifiOutage(snap); - if (last_connected) { - s_wifi_connected_at = 0; - // Disconnect all slot clients when WiFi drops - for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { - if (_slots[i].client && _slots[i].connected) { - _slots[i].client->disconnect(); - } - } - } else if (snap.down) { - // Backoff ladder + wrap-safe timing live in MQTTConnectionPolicy (Phase 6), - // exercised by host tests. Behavior is unchanged: both the link-down - // duration and the since-last-attempt interval must clear the current rung - // (elapsedMs is the wrap-safe form of the old ULONG_MAX branch). - if (MQTTConnectionPolicy::wifiReconnectDue( - (uint32_t)now, snap.started_ms, - (uint32_t)_last_wifi_reconnect_attempt, - _wifi_reconnect_backoff_attempt)) { - _last_wifi_reconnect_attempt = now; - _wifi_reconnect_backoff_attempt = - MQTTConnectionPolicy::nextWifiBackoffAttempt(_wifi_reconnect_backoff_attempt); - WiFi.disconnect(); - WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); - } - } - _last_wifi_status = current_wifi_status; - } - return transitioned_to_connected; + return actions.retry_disconnected_slots_now; } bool MQTTBridge::isReady() const { - return _initialized && isWiFiConfigValid(_obs); + return _initialized && isNetworkConfigValid(_obs); } bool MQTTBridge::isIATAValid() const { @@ -2943,10 +2817,10 @@ void MQTTBridge::loop() { return; #else unsigned long now = millis(); - if (handleWiFiConnection(now) && !_ntp_synced) { + if (handleNetworkConnection(now) && !_ntp_synced) { syncTimeWithNTP(); } - if (_ntp_sync_pending && WiFi.status() == WL_CONNECTED) { + if (_ntp_sync_pending && _network->isConnected()) { _ntp_sync_pending = false; syncTimeWithNTP(); } @@ -2986,7 +2860,7 @@ void MQTTBridge::loop() { checkConfigurationMismatch(); // Periodic NTP refresh (every hour) — lightweight, non-blocking. - if (WiFi.status() == WL_CONNECTED && millis() - _last_ntp_sync > 3600000) { + if (_network->isConnected() && millis() - _last_ntp_sync > 3600000) { refreshNTP(); } @@ -3977,8 +3851,8 @@ void MQTTBridge::refreshNTP() { } bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { - if (!WiFi.isConnected()) { - MQTT_DEBUG_PRINTLN("Cannot sync time - WiFi not connected"); + if (!_network->isConnected()) { + MQTT_DEBUG_PRINTLN("Cannot sync time - %s not connected", _network->mediumName()); return false; } @@ -4026,7 +3900,7 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { // Skipping is what keeps the credit honest; the name that answered is the name // recorded. IPAddress resolved_ip; - if (!WiFi.hostByName(server, resolved_ip)) { + if (!_network->resolveHost(server, resolved_ip)) { MQTT_DEBUG_PRINTLN("NTP: %s does not resolve — skipping, not attempting a send", server); continue; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index e2d27253..5038db18 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -2,8 +2,8 @@ #include "MeshCore.h" #include "helpers/bridges/BridgeBase.h" +#include "helpers/NetworkInterface.h" #include -#include #include #include #include @@ -219,11 +219,6 @@ private: bool _ntp_synced; bool _ntp_sync_pending; // Flag to trigger NTP sync from loop() instead of event handler bool _slots_setup_done; // Deferred: slots set up after NTP sync - // WiFi.onEvent() handler registered once and never removed by end(); the bridge - // object is reused across restarts, so re-registering would leak handlers and - // duplicate every connect/disconnect log line. Inline-initialised so it survives - // construction and is NOT reset by end(). - bool _wifi_event_registered = false; int _max_active_slots; // Runtime limit: 5 with PSRAM, 2 without // Pending slot reconfigure: set from CLI (Core 1), processed by MQTT task (Core 0) @@ -406,25 +401,8 @@ private: unsigned long _last_config_warning; // Throttle configuration mismatch warnings static const unsigned long CONFIG_WARNING_INTERVAL = 300000; // Log every 5 minutes max - // WiFi connection state and exponential backoff - unsigned long _last_wifi_check; - wl_status_t _last_wifi_status; - bool _wifi_status_initialized; - // Packed OutageSnapshot; Core 0 (event + MQTT task) stores, Core 1 loads. - std::atomic _wifi_outage_bits; - unsigned long _last_wifi_reconnect_attempt; - uint8_t _wifi_reconnect_backoff_attempt; // 0..5 → 15s, 30s, 60s, 120s, 300s; reset on connect unsigned long _last_slot_reconnect_ms; // guards against concurrent TLS handshakes (15 s inter-slot gap) - AlertFaultPolicy::OutageSnapshot wifiOutage() const { - return AlertFaultPolicy::unpackOutageSnapshot( - _wifi_outage_bits.load(std::memory_order_acquire)); - } - void setWifiOutage(AlertFaultPolicy::OutageSnapshot snap) { - _wifi_outage_bits.store(AlertFaultPolicy::packOutageSnapshot(snap), - std::memory_order_release); - } - // Optional pointers for collecting stats internally (set by mesh if available) mesh::Dispatcher* _dispatcher; // For air times and errors mesh::Radio* _radio; // For noise floor @@ -483,13 +461,13 @@ private: void processPacketQueue(); bool publishStatus(); // Returns true if status was successfully published - bool handleWiFiConnection(unsigned long now); + bool handleNetworkConnection(unsigned long now); // FreeRTOS task function (runs on Core 0) #ifdef ESP_PLATFORM static void mqttTask(void* parameter); void mqttTaskLoop(); // Main loop for MQTT task - void initializeWiFiInTask(); // WiFi initialization moved to task + void initializeNetworkInTask(); // Selected-link initialization moved to task #endif bool publishPacket(mesh::Packet* packet, bool is_tx, bool& has_eligible_target, const uint8_t* raw_data = nullptr, int raw_len = 0, @@ -547,6 +525,7 @@ private: // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // _prefs (held by BridgeBase) still provides upstream fields (freq/sf/node_name…). MQTTPrefs* _obs = nullptr; + NetworkInterface* _network = nullptr; public: MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mgr, mesh::RTCClock *rtc, mesh::LocalIdentity *identity); @@ -655,13 +634,17 @@ public: static unsigned long getWifiConnectedAtMillis(); /** - * Current WiFi outage snapshot for AlertReporter: down, started_ms, and the + * Current selected-network outage snapshot for AlertReporter: down, + * started_ms, and the * initiating disconnect reason. Distinct from getLastWifiDisconnectTime() / * getLastWifiDisconnectReason(), which follow the most recent ESP-IDF * DISCONNECTED event and are overwritten by STA-backoff WiFi.disconnect() * (reason 8 / ASSOC_LEAVE). */ - AlertFaultPolicy::OutageSnapshot getWifiOutageSnapshot() const { return wifiOutage(); } + AlertFaultPolicy::OutageSnapshot getWifiOutageSnapshot() const { + return _network ? _network->outageSnapshot() + : AlertFaultPolicy::OutageSnapshot{false, 0, 0}; + } /** * Per-slot outage accessors used by AlertReporter to detect prolonged @@ -723,7 +706,7 @@ public: uint16_t filter_mask; }; static bool getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out); - /** True when WiFi is set and at least one MQTT slot can run (preset + custom host if needed). */ + /** True when the selected network is configured and at least one MQTT slot can run. */ static bool isConfigValid(const MQTTPrefs* obs); static void formatSlotDiagReply(char* buf, size_t bufsize, int slot_index); static uint8_t getLastWifiDisconnectReason(); diff --git a/src/helpers/ethernet/ch390/CH390Config.h b/src/helpers/ethernet/ch390/CH390Config.h new file mode 100644 index 00000000..1b3a5bd9 --- /dev/null +++ b/src/helpers/ethernet/ch390/CH390Config.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +/** + * Bring up the repository's CH390 lwIP interface from the board build flags. + * Shared by the companion transport wrapper and the observer network adapter so + * pin and static-IP behavior cannot drift between the two paths. + */ +static inline bool beginConfiguredCH390() { + ch390_config_t config = CH390_DEFAULT_CONFIG(); + config.spi_miso_gpio = ETH_MISO_PIN; + config.spi_mosi_gpio = ETH_MOSI_PIN; + config.spi_sck_gpio = ETH_SCLK_PIN; + config.spi_cs_gpio = ETH_CS_PIN; + config.int_gpio = ETH_INT_PIN; + if (!CH390.begin(config)) return false; + +#if defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET) + IPAddress ip(ETHERNET_STATIC_IP); + IPAddress gateway(ETHERNET_STATIC_GATEWAY); + IPAddress subnet(ETHERNET_STATIC_SUBNET); + #if defined(ETHERNET_STATIC_DNS) + IPAddress dns(ETHERNET_STATIC_DNS); + CH390.config(ip, gateway, subnet, dns); + #else + CH390.config(ip, gateway, subnet); + #endif +#endif + return true; +} diff --git a/src/helpers/ethernet/ch390/CH390EthernetInterface.cpp b/src/helpers/ethernet/ch390/CH390EthernetInterface.cpp index 7f696245..ed262bfb 100644 --- a/src/helpers/ethernet/ch390/CH390EthernetInterface.cpp +++ b/src/helpers/ethernet/ch390/CH390EthernetInterface.cpp @@ -1,4 +1,5 @@ #include "CH390EthernetInterface.h" +#include "CH390Config.h" void onWiFiEvent(WiFiEvent_t event) { switch(event){ @@ -29,26 +30,12 @@ bool CH390EthernetInterface::begin() { // listen to ethernet events WiFi.onEvent(onWiFiEvent); - // Init CH390 - ch390_config_t config = CH390_DEFAULT_CONFIG(); - config.spi_miso_gpio = ETH_MISO_PIN; - config.spi_mosi_gpio = ETH_MOSI_PIN; - config.spi_sck_gpio = ETH_SCLK_PIN; - config.spi_cs_gpio = ETH_CS_PIN; - config.int_gpio = ETH_INT_PIN; - if (!CH390.begin(config)) { + // Init CH390 using the same board configuration as the observer uplink. + if (!beginConfiguredCH390()) { ETHERNET_DEBUG_PRINTLN("Failed to initialize CH390 hardware."); return false; } - // Setup Static IP if build flags are present - #if defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET) - IPAddress ip(ETHERNET_STATIC_IP); - IPAddress gw(ETHERNET_STATIC_GATEWAY); - IPAddress sn(ETHERNET_STATIC_SUBNET); - CH390.config(ip, gw, sn); - #endif - // Start Server server.begin(ETHERNET_TCP_PORT); ETHERNET_DEBUG_PRINTLN("listening on TCP port: %d", ETHERNET_TCP_PORT); diff --git a/test/README.md b/test/README.md index 647b6daf..0cba8fff 100644 --- a/test/README.md +++ b/test/README.md @@ -29,6 +29,7 @@ does not reflect the GoogleTest count — run the built binary directly | `test_topic_template` | `src/helpers/MQTTTopicTemplate.h` | `{iata}/{device}/{token}/{type}` expansion, overflow/NUL-termination, and a buffer-size fuzz | | `test_mqtt_topic_router` | `src/helpers/MQTTTopicRouter.h` | complete preset/custom topic-routing contract; MeshRank all types except raw; required identifiers; invalid inputs/slots; exact buffer boundaries | | `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover; WiFi current-outage start sticky across STA reconnect attempts | +| `test_network_policy` | `src/helpers/NetworkPolicy.h` | transport-neutral MQTT link-transition actions and `start ota` selected-LAN versus forced/fallback SoftAP choice | | `test_alert_fault_policy` | `src/helpers/AlertFaultPolicy.h` | WiFi/MQTT fault edge detector; `OutageSnapshot` (down / started_ms / initiating reason) fed to tick and `formatWifiAlert`; reason-8 reconnects change neither duration nor initiating reason; flap between status polls; down at `millis()==0`; packed 64-bit cross-task word; rate-limit floor and first-fire; 5 s poll cadence and `millis()` rollover | | `test_display_viewport` | `src/helpers/ui/DisplayViewport.h`, `src/helpers/ui/DisplayFrameSignature.h` | logical-to-physical portrait mapping; fractional span coverage; fitted-width conversion; preferred/fallback text scaling; stable visible-frame change detection | | `test_mqtt_packet_queue_policy` | `src/helpers/MQTTPacketQueuePolicy.h` | queue-full eviction; stale-disconnect flush; adaptive drain limits; bounded QoS0 retries; exact timing boundaries and 32-bit `millis()` rollover | diff --git a/test/test_alert_fault_policy/test_alert_fault_policy.cpp b/test/test_alert_fault_policy/test_alert_fault_policy.cpp index 42ba27d7..77bdecbd 100644 --- a/test/test_alert_fault_policy/test_alert_fault_policy.cpp +++ b/test/test_alert_fault_policy/test_alert_fault_policy.cpp @@ -350,6 +350,17 @@ TEST(AlertFaultPolicy, FormatWifiAlertUsesSnapshotReason) { EXPECT_STREQ("WiFi down 47m", text); } +TEST(AlertFaultPolicy, FormatNetworkAlertUsesSelectedMediumLabel) { + Alert::Fault f = OkFault(); + const Alert::OutageSnapshot snap = Down(1000, 0); + const Alert::TickResult r = Alert::tick( + f, 1000 + kWifiThresh, snap, kWifiThresh, kMinInterval); + char text[80]; + ASSERT_TRUE(Alert::formatNetworkAlert( + text, sizeof(text), "Ethernet", r, snap)); + EXPECT_STREQ("Ethernet down 30m", text); +} + TEST(AlertFaultPolicy, FormatMqttSlotMessages) { char text[100]; Alert::formatMqttDown(text, sizeof(text), 1, "analyzer-us", 30U * 60000U); diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp new file mode 100644 index 00000000..201c7451 --- /dev/null +++ b/test/test_network_policy/test_network_policy.cpp @@ -0,0 +1,36 @@ +#include + +#include "helpers/NetworkPolicy.h" + +TEST(NetworkPolicy, MqttDownDisconnectsSlotsWithoutRequestingImmediateRetry) { + const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Down); + EXPECT_TRUE(actions.disconnect_slots); + EXPECT_FALSE(actions.retry_disconnected_slots_now); +} + +TEST(NetworkPolicy, MqttUpRequestsImmediateRetryWithoutDisconnectingSlots) { + const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Up); + EXPECT_FALSE(actions.disconnect_slots); + EXPECT_TRUE(actions.retry_disconnected_slots_now); +} + +TEST(NetworkPolicy, NoTransitionHasNoMqttSideEffects) { + const auto actions = NetworkPolicy::mqttActions(NetworkTransition::None); + EXPECT_FALSE(actions.disconnect_slots); + EXPECT_FALSE(actions.retry_disconnected_slots_now); +} + +TEST(NetworkPolicy, StartOtaUsesReachableSelectedNetworkByDefault) { + EXPECT_TRUE(NetworkPolicy::startOtaUsesSelectedNetwork(false, true)); + EXPECT_FALSE(NetworkPolicy::startOtaUsesSelectedNetwork(false, false)); +} + +TEST(NetworkPolicy, StartOtaForceApOverridesAReachableSelectedNetwork) { + EXPECT_FALSE(NetworkPolicy::startOtaUsesSelectedNetwork(true, true)); + EXPECT_FALSE(NetworkPolicy::startOtaUsesSelectedNetwork(true, false)); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index 7da14112..2ccf6f5c 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -37,22 +37,31 @@ build_src_filter = ${esp32_base.build_src_filter} lib_deps = ${esp32_base.lib_deps} stevemarple/MicroNMEA @ ^2.0.6 -[ThinkNode_M7_ethernet] +[ThinkNode_M7_ch390] build_flags = - -D ETHERNET_ENABLED -D ETHERNET_USE_CH390 - -D ETHERNET_CLASS=CH390EthernetInterface -D ETH_MISO_PIN=14 -D ETH_MOSI_PIN=48 -D ETH_SCLK_PIN=47 -D ETH_CS_PIN=21 -D ETH_INT_PIN=45 -D ETHERNET_DEBUG_LOGGING=1 +lib_deps = + https://github.com/liamcottle/ESP32-CH390.git#47b401f1de546118b03c18b5689dedec45871f2d + +; Existing companion/CLI transport overlay. MQTT Ethernet observers consume the +; CH390 fragment above without ETHERNET_ENABLED, whose meaning in the simple +; repeater/room entry points is the legacy nRF52 Ethernet CLI. +[ThinkNode_M7_ethernet] +build_flags = + ${ThinkNode_M7_ch390.build_flags} + -D ETHERNET_ENABLED + -D ETHERNET_CLASS=CH390EthernetInterface build_src_filter = + + lib_deps = - https://github.com/liamcottle/ESP32-CH390.git#47b401f1de546118b03c18b5689dedec45871f2d + ${ThinkNode_M7_ch390.lib_deps} [env:ThinkNode_M7_repeater] extends = ThinkNode_M7 @@ -180,9 +189,10 @@ extends = ThinkNode_M7 build_src_filter = ${ThinkNode_M7.build_src_filter} +<../examples/kiss_modem/> -; MQTT observer envs are WiFi-only: the bridge's link management is bound to the -; WiFi station API, so the onboard CH390 cannot carry MQTT yet. The board has -; PSRAM, so MAX_NEIGHBOURS enables WITH_MQTT_NEIGHBORS (see MQTTBridge.h). +; Wi-Fi remains the default observer transport. Ethernet twins below select the +; onboard CH390 through NETWORK_USE_ETHERNET without enabling the legacy CLI +; transport. The board has PSRAM, so MAX_NEIGHBOURS enables +; WITH_MQTT_NEIGHBORS (see MQTTBridge.h). [env:ThinkNode_M7_repeater_observer_mqtt] extends = ThinkNode_M7 extra_scripts = @@ -227,6 +237,17 @@ lib_deps = paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent +[env:ThinkNode_M7_repeater_observer_mqtt_ethernet] +extends = env:ThinkNode_M7_repeater_observer_mqtt +build_flags = + ${env:ThinkNode_M7_repeater_observer_mqtt.build_flags} + ${ThinkNode_M7_ch390.build_flags} + -D NETWORK_USE_ETHERNET=1 +build_src_filter = ${env:ThinkNode_M7_repeater_observer_mqtt.build_src_filter} +lib_deps = + ${env:ThinkNode_M7_repeater_observer_mqtt.lib_deps} + ${ThinkNode_M7_ch390.lib_deps} + [env:ThinkNode_M7_room_server_observer_mqtt] extends = ThinkNode_M7 extra_scripts = @@ -271,3 +292,14 @@ lib_deps = JChristensen/Timezone paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent + +[env:ThinkNode_M7_room_server_observer_mqtt_ethernet] +extends = env:ThinkNode_M7_room_server_observer_mqtt +build_flags = + ${env:ThinkNode_M7_room_server_observer_mqtt.build_flags} + ${ThinkNode_M7_ch390.build_flags} + -D NETWORK_USE_ETHERNET=1 +build_src_filter = ${env:ThinkNode_M7_room_server_observer_mqtt.build_src_filter} +lib_deps = + ${env:ThinkNode_M7_room_server_observer_mqtt.lib_deps} + ${ThinkNode_M7_ch390.lib_deps} From 50e2b2acbae5a9431f8c143f6d71b7ea98299ed0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 1 Sep 2026 21:18:22 -0700 Subject: [PATCH 51/93] feat: add automatic network failover --- MQTT_IMPLEMENTATION.md | 7 +- examples/simple_repeater/MyMesh.cpp | 41 ++-- examples/simple_repeater/MyMesh.h | 7 + examples/simple_repeater/UITask.cpp | 5 +- examples/simple_room_server/MyMesh.cpp | 41 ++-- examples/simple_room_server/MyMesh.h | 7 + examples/simple_room_server/UITask.cpp | 5 +- scripts/webconfig_mock_server.py | 3 + src/helpers/AlertReporter.cpp | 4 +- src/helpers/ESP32Board.cpp | 7 + src/helpers/MQTTDefaults.h | 1 + src/helpers/MQTTPrefsSerializer.h | 13 +- src/helpers/MQTTPrefsStorage.h | 10 + src/helpers/NetworkInterface.cpp | 231 +++++++++++++++++- src/helpers/NetworkInterface.h | 19 ++ src/helpers/NetworkPolicy.h | 72 +++++- src/helpers/bridges/MQTTBridge.cpp | 30 ++- src/helpers/esp32/WebConfigServer.cpp | 92 +++++-- src/helpers/esp32/WebConfigServer.h | 14 +- .../test_mqtt_prefs_serializer.cpp | 14 ++ .../test_network_policy.cpp | 56 +++++ variants/thinknode_m7/platformio.ini | 6 +- webui/index.html | 1 + 23 files changed, 600 insertions(+), 86 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index ac74485d..774ff63a 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -80,7 +80,7 @@ reboot **9. Verify configuration** ```bash get wifi.ssid -get wifi.status +get link.status get bridge.enabled get mqtt.rx get mqtt.tx @@ -541,12 +541,13 @@ These settings apply across all MQTT slots: - `set mqtt.owner <64-hex-char-public-key>` - Set owner public key - `set mqtt.email ` - Set owner email address -### WiFi Commands +### Network and WiFi Commands #### Get Commands - `get wifi.ssid` - Get WiFi SSID - `get wifi.pwd` - Get WiFi password -- `get wifi.status` - Get WiFi connection status, IP, RSSI, and uptime +- `get link.status` - Get the selected network medium, connection status, IP, signal when available, and uptime +- `get wifi.status` - WiFi-only compatibility alias; reports n/a when another medium is selected - `get wifi.powersave` - Get WiFi power save mode (none/min/max) #### Set Commands diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 48322c6a..cfbe35d8 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -4,7 +4,7 @@ #include // for qsort() #include #include -#if defined(ESP_PLATFORM) && !defined(NETWORK_USE_ETHERNET) +#if defined(ESP_PLATFORM) #include #endif #if defined(WITH_MQTT_NEIGHBORS) @@ -1147,6 +1147,17 @@ void MyMesh::begin(FILESYSTEM *fs) { } #endif + NetworkInterface& boot_network = activeNetworkInterface(); + if (boot_network.isAutomatic()) { + MQTTPrefs* obs = _cli.getObserverPrefs(); + Serial.printf("Network: probing Ethernet for up to %lums\n", + (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); + boot_network.bootstrap(obs->wifi_ssid, obs->wifi_password, + NETWORK_ETHERNET_BOOT_WAIT_MS); + Serial.printf("Network: selected %s (%s)\n", boot_network.mediumName(), + boot_network.statusName()); + } + acl.load(_fs, self_id); // TODO: key_store.begin(); region_map.load(_fs); @@ -1221,15 +1232,13 @@ void MyMesh::begin(FILESYSTEM *fs) { #endif #if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) - // First-boot setup portal: raised only when no WiFi has ever been configured, - // so an OTA onto a deployed (configured) node can never open an AP. - #if !defined(NETWORK_USE_ETHERNET) - if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + // Existing Wi-Fi installs are complete by virtue of their stored SSID. An + // Ethernet-only install uses the explicit completion marker instead. + if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { char wc_reply[160]; startWebConfig(false, wc_reply); Serial.println(wc_reply); } - #endif #endif radio_driver.setParams(_prefs.freq, _prefs.bw, _prefs.sf, _prefs.cr); @@ -1474,11 +1483,6 @@ void MyMesh::clearStats() { #ifdef WITH_WEBCONFIG bool MyMesh::startWebConfig(bool force_ap, char* reply) { -#if defined(NETWORK_USE_ETHERNET) - (void)force_ap; - strcpy(reply, "Err: webconfig unavailable on Ethernet observer; use serial CLI"); - return true; -#else if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); @@ -1491,18 +1495,21 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { } if (force_ap) { // The setup AP owns WiFi outright; refuse while the bridge holds the STA. - if (bridge && bridge->isRunning()) { + if (bridge && bridge->isRunning() && + activeNetworkInterface().medium() != NetworkMedium::Ethernet) { strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); return true; } _webconfig->startSetupMode(reply); - } else if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { - _webconfig->startSetupMode(reply); // unconfigured: same portal as first boot + } else if (activeNetworkInterface().isConnected()) { + _webconfig->startLanMode(activeNetworkInterface().localIP(), + !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); + } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { + _webconfig->startSetupMode(reply); } else { - _webconfig->startLanMode(reply); // reports "WiFi not connected" if down + strcpy(reply, "Err: selected network not connected"); } return true; -#endif } bool MyMesh::stopWebConfig(char* reply) { @@ -1542,11 +1549,9 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { const int signal = network.rssi(); wifi_rssi = signal == INT_MIN ? 0 : signal; } -#if !defined(NETWORK_USE_ETHERNET) else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); } -#endif int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 539a1715..6cc179d8 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -507,6 +507,13 @@ public: _wc_slot_restart_mask = 0; } void onConfigBatchEnd() override; + bool onInitialSetupComplete() override { + MQTTPrefs* obs = _cli.getObserverPrefs(); + obs->network_setup_complete = 1; + if (_cli.saveObserverPrefs(_fs)) return true; + obs->network_setup_complete = 0; + return false; + } void buildStatsJson(char* buf, size_t buf_size) override; #endif diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index ee66520f..506aef64 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -174,15 +174,16 @@ void UITask::renderCurrScreen() { } char wc_ssid[33], wc_ip[16]; if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + const bool lan_setup = WebConfigServer::isLanSetup(); // setup portal active: show join instructions instead of the home screen _display->setTextSize(1); _display->setColor(UIColor::corp_blue); _display->setCursor(0, 0); - _display->print("Observer WiFi Setup"); + _display->print(lan_setup ? "Observer Ethernet Setup" : "Observer WiFi Setup"); _display->setColor(UIColor::primary_txt); _display->setCursor(0, 14); - _display->print("Join WiFi:"); + _display->print(lan_setup ? "Login code:" : "Join WiFi:"); _display->setColor(UIColor::warning_txt); _display->setCursor(6, 24); _display->print(wc_ssid); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 90d5ea4b..01b157cc 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -3,7 +3,7 @@ #include #include #include -#if defined(ESP_PLATFORM) && !defined(NETWORK_USE_ETHERNET) +#if defined(ESP_PLATFORM) #include #endif #if defined(WITH_MQTT_NEIGHBORS) @@ -948,6 +948,17 @@ void MyMesh::begin(FILESYSTEM *fs) { // load persisted prefs _cli.loadPrefs(_fs); + NetworkInterface& boot_network = activeNetworkInterface(); + if (boot_network.isAutomatic()) { + MQTTPrefs* obs = _cli.getObserverPrefs(); + Serial.printf("Network: probing Ethernet for up to %lums\n", + (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); + boot_network.bootstrap(obs->wifi_ssid, obs->wifi_password, + NETWORK_ETHERNET_BOOT_WAIT_MS); + Serial.printf("Network: selected %s (%s)\n", boot_network.mediumName(), + boot_network.statusName()); + } + acl.load(_fs, self_id); region_map.load(_fs); @@ -1023,15 +1034,13 @@ void MyMesh::begin(FILESYSTEM *fs) { #endif #if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) - // First-boot setup portal: raised only when no WiFi has ever been configured, - // so an OTA onto a deployed (configured) node can never open an AP. - #if !defined(NETWORK_USE_ETHERNET) - if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + // Existing Wi-Fi installs are complete by virtue of their stored SSID. An + // Ethernet-only install uses the explicit completion marker instead. + if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { char wc_reply[160]; startWebConfig(false, wc_reply); Serial.println(wc_reply); } - #endif #endif } @@ -1276,11 +1285,6 @@ void MyMesh::formatPacketStatsReply(char *reply) { #ifdef WITH_WEBCONFIG bool MyMesh::startWebConfig(bool force_ap, char* reply) { -#if defined(NETWORK_USE_ETHERNET) - (void)force_ap; - strcpy(reply, "Err: webconfig unavailable on Ethernet observer; use serial CLI"); - return true; -#else if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); @@ -1293,18 +1297,21 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { } if (force_ap) { // The setup AP owns WiFi outright; refuse while the bridge holds the STA. - if (bridge && bridge->isRunning()) { + if (bridge && bridge->isRunning() && + activeNetworkInterface().medium() != NetworkMedium::Ethernet) { strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); return true; } _webconfig->startSetupMode(reply); - } else if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { - _webconfig->startSetupMode(reply); // unconfigured: same portal as first boot + } else if (activeNetworkInterface().isConnected()) { + _webconfig->startLanMode(activeNetworkInterface().localIP(), + !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); + } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { + _webconfig->startSetupMode(reply); } else { - _webconfig->startLanMode(reply); // reports "WiFi not connected" if down + strcpy(reply, "Err: selected network not connected"); } return true; -#endif } bool MyMesh::stopWebConfig(char* reply) { @@ -1344,11 +1351,9 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { const int signal = network.rssi(); wifi_rssi = signal == INT_MIN ? 0 : signal; } -#if !defined(NETWORK_USE_ETHERNET) else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); } -#endif int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 47f2f68d..f5e0be7e 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -497,6 +497,13 @@ public: _wc_slot_restart_mask = 0; } void onConfigBatchEnd() override; + bool onInitialSetupComplete() override { + MQTTPrefs* obs = _cli.getObserverPrefs(); + obs->network_setup_complete = 1; + if (_cli.saveObserverPrefs(_fs)) return true; + obs->network_setup_complete = 0; + return false; + } void buildStatsJson(char* buf, size_t buf_size) override; #endif }; diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index 8dab1cdc..475d20a3 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -155,15 +155,16 @@ void UITask::renderCurrScreen() { } char wc_ssid[33], wc_ip[16]; if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + const bool lan_setup = WebConfigServer::isLanSetup(); // setup portal active: show join instructions instead of the home screen _display->setTextSize(1); _display->setColor(UIColor::corp_blue); _display->setCursor(0, 0); - _display->print("Observer WiFi Setup"); + _display->print(lan_setup ? "Observer Ethernet Setup" : "Observer WiFi Setup"); _display->setColor(UIColor::primary_txt); _display->setCursor(0, 14); - _display->print("Join WiFi:"); + _display->print(lan_setup ? "Login code:" : "Join WiFi:"); _display->setColor(UIColor::warning_txt); _display->setCursor(6, 24); _display->print(wc_ssid); diff --git a/scripts/webconfig_mock_server.py b/scripts/webconfig_mock_server.py index c38d804e..547889a1 100644 --- a/scripts/webconfig_mock_server.py +++ b/scripts/webconfig_mock_server.py @@ -497,6 +497,9 @@ GETTERS = { "wifi.status": lambda c: ( "SSID: %s\nIP: 192.168.1.42\nRSSI: -58 dBm\nUptime: %dm" % (c["wifi"]["ssid"] or "(not set)", int(time.time() - ST.start) // 60)), + "link.status": lambda c: ( + "wifi: connected, IP: 192.168.1.42, RSSI: -58 dBm, Uptime: %dm" + % (int(time.time() - ST.start) // 60)), "mqtt.status": lambda c: cli_mqtt_status(c), "mqtt.presets": lambda c: "\n".join( "%2d. %s%s" % (i + 1, n, "" if nd == "none" else " (needs %s)" % nd) diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index 171d81c8..d9617ca7 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -220,7 +220,7 @@ void AlertReporter::onLoop(unsigned long now_ms) { char text[80]; AlertFaultPolicy::formatNetworkAlert( text, sizeof(text), - strcmp(activeNetworkInterface().mediumName(), "ethernet") == 0 + activeNetworkInterface().medium() == NetworkMedium::Ethernet ? "Ethernet" : "WiFi", r, snap); if (sendChannel(text)) { @@ -230,7 +230,7 @@ void AlertReporter::onLoop(unsigned long now_ms) { char text[80]; AlertFaultPolicy::formatNetworkAlert( text, sizeof(text), - strcmp(activeNetworkInterface().mediumName(), "ethernet") == 0 + activeNetworkInterface().medium() == NetworkMedium::Ethernet ? "Ethernet" : "WiFi", r, snap); sendChannel(text); diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 5452c50c..19cdce77 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -14,6 +14,9 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { inhibit_sleep = true; // prevent sleep during OTA + // Manual ElegantOTA owns port 80 and remains active until a successful upload + // reboots the device, so its route lock is intentionally reboot-scoped. + activeNetworkInterface().lockSwitching(); // If the device is already on its selected network, serve ElegantOTA on that // address so it is reachable without joining a separate AP. Otherwise raise @@ -184,16 +187,20 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep // mesh-receive call chain (it overflows the loopTask canary). Run the work in a // dedicated 24 KB-stack task and block here until it finishes. The big stack is // freed when the task exits; on a successful update the chip reboots inside it. + NetworkInterface& network = activeNetworkInterface(); + network.lockSwitching(); OtaTaskArgs args = { this, current_ver, dry_run, reply, false, false }; TaskHandle_t handle = nullptr; BaseType_t ok = xTaskCreatePinnedToCore(ota_task_entry, "ota", 24576, &args, 5, &handle, 1); if (ok != pdPASS) { + network.unlockSwitching(); strcpy(reply, "ERR: OTA task spawn failed"); return false; } while (!args.done) { delay(50); // Arduino delay() yields to other tasks } + network.unlockSwitching(); return args.result; } diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h index c390da0b..ecee1545 100644 --- a/src/helpers/MQTTDefaults.h +++ b/src/helpers/MQTTDefaults.h @@ -112,6 +112,7 @@ static inline void applyMQTTDefaults(MQTTPrefs* prefs) { prefs->display_timeout_secs = DISPLAY_TIMEOUT_DEFAULT_SECS; prefs->display_flip = 0; + prefs->network_setup_complete = 0; } #endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h index f783b52c..b3b968cc 100644 --- a/src/helpers/MQTTPrefsSerializer.h +++ b/src/helpers/MQTTPrefsSerializer.h @@ -39,22 +39,31 @@ class MQTTPrefsSerializer : public ConfigSerializer { class WifiPrefs : public ConfigSerializer { MQTTPrefs* _prefs; - int32_t _power_save; + int32_t _power_save, _setup_complete; bool _seen_ssid = false, _seen_password = false, _seen_power_save = false; + bool _seen_setup_complete = false; protected: void structure() override { defStrict("ssid", _prefs->wifi_ssid, sizeof(_prefs->wifi_ssid), _seen_ssid); defStrict("password", _prefs->wifi_password, sizeof(_prefs->wifi_password), _seen_password); defStrict("power_save", _power_save, _seen_power_save); + defStrict("setup_complete", _setup_complete, _seen_setup_complete); } public: - explicit WifiPrefs(MQTTPrefs* prefs) : _prefs(prefs), _power_save(prefs->wifi_power_save) {} + explicit WifiPrefs(MQTTPrefs* prefs) + : _prefs(prefs), _power_save(prefs->wifi_power_save), + _setup_complete(prefs->network_setup_complete) {} bool apply(bool* repaired) { if (_power_save < 0 || _power_save > 2) { _power_save = 1; *repaired = true; } + if (_setup_complete < 0 || _setup_complete > 1) { + _setup_complete = 0; + *repaired = true; + } _prefs->wifi_power_save = static_cast(_power_save); + _prefs->network_setup_complete = static_cast(_setup_complete); return true; } }; diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h index 7401825b..c5072f15 100644 --- a/src/helpers/MQTTPrefsStorage.h +++ b/src/helpers/MQTTPrefsStorage.h @@ -126,8 +126,18 @@ struct MQTTPrefs { // Rotate the panel 180 degrees from its compiled DISPLAY_ROTATION, for boards // mounted the other way up. Runtime only, like display_timeout_secs. uint8_t display_flip; + + // Explicitly records completion of first-run network onboarding when a node + // is configured over Ethernet and therefore may intentionally have no Wi-Fi + // SSID. Existing Wi-Fi installations remain complete by the helper below. + uint8_t network_setup_complete; }; +static inline bool mqttNetworkSetupComplete(const MQTTPrefs* prefs) { + return prefs != nullptr && + (prefs->network_setup_complete != 0 || prefs->wifi_ssid[0] != '\0'); +} + static const uint16_t DISPLAY_TIMEOUT_DEFAULT_SECS = 60; static const uint16_t DISPLAY_TIMEOUT_MAX_SECS = 3600; diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index 1d4d2d39..ae5273e3 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -11,7 +11,7 @@ #include #include -#if defined(NETWORK_USE_ETHERNET) +#if defined(NETWORK_PREFER_ETHERNET) #include "ethernet/ch390/CH390Config.h" #endif @@ -88,6 +88,7 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { public: const char* mediumName() const override { return "wifi"; } + NetworkMedium medium() const override { return NetworkMedium::WiFi; } const char* statusName() const override { switch (WiFi.status()) { case WL_CONNECTED: return "connected"; @@ -200,13 +201,14 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { } }; -#if defined(NETWORK_USE_ETHERNET) +#if defined(NETWORK_PREFER_ETHERNET) class EthernetNetworkInterface final : public NetworkInterfaceBase { bool _started = false; bool _event_registered = false; public: const char* mediumName() const override { return "ethernet"; } + NetworkMedium medium() const override { return NetworkMedium::Ethernet; } const char* statusName() const override { return isConnected() ? "connected" : "disconnected"; } @@ -281,13 +283,234 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { return WiFi.hostByName(hostname, address); } }; + +class AutomaticNetworkInterface final : public NetworkInterface { + EthernetNetworkInterface _ethernet; + WiFiNetworkInterface _wifi; + NetworkMedium _selected = NetworkMedium::None; + bool _ethernet_started = false; + bool _wifi_started = false; + char _wifi_ssid[33] = {}; + char _wifi_password[65] = {}; + uint32_t _ethernet_stable_since = 0; + uint32_t _selected_down_since = 0; + std::atomic _switch_locks{0}; + + NetworkInterface& selectedInterface() { + return _selected == NetworkMedium::Ethernet + ? static_cast(_ethernet) + : static_cast(_wifi); + } + const NetworkInterface& selectedInterface() const { + return _selected == NetworkMedium::Ethernet + ? static_cast(_ethernet) + : static_cast(_wifi); + } + + bool wifiConfigured() const { return _wifi_ssid[0] != '\0'; } + + void rememberWifi(const char* ssid, const char* password) { + strncpy(_wifi_ssid, ssid ? ssid : "", sizeof(_wifi_ssid) - 1); + _wifi_ssid[sizeof(_wifi_ssid) - 1] = '\0'; + strncpy(_wifi_password, password ? password : "", sizeof(_wifi_password) - 1); + _wifi_password[sizeof(_wifi_password) - 1] = '\0'; + } + + void startWifiFallback() { + if (_wifi_started || !wifiConfigured()) return; + _wifi_started = _wifi.begin(_wifi_ssid, _wifi_password); + } + + void select(NetworkMedium medium) { + if (medium == NetworkMedium::Ethernet) { + // ESP-IDF gives Wi-Fi a higher default-route priority than Ethernet. + // Keep only the selected STA associated so sockets cannot silently stay + // on Wi-Fi after the manager has declared Ethernet active. + WiFi.setAutoReconnect(false); + WiFi.disconnect(false, false); + _wifi_started = false; + } + _selected = medium; + _selected_down_since = 0; + } + + public: + const char* mediumName() const override { + if (_selected == NetworkMedium::Ethernet) return "ethernet"; + if (_selected == NetworkMedium::WiFi) return "wifi"; + return "none"; + } + NetworkMedium medium() const override { return _selected; } + const char* statusName() const override { + return _selected == NetworkMedium::None ? "not_selected" + : selectedInterface().statusName(); + } + int statusCode() const override { + return _selected == NetworkMedium::None ? 0 : selectedInterface().statusCode(); + } + bool configValid(const char* wifi_ssid) const override { + // Hardware availability and stored credentials are configuration. Current + // link/DHCP state is runtime state and must not permanently suppress the + // MQTT task that monitors for a late cable or lease. + return _ethernet_started || (wifi_ssid && wifi_ssid[0] != '\0'); + } + bool isAutomatic() const override { return true; } + + bool begin(const char* wifi_ssid, const char* wifi_password) override { + rememberWifi(wifi_ssid, wifi_password); + if (!_ethernet_started) { + _ethernet_started = _ethernet.begin(nullptr, nullptr); + } + // bootstrap() owns the initial choice. MQTT begin() is intentionally + // idempotent and cannot demote a boot-selected Ethernet link because of a + // momentary status sample between tasks. + if (_selected == NetworkMedium::None) { + if (_ethernet.isConnected()) { + select(NetworkMedium::Ethernet); + } else if (wifiConfigured()) { + startWifiFallback(); + _selected = NetworkMedium::WiFi; + } + } else if (_selected == NetworkMedium::WiFi) { + startWifiFallback(); + } + return _ethernet_started || _wifi_started; + } + + bool bootstrap(const char* wifi_ssid, const char* wifi_password, + uint32_t wait_ms) override { + rememberWifi(wifi_ssid, wifi_password); + if (!_ethernet_started) { + _ethernet_started = _ethernet.begin(nullptr, nullptr); + } + + const uint32_t started_at = millis(); + const uint32_t link_wait_ms = wait_ms < 1500 ? wait_ms : 1500; + while (_ethernet_started && !CH390.linkUp() && + (uint32_t)(millis() - started_at) < link_wait_ms) { + delay(25); + } + while (_ethernet_started && CH390.linkUp() && !_ethernet.isConnected() && + (uint32_t)(millis() - started_at) < wait_ms) { + delay(25); + } + + const NetworkMedium initial = NetworkPolicy::bootSelection( + _ethernet.isConnected(), wifiConfigured()); + if (initial == NetworkMedium::Ethernet) { + select(initial); + } else { + startWifiFallback(); + _selected = initial; + } + return initial != NetworkMedium::None; + } + + NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) override { + const NetworkTransition ethernet_transition = + _ethernet.maintain(now_ms, wifi_power_save); + const NetworkTransition wifi_transition = _wifi_started + ? _wifi.maintain(now_ms, wifi_power_save) + : NetworkTransition::None; + + // Ethernet may recover while a Wi-Fi fallback is still associating. In + // that sequence the selected enum never changes, but Wi-Fi would win + // ESP-IDF's default-route priority once it came up. Tear the unused STA + // down even without a selection edge, and force MQTT to reconnect if it + // had already become reachable. + if (_selected == NetworkMedium::Ethernet && _ethernet.isConnected() && + _wifi_started) { + const bool wifi_had_route = _wifi.isConnected(); + select(NetworkMedium::Ethernet); + if (wifi_had_route) return NetworkTransition::Switched; + } + + if (_ethernet.isConnected()) { + if (_ethernet_stable_since == 0) _ethernet_stable_since = now_ms; + } else { + _ethernet_stable_since = 0; + } + + const bool selected_connected = isConnected(); + if (!selected_connected) { + if (_selected_down_since == 0) _selected_down_since = now_ms; + } else { + _selected_down_since = 0; + } + + const uint32_t selected_down_ms = _selected_down_since == 0 + ? 0 : (uint32_t)(now_ms - _selected_down_since); + if (_selected == NetworkMedium::Ethernet && !_ethernet.isConnected() && + selected_down_ms >= NetworkPolicy::kEthernetDownGraceMs) { + startWifiFallback(); + } + + const uint32_t ethernet_stable_ms = _ethernet_stable_since == 0 + ? 0 : (uint32_t)(now_ms - _ethernet_stable_since); + const NetworkPolicy::AutomaticSelectionInput input = { + _selected, _ethernet.isConnected(), + _wifi_started && _wifi.isConnected(), wifiConfigured(), + _switch_locks.load(std::memory_order_relaxed) != 0, + ethernet_stable_ms, selected_down_ms}; + const NetworkMedium next = NetworkPolicy::automaticSelection(input); + + if (next != _selected) { + const NetworkMedium previous = _selected; + select(next); + return previous == NetworkMedium::None ? NetworkTransition::Up + : NetworkTransition::Switched; + } + + if (_selected == NetworkMedium::Ethernet) return ethernet_transition; + if (_selected == NetworkMedium::WiFi) return wifi_transition; + return NetworkTransition::None; + } + + void lockSwitching() override { + _switch_locks.fetch_add(1, std::memory_order_relaxed); + } + void unlockSwitching() override { + uint8_t value = _switch_locks.load(std::memory_order_relaxed); + while (value != 0 && !_switch_locks.compare_exchange_weak( + value, static_cast(value - 1), + std::memory_order_relaxed, std::memory_order_relaxed)) {} + } + + bool isConnected() const override { + return _selected != NetworkMedium::None && selectedInterface().isConnected(); + } + IPAddress localIP() const override { + return _selected == NetworkMedium::None ? IPAddress() : selectedInterface().localIP(); + } + int rssi() const override { + return _selected == NetworkMedium::None ? INT_MIN : selectedInterface().rssi(); + } + bool resolveHost(const char* hostname, IPAddress& address) const override { + return _selected != NetworkMedium::None && + selectedInterface().resolveHost(hostname, address); + } + unsigned long connectedAtMillis() const override { + return _selected == NetworkMedium::None ? 0 : selectedInterface().connectedAtMillis(); + } + uint8_t lastDisconnectReason() const override { + return _selected == NetworkMedium::None ? 0 : selectedInterface().lastDisconnectReason(); + } + unsigned long lastDisconnectTime() const override { + return _selected == NetworkMedium::None ? 0 : selectedInterface().lastDisconnectTime(); + } + AlertFaultPolicy::OutageSnapshot outageSnapshot() const override { + return _selected == NetworkMedium::None + ? AlertFaultPolicy::OutageSnapshot{false, 0, 0} + : selectedInterface().outageSnapshot(); + } +}; #endif } // namespace NetworkInterface& activeNetworkInterface() { -#if defined(NETWORK_USE_ETHERNET) - static EthernetNetworkInterface network; +#if defined(NETWORK_PREFER_ETHERNET) + static AutomaticNetworkInterface network; #else static WiFiNetworkInterface network; #endif diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkInterface.h index a4e631a1..a5e15274 100644 --- a/src/helpers/NetworkInterface.h +++ b/src/helpers/NetworkInterface.h @@ -8,6 +8,10 @@ #include #include "AlertFaultPolicy.h" +#ifndef NETWORK_ETHERNET_BOOT_WAIT_MS +#define NETWORK_ETHERNET_BOOT_WAIT_MS 8000UL +#endif + /** * Physical network selected for IP-based services. * @@ -21,12 +25,27 @@ class NetworkInterface { virtual ~NetworkInterface() = default; virtual const char* mediumName() const = 0; + virtual NetworkMedium medium() const = 0; virtual const char* statusName() const = 0; virtual int statusCode() const = 0; virtual bool configValid(const char* wifi_ssid) const = 0; virtual bool begin(const char* wifi_ssid, const char* wifi_password) = 0; virtual NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) = 0; + // Automatic selectors are boot-owned so first-run services can use the + // chosen link before MQTT starts. Compatibility Wi-Fi builds keep the + // original MQTT-task-owned begin() path. + virtual bool isAutomatic() const { return false; } + virtual bool bootstrap(const char* wifi_ssid, const char* wifi_password, + uint32_t wait_ms) { + (void)wait_ms; + return begin(wifi_ssid, wifi_password); + } + + // WebConfig and OTA pin the current route for the lifetime of their session. + virtual void lockSwitching() {} + virtual void unlockSwitching() {} + virtual bool isConnected() const = 0; virtual IPAddress localIP() const = 0; virtual int rssi() const = 0; // INT_MIN when the selected medium has no RSSI. diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h index 731ed349..fe434a03 100644 --- a/src/helpers/NetworkPolicy.h +++ b/src/helpers/NetworkPolicy.h @@ -6,6 +6,13 @@ enum class NetworkTransition : uint8_t { None, Up, Down, + Switched, +}; + +enum class NetworkMedium : uint8_t { + None, + Ethernet, + WiFi, }; namespace NetworkPolicy { @@ -17,11 +24,72 @@ struct MQTTTransitionActions { static constexpr MQTTTransitionActions mqttActions(NetworkTransition transition) { return { - transition == NetworkTransition::Down, - transition == NetworkTransition::Up, + transition == NetworkTransition::Down || transition == NetworkTransition::Switched, + transition == NetworkTransition::Up || transition == NetworkTransition::Switched, }; } +struct AutomaticSelectionInput { + NetworkMedium selected; + bool ethernet_connected; + bool wifi_connected; + bool wifi_configured; + bool switching_locked; + uint32_t ethernet_stable_ms; + uint32_t selected_down_ms; +}; + +// Keep short link flaps from bouncing the default route. Ethernet is allowed +// to recover before Wi-Fi is started, and must then remain usable before it can +// preempt a working Wi-Fi fallback. +static constexpr uint32_t kEthernetDownGraceMs = 3000; +static constexpr uint32_t kEthernetFailbackStableMs = 10000; +static constexpr uint32_t kNtpRetryMs = 30000; + +static constexpr bool ntpPendingAfterConnectivitySample( + bool synced, bool pending, bool was_connected, bool connected) { + return pending || (!synced && connected && !was_connected); +} + +static constexpr bool ntpRetryDue(bool synced, bool connected, + uint32_t now_ms, uint32_t last_attempt_ms) { + return !synced && connected && + (uint32_t)(now_ms - last_attempt_ms) >= kNtpRetryMs; +} + +static constexpr NetworkMedium bootSelection(bool ethernet_connected, + bool wifi_configured) { + return ethernet_connected ? NetworkMedium::Ethernet + : wifi_configured ? NetworkMedium::WiFi + : NetworkMedium::None; +} + +static inline NetworkMedium automaticSelection( + const AutomaticSelectionInput& input) { + if (input.switching_locked) return input.selected; + + if (input.selected == NetworkMedium::Ethernet) { + if (input.ethernet_connected) return NetworkMedium::Ethernet; + if (input.wifi_configured && input.wifi_connected && + input.selected_down_ms >= kEthernetDownGraceMs) { + return NetworkMedium::WiFi; + } + return NetworkMedium::Ethernet; + } + + if (input.selected == NetworkMedium::WiFi) { + if (input.ethernet_connected && + input.ethernet_stable_ms >= kEthernetFailbackStableMs) { + return NetworkMedium::Ethernet; + } + return NetworkMedium::WiFi; + } + + if (input.ethernet_connected) return NetworkMedium::Ethernet; + if (input.wifi_configured && input.wifi_connected) return NetworkMedium::WiFi; + return NetworkMedium::None; +} + // `start ota` uses the selected LAN only when reachable and not explicitly // forced to SoftAP. Manifest OTA has no fallback and checks connectivity itself. static constexpr bool startOtaUsesSelectedNetwork(bool force_ap, diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 8b48401f..12bf9a30 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1252,6 +1252,9 @@ void MQTTBridge::mqttTaskLoop() { // Wait a bit for WiFi to start connecting vTaskDelay(pdMS_TO_TICKS(1000)); + bool network_was_connected = _network->isConnected(); + unsigned long last_ntp_attempt = 0; + // Main task loop #ifdef MQTT_MEMORY_DEBUG static unsigned long last_agent_log = 0; @@ -1315,23 +1318,26 @@ void MQTTBridge::mqttTaskLoop() { } } - // A connected observation is enough to schedule NTP; physical event callbacks - // stay encapsulated in the selected network adapter. - if (!_ntp_synced && _network->isConnected() && !_ntp_sync_pending) { - _ntp_sync_pending = true; - } - if (_ntp_sync_pending && _network->isConnected()) { + // Schedule once per link-up edge. Failed syncs are owned by the 30-second + // retry below; re-arming here on every loop would make a blocked NTP path + // run the multi-second sync sequence back-to-back forever. + const bool network_connected = _network->isConnected(); + _ntp_sync_pending = NetworkPolicy::ntpPendingAfterConnectivitySample( + _ntp_synced, _ntp_sync_pending, network_was_connected, + network_connected); + network_was_connected = network_connected; + if (_ntp_sync_pending && network_connected) { _ntp_sync_pending = false; + last_ntp_attempt = now; syncTimeWithNTP(); } // Retry NTP every 30s if initial sync failed (slots can't start without valid time) - if (!_ntp_synced && _network->isConnected()) { - static unsigned long last_ntp_retry = 0; - if (now - last_ntp_retry >= 30000) { - last_ntp_retry = now; - syncTimeWithNTP(); - } + if (NetworkPolicy::ntpRetryDue( + _ntp_synced, network_connected, (uint32_t)now, + (uint32_t)last_ntp_attempt)) { + last_ntp_attempt = now; + syncTimeWithNTP(); } // Process a CLI-requested forced NTP sync (queued from Core 1). Running it here diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 34bdca4b..37740e8c 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -172,17 +173,28 @@ bool WebConfigServer::isRebootPending() { bool WebConfigServer::getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t ip_len) { WebConfigServer* w = _active; - if (w == NULL || w->_mode != MODE_SETUP || w->_stopping) return false; + if (w == NULL || w->_stopping || + (w->_mode != MODE_SETUP && !(w->_mode == MODE_LAN && w->_initial_setup))) { + return false; + } if (ssid && ssid_len > 0) { - strncpy(ssid, w->_ap_ssid, ssid_len - 1); + const char* label = w->_mode == MODE_LAN ? w->_setup_code : w->_ap_ssid; + strncpy(ssid, label, ssid_len - 1); ssid[ssid_len - 1] = 0; } if (ip && ip_len > 0) { - snprintf(ip, ip_len, "%s", WiFi.softAPIP().toString().c_str()); + const IPAddress address = w->_mode == MODE_LAN + ? activeNetworkInterface().localIP() : WiFi.softAPIP(); + snprintf(ip, ip_len, "%s", address.toString().c_str()); } return true; } +bool WebConfigServer::isLanSetup() { + WebConfigServer* w = _active; + return w != NULL && !w->_stopping && w->_initial_setup && w->_mode == MODE_LAN; +} + // --------------------------------------------------------------------------- // Lifecycle // --------------------------------------------------------------------------- @@ -223,8 +235,10 @@ bool WebConfigServer::startSetupMode(char reply[]) { _dns->start(53, "*", ip); // captive portal: every name resolves to us _mode = MODE_SETUP; - _initial_setup = (_obs->wifi_ssid[0] == 0); + _initial_setup = !mqttNetworkSetupComplete(_obs); createServer(); + activeNetworkInterface().lockSwitching(); + _network_locked = true; _was_setup_ap = true; _last_activity = millis(); WiFi.scanNetworks(true); // pre-populate the SSID picker @@ -233,21 +247,42 @@ bool WebConfigServer::startSetupMode(char reply[]) { return true; } -bool WebConfigServer::startLanMode(char reply[]) { +bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[]) { if (_mode != MODE_OFF || _stopping) { strcpy(reply, "Err: webconfig busy"); return false; } - if (WiFi.status() != WL_CONNECTED) { - strcpy(reply, "Err: WiFi not connected"); + activeNetworkInterface().lockSwitching(); + _network_locked = true; + if (!activeNetworkInterface().isConnected() || ip == IPAddress()) { + activeNetworkInterface().unlockSwitching(); + _network_locked = false; + strcpy(reply, "Err: selected network not connected"); return false; } + _initial_setup = initial_setup; + if (_initial_setup) { + for (int i = 0; i < 3; ++i) { + sprintf(&_setup_code[i * 4], "%04X", (unsigned)(esp_random() & 0xffff)); + } + _setup_code[12] = 0; + } else { + _setup_code[0] = 0; + } _mode = MODE_LAN; createServer(); _last_activity = millis(); - int pos = sprintf(reply, "WebConfig started: http://%s/ (admin password login)", - WiFi.localIP().toString().c_str()); + int pos; + if (_initial_setup) { + pos = sprintf(reply, "WebConfig Ethernet setup: http://%s/ code %s", + ip.toString().c_str(), _setup_code); + _setup_reminder_at = millis() + 60000; + if (_setup_reminder_at == 0) _setup_reminder_at = 1; + } else { + pos = sprintf(reply, "WebConfig started: http://%s/ (admin password login)", + ip.toString().c_str()); + } if (heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL) < 60 * 1024) { sprintf(reply + pos, " WARN: low heap"); } @@ -317,8 +352,8 @@ void WebConfigServer::finalizeTeardown() { _dns = NULL; if (_was_setup_ap) { WiFi.softAPdisconnect(true); - // Nothing else owns WiFi when we raised the AP: either the node is - // unconfigured, or `start webconfig ap` required the bridge stopped. + // A Wi-Fi-selected bridge must be stopped before a forced AP. An + // Ethernet-selected bridge can remain up because it does not own the STA. if (_obs->wifi_ssid[0] == 0) { WiFi.mode(WIFI_OFF); } else { @@ -336,6 +371,12 @@ void WebConfigServer::finalizeTeardown() { _batch_reboot_armed = false; _session_token[0] = 0; _stats_json[0] = 0; + _setup_code[0] = 0; + _setup_reminder_at = 0; + if (_network_locked) { + activeNetworkInterface().unlockSwitching(); + _network_locked = false; + } if (_cb) _cb->onWebConfigStopped(); } @@ -358,6 +399,15 @@ void WebConfigServer::tick(uint32_t now) { } if (_mode == MODE_OFF) return; + if (_mode == MODE_LAN && _initial_setup && _setup_reminder_at != 0 && + (int32_t)(now - _setup_reminder_at) >= 0) { + Serial.printf("WC: Ethernet setup http://%s/ code %s\n", + activeNetworkInterface().localIP().toString().c_str(), + _setup_code); + _setup_reminder_at = now + 60000; + if (_setup_reminder_at == 0) _setup_reminder_at = 1; + } + if (_dns) _dns->processNextRequest(); if (_batch_state == BATCH_PENDING) drainBatch(now); @@ -459,6 +509,15 @@ void WebConfigServer::drainBatch(uint32_t now) { return; // more commands next tick } } + if (_initial_setup && _admin_pwd_set && _batch_all_ok && + (_mode == MODE_LAN || _obs->wifi_ssid[0] != '\0')) { + if (_cb->onInitialSetupComplete()) { + _initial_setup = false; + _setup_code[0] = 0; + } else { + _batch_all_ok = false; + } + } _cb->onConfigBatchEnd(); WCLock lock(_mux); _batch_state = BATCH_DONE; @@ -611,7 +670,7 @@ void WebConfigServer::handleStatus(AsyncWebServerRequest* req) { DynamicJsonDocument doc(512); doc["mode"] = (_mode == MODE_SETUP) ? "setup" : "lan"; doc["auth"] = authed; - doc["needs_setup"] = (_obs->wifi_ssid[0] == 0); + doc["needs_setup"] = !mqttNetworkSetupComplete(_obs); doc["name"] = (const char*)_prefs->node_name; char node_id[17]; for (int i = 0; i < 8; i++) sprintf(&node_id[i * 2], "%02x", _pub_key[i]); @@ -657,7 +716,10 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { return; } const char* pwd = doc["password"] | ""; - if (!fixedTimeEquals(pwd, _prefs->password, sizeof(_prefs->password))) { + const char* expected = _initial_setup ? _setup_code : _prefs->password; + const size_t expected_size = _initial_setup ? sizeof(_setup_code) + : sizeof(_prefs->password); + if (!fixedTimeEquals(pwd, expected, expected_size)) { if (++_login_fails >= 5) { _login_lock_until = now + 30000; if (_login_lock_until == 0) _login_lock_until = 1; @@ -836,7 +898,7 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) { // First onboarding is not complete until the known factory password has // been replaced. Enforce this server-side so the Advanced editor or a crafted // request cannot save WiFi and strand the node with the default password. - if (_mode == MODE_SETUP && _initial_setup && !set.containsKey("password") && + if (_initial_setup && !set.containsKey("password") && (reboot_after || set.containsKey("wifi.ssid"))) { req->send(400, "application/json", "{\"error\":\"admin password required for initial setup\"}"); return; @@ -1148,7 +1210,7 @@ void WebConfigServer::handleCliPost(AsyncWebServerRequest* req) { // LAN still holding the factory password is a known credential on someone // else's network. The terminal warned about this client-side, which is a // reminder, not a rule — a pasted script or a direct POST ignored it. - if (_mode == MODE_SETUP && _initial_setup && !seq_sets_pwd && !_admin_pwd_set && + if (_initial_setup && !seq_sets_pwd && !_admin_pwd_set && (defer_reboot || seq_sets_ssid)) { req->send(400, "application/json", "{\"error\":\"admin password required for initial setup — " diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index 0dcebebd..79cb167b 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -6,8 +6,9 @@ // - SETUP: open SoftAP + captive portal, raised automatically on first boot // when no WiFi is configured (wifi_ssid empty), or manually via // `start webconfig ap`. Save -> reboot; auto-stops after an idle timeout. -// - LAN: bound to the existing STA connection (owned by the MQTT bridge -// task), started via `start webconfig`, admin-password login required. +// - LAN: bound to the selected network, started via `start webconfig`, +// admin-password login required. First-run Ethernet uses a random one-time +// login code and requires the operator to replace the admin password. // // Concurrency model: AsyncWebServer handlers run on the async_tcp task and // must never touch the CLI, prefs persistence, or the radio. Config writes @@ -57,6 +58,9 @@ public: // `set` handlers can be coalesced into one. virtual void onConfigBatchStart() {} virtual void onConfigBatchEnd() {} + // Persist completion of Ethernet onboarding after its password-setting + // batch succeeds. Called from the loop task. + virtual bool onInitialSetupComplete() { return false; } // Fill buf with the stats JSON snapshot. Called from tick() (loop task). virtual void buildStatsJson(char* buf, size_t buf_size) = 0; // Teardown finished (session + DNS freed, WiFi mode restored). @@ -72,6 +76,7 @@ public: // Fills the AP SSID and portal IP; either buffer may be NULL to just poll. // Call from the loop task only (same task that changes the mode). static bool getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t ip_len); + static bool isLanSetup(); // For the device display: true once a config save completed and the node is // about to reboot — ground truth for the user even if the browser lost its @@ -79,7 +84,7 @@ public: static bool isRebootPending(); bool startSetupMode(char reply[]); // open SoftAP + DNS captive portal - bool startLanMode(char reply[]); // bind to existing STA connection + bool startLanMode(IPAddress ip, bool initial_setup, char reply[]); void requestStop(); // stop listening and detach this session void tick(uint32_t now); // call every loop iteration @@ -136,11 +141,13 @@ private: bool _stopping = false; bool _was_setup_ap = false; bool _initial_setup = false; + bool _network_locked = false; // A `password` command has succeeded this session. Lets the CLI satisfy the // initial-setup invariant across separate submissions; the form batch always // sends the password with the rest, so it never needed the memory. bool _admin_pwd_set = false; char _ap_ssid[33] = {0}; + char _setup_code[13] = {0}; // Currently attached session, also used by the display's setup-info poll. static WebConfigServer* _active; @@ -166,6 +173,7 @@ private: // LAN-mode session (single slot; new login evicts the old session) char _session_token[33] = {0}; uint32_t _session_last_seen = 0; + uint32_t _setup_reminder_at = 0; uint8_t _login_fails = 0; uint32_t _login_lock_until = 0; diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp index 421ea52b..3427bb60 100644 --- a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -113,6 +113,7 @@ TEST(MQTTPrefsSerializer, RoundTripsEveryGroupAndNumericSlotKeys) { source.snmp_enabled = 1; source.radio_watchdog_minutes = 120; source.alert_enabled = 1; + source.network_setup_complete = 1; strcpy(source.alert_psk_hex, "0123456789abcdef0123456789abcdef"); strcpy(source.alert_hashtag, "#ops"); strcpy(source.alert_region, "PNW"); @@ -137,6 +138,19 @@ TEST(MQTTPrefsSerializer, RoundTripsEveryGroupAndNumericSlotKeys) { EXPECT_EQ(0x8001, loaded.mqtt_slot_packet_filter[5]); EXPECT_EQ(MQTT_NEIGHBORS_MAX_INTERVAL_MS, loaded.mqtt_neighbors_interval); EXPECT_STREQ("PNW", loaded.alert_region); + EXPECT_EQ(1, loaded.network_setup_complete); +} + +TEST(MQTTPrefsSerializer, NetworkOnboardingSupportsWifiAndEthernetOnlyInstalls) { + MQTTPrefs prefs = defaults(); + EXPECT_FALSE(mqttNetworkSetupComplete(&prefs)); + + strcpy(prefs.wifi_ssid, "existing-install"); + EXPECT_TRUE(mqttNetworkSetupComplete(&prefs)); + + prefs.wifi_ssid[0] = 0; + prefs.network_setup_complete = 1; + EXPECT_TRUE(mqttNetworkSetupComplete(&prefs)); } TEST(MQTTPrefsSerializer, MissingOptionalKeysKeepDefaults) { diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp index 201c7451..93e0a1f2 100644 --- a/test/test_network_policy/test_network_policy.cpp +++ b/test/test_network_policy/test_network_policy.cpp @@ -20,6 +20,62 @@ TEST(NetworkPolicy, NoTransitionHasNoMqttSideEffects) { EXPECT_FALSE(actions.retry_disconnected_slots_now); } +TEST(NetworkPolicy, LinkSwitchReconnectsMqttSlotsImmediately) { + const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Switched); + EXPECT_TRUE(actions.disconnect_slots); + EXPECT_TRUE(actions.retry_disconnected_slots_now); +} + +TEST(NetworkPolicy, BootPrefersEthernetAndOtherwiseUsesConfiguredWifi) { + EXPECT_EQ(NetworkMedium::Ethernet, NetworkPolicy::bootSelection(true, true)); + EXPECT_EQ(NetworkMedium::WiFi, NetworkPolicy::bootSelection(false, true)); + EXPECT_EQ(NetworkMedium::None, NetworkPolicy::bootSelection(false, false)); +} + +TEST(NetworkPolicy, EthernetFailureWaitsForGraceAndConnectedWifi) { + NetworkPolicy::AutomaticSelectionInput input = { + NetworkMedium::Ethernet, false, true, true, false, 0, + NetworkPolicy::kEthernetDownGraceMs - 1}; + EXPECT_EQ(NetworkMedium::Ethernet, NetworkPolicy::automaticSelection(input)); + input.selected_down_ms = NetworkPolicy::kEthernetDownGraceMs; + EXPECT_EQ(NetworkMedium::WiFi, NetworkPolicy::automaticSelection(input)); +} + +TEST(NetworkPolicy, FailbackRequiresStableEthernet) { + NetworkPolicy::AutomaticSelectionInput input = { + NetworkMedium::WiFi, true, true, true, false, + NetworkPolicy::kEthernetFailbackStableMs - 1, 0}; + EXPECT_EQ(NetworkMedium::WiFi, NetworkPolicy::automaticSelection(input)); + input.ethernet_stable_ms = NetworkPolicy::kEthernetFailbackStableMs; + EXPECT_EQ(NetworkMedium::Ethernet, NetworkPolicy::automaticSelection(input)); +} + +TEST(NetworkPolicy, SwitchingLockPinsTheCurrentMedium) { + const NetworkPolicy::AutomaticSelectionInput input = { + NetworkMedium::WiFi, true, true, true, true, + NetworkPolicy::kEthernetFailbackStableMs, 0}; + EXPECT_EQ(NetworkMedium::WiFi, NetworkPolicy::automaticSelection(input)); +} + +TEST(NetworkPolicy, NtpIsScheduledOncePerConnectivityEdge) { + EXPECT_TRUE(NetworkPolicy::ntpPendingAfterConnectivitySample( + false, false, false, true)); + EXPECT_FALSE(NetworkPolicy::ntpPendingAfterConnectivitySample( + false, false, true, true)); + EXPECT_TRUE(NetworkPolicy::ntpPendingAfterConnectivitySample( + false, true, true, true)); + EXPECT_FALSE(NetworkPolicy::ntpPendingAfterConnectivitySample( + true, false, false, true)); +} + +TEST(NetworkPolicy, FailedNtpSyncWaitsForThirtySecondRetryBoundary) { + EXPECT_FALSE(NetworkPolicy::ntpRetryDue(false, true, 29999, 0)); + EXPECT_TRUE(NetworkPolicy::ntpRetryDue(false, true, 30000, 0)); + EXPECT_FALSE(NetworkPolicy::ntpRetryDue(true, true, 60000, 0)); + EXPECT_FALSE(NetworkPolicy::ntpRetryDue(false, false, 60000, 0)); + EXPECT_TRUE(NetworkPolicy::ntpRetryDue(false, true, 10, 0xffff8000u)); +} + TEST(NetworkPolicy, StartOtaUsesReachableSelectedNetworkByDefault) { EXPECT_TRUE(NetworkPolicy::startOtaUsesSelectedNetwork(false, true)); EXPECT_FALSE(NetworkPolicy::startOtaUsesSelectedNetwork(false, false)); diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index 2ccf6f5c..890faee2 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -190,7 +190,7 @@ build_src_filter = ${ThinkNode_M7.build_src_filter} +<../examples/kiss_modem/> ; Wi-Fi remains the default observer transport. Ethernet twins below select the -; onboard CH390 through NETWORK_USE_ETHERNET without enabling the legacy CLI +; onboard CH390 through the Ethernet-preferred network manager without enabling the legacy CLI ; transport. The board has PSRAM, so MAX_NEIGHBOURS enables ; WITH_MQTT_NEIGHBORS (see MQTTBridge.h). [env:ThinkNode_M7_repeater_observer_mqtt] @@ -242,7 +242,7 @@ extends = env:ThinkNode_M7_repeater_observer_mqtt build_flags = ${env:ThinkNode_M7_repeater_observer_mqtt.build_flags} ${ThinkNode_M7_ch390.build_flags} - -D NETWORK_USE_ETHERNET=1 + -D NETWORK_PREFER_ETHERNET=1 build_src_filter = ${env:ThinkNode_M7_repeater_observer_mqtt.build_src_filter} lib_deps = ${env:ThinkNode_M7_repeater_observer_mqtt.lib_deps} @@ -298,7 +298,7 @@ extends = env:ThinkNode_M7_room_server_observer_mqtt build_flags = ${env:ThinkNode_M7_room_server_observer_mqtt.build_flags} ${ThinkNode_M7_ch390.build_flags} - -D NETWORK_USE_ETHERNET=1 + -D NETWORK_PREFER_ETHERNET=1 build_src_filter = ${env:ThinkNode_M7_room_server_observer_mqtt.build_src_filter} lib_deps = ${env:ThinkNode_M7_room_server_observer_mqtt.lib_deps} diff --git a/webui/index.html b/webui/index.html index ab7b43d6..a71bd1e4 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1546,6 +1546,7 @@ var CLI_KEYS=[ ["wifi.ssid","WiFi network name",0], ["wifi.pwd","WiFi password",0], ["wifi.powersave","WiFi power-save mode",0,"none|min|max"], + ["link.status","Selected network, IP, signal and uptime",1], ["wifi.status","WiFi connection, IP, RSSI and uptime",1], ["mqtt.origin","Observer name in published messages",0], ["mqtt.iata","IATA region code used in topic paths",0], From cdde4bf673dbfba614fc72b4b61d7a5e08a1c9a7 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 2 Sep 2026 21:37:30 -0700 Subject: [PATCH 52/93] feat(mqtt): add link diagnostics and improve network logging --- MQTT_IMPLEMENTATION.md | 8 ++ examples/simple_repeater/MyMesh.cpp | 6 +- examples/simple_room_server/MyMesh.cpp | 6 +- scripts/webconfig_mock_server.py | 3 + src/helpers/CommonCLI_Observer.cpp | 2 + src/helpers/NetworkInterface.cpp | 117 ++++++++++++++++-- src/helpers/NetworkInterface.h | 1 + src/helpers/NetworkPolicy.h | 83 ++++++++++++- src/helpers/bridges/MQTTBridge.cpp | 58 ++++++--- .../test_network_policy.cpp | 62 +++++++++- webui/index.html | 1 + 11 files changed, 317 insertions(+), 30 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 774ff63a..69fd7b58 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -81,6 +81,7 @@ reboot ```bash get wifi.ssid get link.status +get link.diag get bridge.enabled get mqtt.rx get mqtt.tx @@ -547,6 +548,7 @@ These settings apply across all MQTT slots: - `get wifi.ssid` - Get WiFi SSID - `get wifi.pwd` - Get WiFi password - `get link.status` - Get the selected network medium, connection status, IP, signal when available, and uptime +- `get link.diag` - Explain automatic Ethernet selection using controller initialization, link event, IP, WiFi fallback, route-lock, and reason state - `get wifi.status` - WiFi-only compatibility alias; reports n/a when another medium is selected - `get wifi.powersave` - Get WiFi power save mode (none/min/max) @@ -859,6 +861,12 @@ the radio actually performs in that case. ### Connection Handling - Automatic reconnection with exponential backoff per slot; a slot that stays down through the full backoff ladder is retried on a slow periodic probe instead of hammering the broker +- Ethernet-preferred builds wait the full configured boot probe window for an Ethernet IP; + delayed or unavailable PHY carrier reporting does not shorten the DHCP deadline. The boot + selection log includes the measured probe duration. +- Ethernet/WiFi transitions are logged. A lost or changed route stops every started MQTT + client, including one whose disconnect callback arrived first; once a usable route returns, + route-caused backoff is cleared and one immediate reconnect is allowed - Packets are queued while a slot is disconnected and flushed when it recovers ### Raw Radio Data Capture diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index cfbe35d8..1d16accc 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1152,10 +1152,12 @@ void MyMesh::begin(FILESYSTEM *fs) { MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); + const uint32_t ethernet_probe_started_at = millis(); boot_network.bootstrap(obs->wifi_ssid, obs->wifi_password, NETWORK_ETHERNET_BOOT_WAIT_MS); - Serial.printf("Network: selected %s (%s)\n", boot_network.mediumName(), - boot_network.statusName()); + Serial.printf("Network: selected %s (%s) after %lums\n", + boot_network.mediumName(), boot_network.statusName(), + (unsigned long)(millis() - ethernet_probe_started_at)); } acl.load(_fs, self_id); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 01b157cc..386c36b4 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -953,10 +953,12 @@ void MyMesh::begin(FILESYSTEM *fs) { MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); + const uint32_t ethernet_probe_started_at = millis(); boot_network.bootstrap(obs->wifi_ssid, obs->wifi_password, NETWORK_ETHERNET_BOOT_WAIT_MS); - Serial.printf("Network: selected %s (%s)\n", boot_network.mediumName(), - boot_network.statusName()); + Serial.printf("Network: selected %s (%s) after %lums\n", + boot_network.mediumName(), boot_network.statusName(), + (unsigned long)(millis() - ethernet_probe_started_at)); } acl.load(_fs, self_id); diff --git a/scripts/webconfig_mock_server.py b/scripts/webconfig_mock_server.py index 547889a1..74c6b730 100644 --- a/scripts/webconfig_mock_server.py +++ b/scripts/webconfig_mock_server.py @@ -500,6 +500,9 @@ GETTERS = { "link.status": lambda c: ( "wifi: connected, IP: 192.168.1.42, RSSI: -58 dBm, Uptime: %dm" % (int(time.time() - ST.start) // 60)), + "link.diag": lambda c: ( + "why:ethernet-not-enabled selected:wifi\n" + "wifi:state:connected ip:192.168.1.42"), "mqtt.status": lambda c: cli_mqtt_status(c), "mqtt.presets": lambda c: "\n".join( "%2d. %s%s" % (i + 1, n, "" if nd == "none" else " (needs %s)" % nd) diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 28ac768b..273666be 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -1038,6 +1038,8 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf } else { strcpy(reply, _mqtt_prefs.wifi_password[0] ? "> ******** (serial only)" : "> (not set)"); } + } else if (strcmp(config, "link.diag") == 0) { + activeNetworkInterface().formatDiagnostics(reply, 160); } else if (memcmp(config, "link.status", 11) == 0 || memcmp(config, "wifi.status", 11) == 0) { NetworkInterface& network = activeNetworkInterface(); diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index ae5273e3..c22399a1 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -199,12 +199,31 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { bool resolveHost(const char* hostname, IPAddress& address) const override { return WiFi.hostByName(hostname, address); } + void formatDiagnostics(char* reply, size_t reply_size) const override { + snprintf(reply, reply_size, + "> why:ethernet-not-enabled selected:wifi\n" + "wifi:state:%s ip:%s", + statusName(), + localIP().toString().c_str()); + } }; #if defined(NETWORK_PREFER_ETHERNET) class EthernetNetworkInterface final : public NetworkInterfaceBase { + public: + enum class EventState : uint8_t { + None, + Started, + LinkDown, + LinkUp, + GotIp, + Stopped, + }; + + private: bool _started = false; bool _event_registered = false; + std::atomic _event_state{static_cast(EventState::None)}; public: const char* mediumName() const override { return "ethernet"; } @@ -220,14 +239,30 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { if (!_event_registered) { WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t) { switch (event) { + case ARDUINO_EVENT_ETH_START: + _event_state.store(static_cast(EventState::Started), + std::memory_order_relaxed); + break; + case ARDUINO_EVENT_ETH_CONNECTED: + _event_state.store(static_cast(EventState::LinkUp), + std::memory_order_relaxed); + break; case ARDUINO_EVENT_ETH_GOT_IP: + _event_state.store(static_cast(EventState::GotIp), + std::memory_order_relaxed); noteConnected(millis()); break; case ARDUINO_EVENT_ETH_DISCONNECTED: + _event_state.store(static_cast(EventState::LinkDown), + std::memory_order_relaxed); // Ethernet has no 802.11 reason code; zero means unavailable. noteDisconnected(millis(), 0); _connected_at.store(0, std::memory_order_relaxed); break; + case ARDUINO_EVENT_ETH_STOP: + _event_state.store(static_cast(EventState::Stopped), + std::memory_order_relaxed); + break; default: break; } @@ -282,6 +317,49 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { // DNS follows the selected esp_netif even though this entry point is named WiFi. return WiFi.hostByName(hostname, address); } + void formatDiagnostics(char* reply, size_t reply_size) const override { + snprintf(reply, reply_size, "> ethernet:%s ip=%s", + statusName(), localIP().toString().c_str()); + } + + EventState eventState() const { + return static_cast( + _event_state.load(std::memory_order_relaxed)); + } + bool sampleLink(bool& known) const { + known = false; + if (!_started) return false; + + // IEEE 802.3 BMSR link status is latch-low. Read it twice so the second + // value is the current carrier state rather than a remembered link flap. + (void)CH390.readPHY(0x01); + const uint32_t bmsr = CH390.readPHY(0x01) & 0xffffu; + if (bmsr != 0 && bmsr != 0xffffu) { + known = true; + return (bmsr & (1u << 2)) != 0; + } + + // A failed/unsupported direct PHY read can still use the driver's events. + const EventState state = eventState(); + known = state == EventState::LinkDown || state == EventState::LinkUp || + state == EventState::GotIp || state == EventState::Stopped; + return state == EventState::LinkUp || state == EventState::GotIp; + } + bool linkUp() const { + bool known = false; + return sampleLink(known); + } + const char* eventName() const { + switch (eventState()) { + case EventState::None: return "none"; + case EventState::Started: return "started"; + case EventState::LinkDown: return "link-down"; + case EventState::LinkUp: return "link-up"; + case EventState::GotIp: return "got-ip"; + case EventState::Stopped: return "stopped"; + } + return "unknown"; + } }; class AutomaticNetworkInterface final : public NetworkInterface { @@ -385,13 +463,9 @@ class AutomaticNetworkInterface final : public NetworkInterface { } const uint32_t started_at = millis(); - const uint32_t link_wait_ms = wait_ms < 1500 ? wait_ms : 1500; - while (_ethernet_started && !CH390.linkUp() && - (uint32_t)(millis() - started_at) < link_wait_ms) { - delay(25); - } - while (_ethernet_started && CH390.linkUp() && !_ethernet.isConnected() && - (uint32_t)(millis() - started_at) < wait_ms) { + while (NetworkPolicy::ethernetBootProbePending( + _ethernet_started, _ethernet.isConnected(), + (uint32_t)(millis() - started_at), wait_ms)) { delay(25); } @@ -489,6 +563,35 @@ class AutomaticNetworkInterface final : public NetworkInterface { return _selected != NetworkMedium::None && selectedInterface().resolveHost(hostname, address); } + void formatDiagnostics(char* reply, size_t reply_size) const override { + const bool ethernet_connected = _ethernet.isConnected(); + bool ethernet_link_known = false; + bool ethernet_link_up = _ethernet.sampleLink(ethernet_link_known); + ethernet_link_known = ethernet_link_known || ethernet_connected; + ethernet_link_up = ethernet_link_up || ethernet_connected; + const bool switching_locked = + _switch_locks.load(std::memory_order_relaxed) != 0; + const uint32_t now_ms = millis(); + const uint32_t ethernet_stable_ms = _ethernet_stable_since == 0 + ? 0 : (uint32_t)(now_ms - _ethernet_stable_since); + const NetworkDiagnosticReason reason = + NetworkPolicy::automaticDiagnosticReason( + _ethernet_started, ethernet_link_known, ethernet_link_up, + ethernet_connected, _selected, switching_locked, + ethernet_stable_ms); + snprintf(reply, reply_size, + "> why:%s selected:%s lock:%s\n" + "eth:init:%s evt:%s link:%s ip:%s\n" + "wifi:cfg:%s started:%s link:%s", + NetworkPolicy::diagnosticReasonName(reason), mediumName(), + switching_locked ? "yes" : "no", + _ethernet_started ? "ok" : "failed", _ethernet.eventName(), + ethernet_link_known ? (ethernet_link_up ? "up" : "down") + : "unknown", + _ethernet.localIP().toString().c_str(), + wifiConfigured() ? "yes" : "no", _wifi_started ? "yes" : "no", + (_wifi_started && _wifi.isConnected()) ? "up" : "down"); + } unsigned long connectedAtMillis() const override { return _selected == NetworkMedium::None ? 0 : selectedInterface().connectedAtMillis(); } diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkInterface.h index a5e15274..646a73ce 100644 --- a/src/helpers/NetworkInterface.h +++ b/src/helpers/NetworkInterface.h @@ -50,6 +50,7 @@ class NetworkInterface { virtual IPAddress localIP() const = 0; virtual int rssi() const = 0; // INT_MIN when the selected medium has no RSSI. virtual bool resolveHost(const char* hostname, IPAddress& address) const = 0; + virtual void formatDiagnostics(char* reply, size_t reply_size) const = 0; virtual unsigned long connectedAtMillis() const = 0; virtual uint8_t lastDisconnectReason() const = 0; diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h index fe434a03..5d083eb4 100644 --- a/src/helpers/NetworkPolicy.h +++ b/src/helpers/NetworkPolicy.h @@ -15,20 +15,39 @@ enum class NetworkMedium : uint8_t { WiFi, }; +enum class NetworkDiagnosticReason : uint8_t { + EthernetActive, + EthernetInitFailed, + EthernetLinkUnknown, + EthernetLinkDown, + EthernetAwaitingIp, + EthernetStabilizing, + SwitchingLocked, + EthernetReady, +}; + namespace NetworkPolicy { struct MQTTTransitionActions { - bool disconnect_slots; + bool stop_started_slots; bool retry_disconnected_slots_now; + bool reset_reconnect_backoff; }; static constexpr MQTTTransitionActions mqttActions(NetworkTransition transition) { return { transition == NetworkTransition::Down || transition == NetworkTransition::Switched, transition == NetworkTransition::Up || transition == NetworkTransition::Switched, + transition == NetworkTransition::Up || transition == NetworkTransition::Switched, }; } +static constexpr const char* mediumName(NetworkMedium medium) { + return medium == NetworkMedium::Ethernet ? "ethernet" + : medium == NetworkMedium::WiFi ? "wifi" + : "none"; +} + struct AutomaticSelectionInput { NetworkMedium selected; bool ethernet_connected; @@ -46,6 +65,58 @@ static constexpr uint32_t kEthernetDownGraceMs = 3000; static constexpr uint32_t kEthernetFailbackStableMs = 10000; static constexpr uint32_t kNtpRetryMs = 30000; +static inline NetworkDiagnosticReason automaticDiagnosticReason( + bool ethernet_initialized, bool ethernet_link_known, + bool ethernet_link_up, bool ethernet_connected, NetworkMedium selected, + bool switching_locked, uint32_t ethernet_stable_ms) { + if (!ethernet_initialized) { + return NetworkDiagnosticReason::EthernetInitFailed; + } + if (!ethernet_link_known) { + return NetworkDiagnosticReason::EthernetLinkUnknown; + } + if (!ethernet_link_up) { + return NetworkDiagnosticReason::EthernetLinkDown; + } + if (!ethernet_connected) { + return NetworkDiagnosticReason::EthernetAwaitingIp; + } + if (selected == NetworkMedium::Ethernet) { + return NetworkDiagnosticReason::EthernetActive; + } + if (switching_locked) { + return NetworkDiagnosticReason::SwitchingLocked; + } + if (selected == NetworkMedium::WiFi && + ethernet_stable_ms < kEthernetFailbackStableMs) { + return NetworkDiagnosticReason::EthernetStabilizing; + } + return NetworkDiagnosticReason::EthernetReady; +} + +static inline const char* diagnosticReasonName( + NetworkDiagnosticReason reason) { + switch (reason) { + case NetworkDiagnosticReason::EthernetActive: + return "ethernet-active"; + case NetworkDiagnosticReason::EthernetInitFailed: + return "ethernet-init-failed"; + case NetworkDiagnosticReason::EthernetLinkUnknown: + return "ethernet-link-unknown"; + case NetworkDiagnosticReason::EthernetLinkDown: + return "ethernet-link-down"; + case NetworkDiagnosticReason::EthernetAwaitingIp: + return "ethernet-awaiting-ip"; + case NetworkDiagnosticReason::EthernetStabilizing: + return "ethernet-stabilizing"; + case NetworkDiagnosticReason::SwitchingLocked: + return "switching-locked"; + case NetworkDiagnosticReason::EthernetReady: + return "ethernet-ready"; + } + return "unknown"; +} + static constexpr bool ntpPendingAfterConnectivitySample( bool synced, bool pending, bool was_connected, bool connected) { return pending || (!synced && connected && !was_connected); @@ -64,6 +135,16 @@ static constexpr NetworkMedium bootSelection(bool ethernet_connected, : NetworkMedium::None; } +// Once the Ethernet controller has initialized, allow the entire boot probe +// window for link negotiation and DHCP. PHY carrier is useful diagnostic data, +// but it must not shorten the advertised deadline when carrier reporting lags. +static constexpr bool ethernetBootProbePending(bool ethernet_initialized, + bool ethernet_connected, + uint32_t elapsed_ms, + uint32_t wait_ms) { + return ethernet_initialized && !ethernet_connected && elapsed_ms < wait_ms; +} + static inline NetworkMedium automaticSelection( const AutomaticSelectionInput& input) { if (input.switching_locked) return input.selected; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 12bf9a30..cfbe3ad5 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1307,16 +1307,7 @@ void MQTTBridge::mqttTaskLoop() { } #endif - bool network_just_connected = handleNetworkConnection(now); - if (network_just_connected) { - // The uplink recovered — reset last_reconnect_attempt for disconnected slots so they - // retry immediately rather than waiting up to 5 min for backoff timers to expire. - for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { - if (_slots[i].enabled && _slots[i].initial_connect_done && !_slots[i].connected) { - _slots[i].last_reconnect_attempt = 0; - } - } - } + handleNetworkConnection(now); // Schedule once per link-up edge. Failed syncs are owned by the 30-second // retry below; re-arming here on every loop would make a blocked NTP path @@ -2745,19 +2736,58 @@ void MQTTBridge::checkConfigurationMismatch() { } bool MQTTBridge::handleNetworkConnection(unsigned long now) { + const NetworkMedium previous_medium = _network->medium(); const NetworkTransition transition = _network->maintain((uint32_t)now, _obs->wifi_power_save); + const NetworkMedium selected_medium = _network->medium(); + + if (transition == NetworkTransition::Down) { + MQTT_DEBUG_PRINTLN("Network: %s link down", NetworkPolicy::mediumName(previous_medium)); + } else if (transition == NetworkTransition::Up) { + MQTT_DEBUG_PRINTLN("Network: %s link up (%s, IP %s)", + NetworkPolicy::mediumName(selected_medium), + _network->statusName(), _network->localIP().toString().c_str()); + } else if (transition == NetworkTransition::Switched) { + MQTT_DEBUG_PRINTLN("Network: switched %s -> %s (%s, IP %s)", + NetworkPolicy::mediumName(previous_medium), + NetworkPolicy::mediumName(selected_medium), + _network->statusName(), _network->localIP().toString().c_str()); + } + const NetworkPolicy::MQTTTransitionActions actions = NetworkPolicy::mqttActions(transition); - if (actions.disconnect_slots) { + if (actions.stop_started_slots) { // Broker ownership stays in the bridge. The physical adapter reports the - // edge; the bridge explicitly closes every slot instead of waiting for - // eventual socket timeouts. + // edge; the bridge explicitly stops every started slot instead of waiting + // for eventual socket timeouts. Do not gate this on slot.connected: the + // ESP-MQTT disconnect callback can clear that flag before the network edge + // reaches this task, but its client task and transport can still be alive. for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { - if (_slots[i].client && _slots[i].connected) { + if (_slots[i].client && _slots[i].client->isStarted()) { + MQTT_DEBUG_PRINTLN("MQTT%d stopping for network transition", i + 1); _slots[i].client->disconnect(); } + _slots[i].connected = false; + _slots[i].connected_at_ms = 0; } + updateCachedConnectionStatus(); + } + + if (actions.reset_reconnect_backoff) { + // A usable route is a new connection epoch. Failures earned on the old + // route must not strand the replacement route on the 5-minute rung or at + // the circuit breaker. Preserve JWTs and slot configuration, but give each + // disconnected active slot one immediate, freshly guarded attempt. + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].initial_connect_done && !_slots[i].connected) { + _slots[i].reconnect_backoff = 0; + _slots[i].max_backoff_failures = 0; + _slots[i].circuit_breaker_tripped = false; + _slots[i].last_reconnect_attempt = + now - MQTTConnectionPolicy::reconnectDelayMs(0, static_cast(i)); + } + } + _last_slot_reconnect_ms = now - MQTTConnectionPolicy::kReconnectGuardMs; } return actions.retry_disconnected_slots_now; } diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp index 93e0a1f2..b1131a14 100644 --- a/test/test_network_policy/test_network_policy.cpp +++ b/test/test_network_policy/test_network_policy.cpp @@ -4,26 +4,36 @@ TEST(NetworkPolicy, MqttDownDisconnectsSlotsWithoutRequestingImmediateRetry) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Down); - EXPECT_TRUE(actions.disconnect_slots); + EXPECT_TRUE(actions.stop_started_slots); EXPECT_FALSE(actions.retry_disconnected_slots_now); + EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, MqttUpRequestsImmediateRetryWithoutDisconnectingSlots) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Up); - EXPECT_FALSE(actions.disconnect_slots); + EXPECT_FALSE(actions.stop_started_slots); EXPECT_TRUE(actions.retry_disconnected_slots_now); + EXPECT_TRUE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, NoTransitionHasNoMqttSideEffects) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::None); - EXPECT_FALSE(actions.disconnect_slots); + EXPECT_FALSE(actions.stop_started_slots); EXPECT_FALSE(actions.retry_disconnected_slots_now); + EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, LinkSwitchReconnectsMqttSlotsImmediately) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Switched); - EXPECT_TRUE(actions.disconnect_slots); + EXPECT_TRUE(actions.stop_started_slots); EXPECT_TRUE(actions.retry_disconnected_slots_now); + EXPECT_TRUE(actions.reset_reconnect_backoff); +} + +TEST(NetworkPolicy, MediumNamesAreStableForTransitionLogs) { + EXPECT_STREQ("none", NetworkPolicy::mediumName(NetworkMedium::None)); + EXPECT_STREQ("ethernet", NetworkPolicy::mediumName(NetworkMedium::Ethernet)); + EXPECT_STREQ("wifi", NetworkPolicy::mediumName(NetworkMedium::WiFi)); } TEST(NetworkPolicy, BootPrefersEthernetAndOtherwiseUsesConfiguredWifi) { @@ -32,6 +42,15 @@ TEST(NetworkPolicy, BootPrefersEthernetAndOtherwiseUsesConfiguredWifi) { EXPECT_EQ(NetworkMedium::None, NetworkPolicy::bootSelection(false, false)); } +TEST(NetworkPolicy, EthernetBootProbeHonorsFullDeadlineUntilConnected) { + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 0, 8000)); + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 1500, 8000)); + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 7999, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(true, false, 8000, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(true, true, 100, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(false, false, 100, 8000)); +} + TEST(NetworkPolicy, EthernetFailureWaitsForGraceAndConnectedWifi) { NetworkPolicy::AutomaticSelectionInput input = { NetworkMedium::Ethernet, false, true, true, false, 0, @@ -57,6 +76,41 @@ TEST(NetworkPolicy, SwitchingLockPinsTheCurrentMedium) { EXPECT_EQ(NetworkMedium::WiFi, NetworkPolicy::automaticSelection(input)); } +TEST(NetworkPolicy, DiagnosticsIdentifyEthernetFallbackCause) { + using Reason = NetworkDiagnosticReason; + EXPECT_EQ(Reason::EthernetInitFailed, + NetworkPolicy::automaticDiagnosticReason( + false, false, false, false, NetworkMedium::WiFi, false, 0)); + EXPECT_EQ(Reason::EthernetLinkUnknown, + NetworkPolicy::automaticDiagnosticReason( + true, false, false, false, NetworkMedium::WiFi, false, 0)); + EXPECT_EQ(Reason::EthernetLinkDown, + NetworkPolicy::automaticDiagnosticReason( + true, true, false, false, NetworkMedium::WiFi, false, 0)); + EXPECT_EQ(Reason::EthernetAwaitingIp, + NetworkPolicy::automaticDiagnosticReason( + true, true, true, false, NetworkMedium::WiFi, false, 0)); +} + +TEST(NetworkPolicy, DiagnosticsExplainWhyReadyEthernetHasNotBeenSelected) { + using Reason = NetworkDiagnosticReason; + EXPECT_EQ(Reason::SwitchingLocked, + NetworkPolicy::automaticDiagnosticReason( + true, true, true, true, NetworkMedium::WiFi, true, + NetworkPolicy::kEthernetFailbackStableMs)); + EXPECT_EQ(Reason::EthernetStabilizing, + NetworkPolicy::automaticDiagnosticReason( + true, true, true, true, NetworkMedium::WiFi, false, + NetworkPolicy::kEthernetFailbackStableMs - 1)); + EXPECT_EQ(Reason::EthernetReady, + NetworkPolicy::automaticDiagnosticReason( + true, true, true, true, NetworkMedium::WiFi, false, + NetworkPolicy::kEthernetFailbackStableMs)); + EXPECT_EQ(Reason::EthernetActive, + NetworkPolicy::automaticDiagnosticReason( + true, true, true, true, NetworkMedium::Ethernet, false, 0)); +} + TEST(NetworkPolicy, NtpIsScheduledOncePerConnectivityEdge) { EXPECT_TRUE(NetworkPolicy::ntpPendingAfterConnectivitySample( false, false, false, true)); diff --git a/webui/index.html b/webui/index.html index a71bd1e4..6a851c1f 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1547,6 +1547,7 @@ var CLI_KEYS=[ ["wifi.pwd","WiFi password",0], ["wifi.powersave","WiFi power-save mode",0,"none|min|max"], ["link.status","Selected network, IP, signal and uptime",1], + ["link.diag","Ethernet selection and fallback diagnostics",1], ["wifi.status","WiFi connection, IP, RSSI and uptime",1], ["mqtt.origin","Observer name in published messages",0], ["mqtt.iata","IATA region code used in topic paths",0], From af2e3d98275e02c084d19473f5aa8aefee0b56f4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 3 Sep 2026 15:38:13 -0700 Subject: [PATCH 53/93] feat(mqtt): initialize shared network event runtime for Ethernet --- MQTT_IMPLEMENTATION.md | 2 ++ src/helpers/NetworkInterface.cpp | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 69fd7b58..f5a5f01e 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -864,6 +864,8 @@ the radio actually performs in that case. - Ethernet-preferred builds wait the full configured boot probe window for an Ethernet IP; delayed or unavailable PHY carrier reporting does not shorten the DHCP deadline. The boot selection log includes the measured probe duration. +- CH390 startup initializes Arduino's shared network event runtime without associating WiFi; + this keeps the framework's DNS and TLS hostname paths safe when Ethernet wins directly. - Ethernet/WiFi transitions are logged. A lost or changed route stops every started MQTT client, including one whose disconnect callback arrived first; once a usable route returns, route-caused backoff is cleared and one immediate reconnect is allowed diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index c22399a1..c8953aac 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -13,6 +13,12 @@ #if defined(NETWORK_PREFER_ETHERNET) #include "ethernet/ch390/CH390Config.h" + +// Arduino-ESP32's built-in ETH implementation calls this before bringing up +// Ethernet. It creates the shared Arduino network event group/task used by +// WiFiGenericClass::hostByName(), even when no Wi-Fi interface is started. +// ESP32-CH390 initializes esp_netif directly and omits this Arduino layer. +extern void tcpipInit(); #endif namespace { @@ -236,6 +242,10 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { bool begin(const char*, const char*) override { if (_started) return true; + // DNS and WiFiClientSecure are transport-neutral sockets in this Arduino + // core, but their hostname path still uses WiFiGeneric's event group. + // Initialize that shared runtime without enabling or associating Wi-Fi. + tcpipInit(); if (!_event_registered) { WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t) { switch (event) { From 17409e56a089cf85588502c9f6ee6490fd6a02b5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 3 Sep 2026 17:16:30 -0700 Subject: [PATCH 54/93] feat(network): implement hostname configuration for automatic interfaces --- examples/simple_repeater/MyMesh.cpp | 6 ++ examples/simple_room_server/MyMesh.cpp | 6 ++ src/helpers/NetworkHostname.h | 101 ++++++++++++++++++ src/helpers/NetworkInterface.cpp | 24 ++++- src/helpers/NetworkInterface.h | 3 + src/helpers/ethernet/ch390/CH390Config.h | 24 ++++- test/README.md | 1 + .../test_network_hostname.cpp | 95 ++++++++++++++++ 8 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 src/helpers/NetworkHostname.h create mode 100644 test/test_network_hostname/test_network_hostname.cpp diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 1d16accc..121183c4 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -4,6 +4,7 @@ #include // for qsort() #include #include +#include #if defined(ESP_PLATFORM) #include #endif @@ -1149,6 +1150,11 @@ void MyMesh::begin(FILESYSTEM *fs) { NetworkInterface& boot_network = activeNetworkInterface(); if (boot_network.isAutomatic()) { + char network_hostname[NetworkHostname::kBufferSize]; + NetworkHostname::build(network_hostname, sizeof(network_hostname), + _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); + boot_network.setHostname(network_hostname); + Serial.printf("Network: hostname %s\n", network_hostname); MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 386c36b4..c0bd7015 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #if defined(ESP_PLATFORM) #include #endif @@ -950,6 +951,11 @@ void MyMesh::begin(FILESYSTEM *fs) { NetworkInterface& boot_network = activeNetworkInterface(); if (boot_network.isAutomatic()) { + char network_hostname[NetworkHostname::kBufferSize]; + NetworkHostname::build(network_hostname, sizeof(network_hostname), + _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); + boot_network.setHostname(network_hostname); + Serial.printf("Network: hostname %s\n", network_hostname); MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); diff --git a/src/helpers/NetworkHostname.h b/src/helpers/NetworkHostname.h new file mode 100644 index 00000000..d4dfe7c4 --- /dev/null +++ b/src/helpers/NetworkHostname.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include + +namespace NetworkHostname { + +// ESP-IDF documents a 32-byte hostname limit. Keep one byte for the trailing +// NUL so the same buffer is safe through Arduino and lwIP APIs. +static constexpr size_t kMaxLength = 31; +static constexpr size_t kBufferSize = kMaxLength + 1; + +static inline bool isAsciiAlphaNumeric(uint8_t ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9'); +} + +static inline char asciiLower(uint8_t ch) { + return ch >= 'A' && ch <= 'Z' ? static_cast(ch + ('a' - 'A')) + : static_cast(ch); +} + +/** + * Build a DHCP-safe hostname from the MeshCore node name. + * + * The result is lowercase, starts with "meshcore-", contains only letters, + * digits and hyphens, and never exceeds ESP-IDF's practical 31-character + * payload limit. If the readable name must be shortened, six hex digits from + * the stable node identity are retained to reduce truncation collisions. + */ +static inline bool build(char* dest, size_t dest_size, const char* node_name, + const uint8_t* stable_id, size_t stable_id_size) { + if (!dest || dest_size == 0) return false; + dest[0] = '\0'; + + static constexpr char kPrefix[] = "meshcore-"; + static constexpr char kFallback[] = "node"; + static constexpr char kHex[] = "0123456789abcdef"; + static constexpr size_t kPrefixLength = sizeof(kPrefix) - 1; + static constexpr size_t kSuffixLength = 7; // '-' plus six hex digits. + + // NodePrefs::node_name currently holds at most 31 bytes. A larger scratch + // buffer keeps this helper safe and independently testable with other input. + char slug[64]; + size_t slug_length = 0; + bool separator_pending = false; + if (node_name) { + for (size_t i = 0; node_name[i] != '\0' && slug_length < sizeof(slug) - 1; + ++i) { + const uint8_t ch = static_cast(node_name[i]); + if (isAsciiAlphaNumeric(ch)) { + if (separator_pending && slug_length > 0 && + slug_length < sizeof(slug) - 1) { + slug[slug_length++] = '-'; + } + if (slug_length < sizeof(slug) - 1) { + slug[slug_length++] = asciiLower(ch); + } + separator_pending = false; + } else if (slug_length > 0) { + separator_pending = true; + } + } + } + + if (slug_length == 0) { + for (size_t i = 0; i < sizeof(kFallback) - 1; ++i) { + slug[slug_length++] = kFallback[i]; + } + } + slug[slug_length] = '\0'; + + size_t max_length = dest_size - 1; + if (max_length > kMaxLength) max_length = kMaxLength; + if (max_length == 0) return false; + + const bool needs_truncation = kPrefixLength + slug_length > max_length; + const bool can_add_identity = needs_truncation && stable_id && + stable_id_size >= 3 && max_length > kPrefixLength + kSuffixLength; + const size_t suffix_length = can_add_identity ? kSuffixLength : 0; + const size_t prefix_length = kPrefixLength < max_length + ? kPrefixLength : max_length; + size_t slug_budget = max_length - prefix_length - suffix_length; + if (slug_budget > slug_length) slug_budget = slug_length; + while (slug_budget > 0 && slug[slug_budget - 1] == '-') --slug_budget; + + size_t out = 0; + for (size_t i = 0; i < prefix_length; ++i) dest[out++] = kPrefix[i]; + for (size_t i = 0; i < slug_budget; ++i) dest[out++] = slug[i]; + if (can_add_identity) { + dest[out++] = '-'; + for (size_t i = 0; i < 3; ++i) { + dest[out++] = kHex[(stable_id[i] >> 4) & 0x0f]; + dest[out++] = kHex[stable_id[i] & 0x0f]; + } + } + dest[out] = '\0'; + return true; +} + +} // namespace NetworkHostname diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index c8953aac..dc0be519 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -77,6 +77,7 @@ class NetworkInterfaceBase : public NetworkInterface { class WiFiNetworkInterface final : public NetworkInterfaceBase { bool _event_registered = false; + char _hostname[32] = {}; char _ssid[33] = {}; char _password[65] = {}; unsigned long _last_reconnect_attempt = 0; @@ -112,6 +113,11 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { return wifi_ssid && wifi_ssid[0] != '\0'; } + void setHostname(const char* hostname) override { + strncpy(_hostname, hostname ? hostname : "", sizeof(_hostname) - 1); + _hostname[sizeof(_hostname) - 1] = '\0'; + } + bool begin(const char* wifi_ssid, const char* wifi_password) override { if (!configValid(wifi_ssid)) return false; strncpy(_ssid, wifi_ssid, sizeof(_ssid) - 1); @@ -119,6 +125,9 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { strncpy(_password, wifi_password ? wifi_password : "", sizeof(_password) - 1); _password[sizeof(_password) - 1] = '\0'; + // Arduino-ESP32 applies this stored value when it creates the STA netif. + // It must be set before WiFi.mode()/begin() for the first DHCP exchange. + if (_hostname[0] != '\0') WiFi.setHostname(_hostname); WiFi.mode(WIFI_STA); WiFi.setAutoReconnect(true); WiFi.setAutoConnect(true); @@ -229,6 +238,7 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { private: bool _started = false; bool _event_registered = false; + char _hostname[32] = {}; std::atomic _event_state{static_cast(EventState::None)}; public: @@ -240,6 +250,11 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { int statusCode() const override { return isConnected() ? 1 : 0; } bool configValid(const char*) const override { return true; } + void setHostname(const char* hostname) override { + strncpy(_hostname, hostname ? hostname : "", sizeof(_hostname) - 1); + _hostname[sizeof(_hostname) - 1] = '\0'; + } + bool begin(const char*, const char*) override { if (_started) return true; // DNS and WiFiClientSecure are transport-neutral sockets in this Arduino @@ -279,7 +294,7 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { }); _event_registered = true; } - _started = beginConfiguredCH390(); + _started = beginConfiguredCH390(_hostname); if (_started && isConnected()) noteConnected(millis()); return _started; } @@ -444,6 +459,13 @@ class AutomaticNetworkInterface final : public NetworkInterface { } bool isAutomatic() const override { return true; } + void setHostname(const char* hostname) override { + // Whichever medium wins now or during a later failover presents the same + // stable DHCP identity to the LAN. + _ethernet.setHostname(hostname); + _wifi.setHostname(hostname); + } + bool begin(const char* wifi_ssid, const char* wifi_password) override { rememberWifi(wifi_ssid, wifi_password); if (!_ethernet_started) { diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkInterface.h index 646a73ce..5ded0ff9 100644 --- a/src/helpers/NetworkInterface.h +++ b/src/helpers/NetworkInterface.h @@ -29,6 +29,9 @@ class NetworkInterface { virtual const char* statusName() const = 0; virtual int statusCode() const = 0; virtual bool configValid(const char* wifi_ssid) const = 0; + // Configure the DHCP hostname before begin()/bootstrap(). Implementations + // retain it for any later fallback interface start. + virtual void setHostname(const char* hostname) = 0; virtual bool begin(const char* wifi_ssid, const char* wifi_password) = 0; virtual NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) = 0; diff --git a/src/helpers/ethernet/ch390/CH390Config.h b/src/helpers/ethernet/ch390/CH390Config.h index 1b3a5bd9..30f110bf 100644 --- a/src/helpers/ethernet/ch390/CH390Config.h +++ b/src/helpers/ethernet/ch390/CH390Config.h @@ -7,7 +7,7 @@ * Shared by the companion transport wrapper and the observer network adapter so * pin and static-IP behavior cannot drift between the two paths. */ -static inline bool beginConfiguredCH390() { +static inline bool beginConfiguredCH390(const char* hostname = nullptr) { ch390_config_t config = CH390_DEFAULT_CONFIG(); config.spi_miso_gpio = ETH_MISO_PIN; config.spi_mosi_gpio = ETH_MOSI_PIN; @@ -16,6 +16,28 @@ static inline bool beginConfiguredCH390() { config.int_gpio = ETH_INT_PIN; if (!CH390.begin(config)) return false; + if (hostname && hostname[0] != '\0') { +#if !(defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET)) + // ESP32-CH390 creates and starts its esp_netif inside begin(), so its + // setHostname() cannot be called earlier. Stop the just-created DHCP + // client, apply the name to that exact Ethernet netif, then start a fresh + // DHCP transaction. This prevents a fast wired link from retaining the + // ESP-IDF default hostname for its initial lease. + const bool dhcp_stopped = CH390.disableDHCP(); + const bool hostname_set = CH390.setHostname(hostname); + const bool dhcp_started = CH390.enableDHCP(); + if (!dhcp_stopped || !hostname_set || !dhcp_started) { + Serial.printf("Network: could not fully apply Ethernet hostname %s\n", + hostname); + } +#else + if (!CH390.setHostname(hostname)) { + Serial.printf("Network: could not apply Ethernet hostname %s\n", + hostname); + } +#endif + } + #if defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET) IPAddress ip(ETHERNET_STATIC_IP); IPAddress gateway(ETHERNET_STATIC_GATEWAY); diff --git a/test/README.md b/test/README.md index 0cba8fff..ee33b4c5 100644 --- a/test/README.md +++ b/test/README.md @@ -30,6 +30,7 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_topic_router` | `src/helpers/MQTTTopicRouter.h` | complete preset/custom topic-routing contract; MeshRank all types except raw; required identifiers; invalid inputs/slots; exact buffer boundaries | | `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover; WiFi current-outage start sticky across STA reconnect attempts | | `test_network_policy` | `src/helpers/NetworkPolicy.h` | transport-neutral MQTT link-transition actions and `start ota` selected-LAN versus forced/fallback SoftAP choice | +| `test_network_hostname` | `src/helpers/NetworkHostname.h` | DHCP-safe `meshcore-` hostname generation; invalid-character collapse; empty fallback; exact length boundary; stable identity suffix on truncation | | `test_alert_fault_policy` | `src/helpers/AlertFaultPolicy.h` | WiFi/MQTT fault edge detector; `OutageSnapshot` (down / started_ms / initiating reason) fed to tick and `formatWifiAlert`; reason-8 reconnects change neither duration nor initiating reason; flap between status polls; down at `millis()==0`; packed 64-bit cross-task word; rate-limit floor and first-fire; 5 s poll cadence and `millis()` rollover | | `test_display_viewport` | `src/helpers/ui/DisplayViewport.h`, `src/helpers/ui/DisplayFrameSignature.h` | logical-to-physical portrait mapping; fractional span coverage; fitted-width conversion; preferred/fallback text scaling; stable visible-frame change detection | | `test_mqtt_packet_queue_policy` | `src/helpers/MQTTPacketQueuePolicy.h` | queue-full eviction; stale-disconnect flush; adaptive drain limits; bounded QoS0 retries; exact timing boundaries and 32-bit `millis()` rollover | diff --git a/test/test_network_hostname/test_network_hostname.cpp b/test/test_network_hostname/test_network_hostname.cpp new file mode 100644 index 00000000..8835c7d9 --- /dev/null +++ b/test/test_network_hostname/test_network_hostname.cpp @@ -0,0 +1,95 @@ +#include + +#include + +#include "helpers/NetworkHostname.h" + +namespace { + +const uint8_t kStableId[] = {0xab, 0xcd, 0x01}; + +void expectDhcpSafe(const char* hostname) { + ASSERT_NE(nullptr, hostname); + const size_t length = strlen(hostname); + EXPECT_GT(length, 0u); + EXPECT_LE(length, NetworkHostname::kMaxLength); + EXPECT_NE('-', hostname[0]); + EXPECT_NE('-', hostname[length - 1]); + for (size_t i = 0; i < length; ++i) { + const char ch = hostname[i]; + EXPECT_TRUE((ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9') || ch == '-') + << "invalid byte at " << i; + } +} + +} // namespace + +TEST(NetworkHostname, PrefixesAndLowercasesReadableNodeName) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + "Hill Repeater 2", kStableId, + sizeof(kStableId))); + EXPECT_STREQ("meshcore-hill-repeater-2", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, CollapsesInvalidRunsAndTrimsEdges) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + " Adam's___Hill / Repeater! ", + kStableId, sizeof(kStableId))); + EXPECT_STREQ("meshcore-adam-s-hill-repeater", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, UsesDocumentedFallbackForEmptySanitizedName) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), "___ !!!", + kStableId, sizeof(kStableId))); + EXPECT_STREQ("meshcore-node", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, KeepsExactBoundaryWithoutIdentitySuffix) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + "abcdefghijklmnopqrstuv", kStableId, + sizeof(kStableId))); + EXPECT_STREQ("meshcore-abcdefghijklmnopqrstuv", hostname); + EXPECT_EQ(NetworkHostname::kMaxLength, strlen(hostname)); +} + +TEST(NetworkHostname, TruncatesReadablePartAndAddsStableIdentitySuffix) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + kStableId, sizeof(kStableId))); + EXPECT_STREQ("meshcore-abcdefghijklmno-abcd01", hostname); + EXPECT_EQ(NetworkHostname::kMaxLength, strlen(hostname)); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, IdentitySuffixSeparatesOtherwiseCollidingNames) { + const uint8_t other_id[] = {0x12, 0x34, 0x56}; + char first[NetworkHostname::kBufferSize]; + char second[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(first, sizeof(first), + "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + kStableId, sizeof(kStableId))); + ASSERT_TRUE(NetworkHostname::build(second, sizeof(second), + "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + other_id, sizeof(other_id))); + EXPECT_STRNE(first, second); + EXPECT_STREQ("meshcore-abcdefghijklmno-123456", second); +} + +TEST(NetworkHostname, RejectsMissingOutputBuffer) { + EXPECT_FALSE(NetworkHostname::build(nullptr, 0, "node", kStableId, + sizeof(kStableId))); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 2c21406380c76400a3867d3b79dd8c9265c66e4f Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 3 Sep 2026 18:36:43 -0700 Subject: [PATCH 55/93] fix(network): set ethernet hostname before startup --- src/helpers/ethernet/ch390/CH390Config.h | 22 +++------------------- variants/thinknode_m7/platformio.ini | 2 +- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/helpers/ethernet/ch390/CH390Config.h b/src/helpers/ethernet/ch390/CH390Config.h index 30f110bf..656603ba 100644 --- a/src/helpers/ethernet/ch390/CH390Config.h +++ b/src/helpers/ethernet/ch390/CH390Config.h @@ -14,29 +14,13 @@ static inline bool beginConfiguredCH390(const char* hostname = nullptr) { config.spi_sck_gpio = ETH_SCLK_PIN; config.spi_cs_gpio = ETH_CS_PIN; config.int_gpio = ETH_INT_PIN; - if (!CH390.begin(config)) return false; - if (hostname && hostname[0] != '\0') { -#if !(defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET)) - // ESP32-CH390 creates and starts its esp_netif inside begin(), so its - // setHostname() cannot be called earlier. Stop the just-created DHCP - // client, apply the name to that exact Ethernet netif, then start a fresh - // DHCP transaction. This prevents a fast wired link from retaining the - // ESP-IDF default hostname for its initial lease. - const bool dhcp_stopped = CH390.disableDHCP(); - const bool hostname_set = CH390.setHostname(hostname); - const bool dhcp_started = CH390.enableDHCP(); - if (!dhcp_stopped || !hostname_set || !dhcp_started) { - Serial.printf("Network: could not fully apply Ethernet hostname %s\n", - hostname); - } -#else if (!CH390.setHostname(hostname)) { - Serial.printf("Network: could not apply Ethernet hostname %s\n", - hostname); + Serial.printf("Network: invalid Ethernet hostname %s\n", hostname); + return false; } -#endif } + if (!CH390.begin(config)) return false; #if defined(ETHERNET_STATIC_IP) && defined(ETHERNET_STATIC_GATEWAY) && defined(ETHERNET_STATIC_SUBNET) IPAddress ip(ETHERNET_STATIC_IP); diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index 890faee2..fa9a0140 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -47,7 +47,7 @@ build_flags = -D ETH_INT_PIN=45 -D ETHERNET_DEBUG_LOGGING=1 lib_deps = - https://github.com/liamcottle/ESP32-CH390.git#47b401f1de546118b03c18b5689dedec45871f2d + https://github.com/agessaman/ESP32-CH390.git#55c425705d9b21df11f95782ae6408707b2a91a9 ; Existing companion/CLI transport overlay. MQTT Ethernet observers consume the ; CH390 fragment above without ETHERNET_ENABLED, whose meaning in the simple From 50543c9b47d05f531d96fffa1e701c7bce428afd Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 3 Sep 2026 20:47:11 -0700 Subject: [PATCH 56/93] fix(network): harden automatic link transitions --- src/helpers/NetworkInterface.cpp | 3 +- src/helpers/NetworkPolicy.h | 5 +-- src/helpers/bridges/MQTTBridge.cpp | 5 ++- src/helpers/esp32/WebConfigServer.cpp | 34 +++++++++++++++++-- src/helpers/ethernet/ch390/CH390Config.h | 1 - .../test_network_policy.cpp | 10 ++++-- 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index dc0be519..ecf105ad 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -455,7 +455,8 @@ class AutomaticNetworkInterface final : public NetworkInterface { // Hardware availability and stored credentials are configuration. Current // link/DHCP state is runtime state and must not permanently suppress the // MQTT task that monitors for a late cable or lease. - return _ethernet_started || (wifi_ssid && wifi_ssid[0] != '\0'); + (void)wifi_ssid; + return true; } bool isAutomatic() const override { return true; } diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h index 5d083eb4..fc16ab2d 100644 --- a/src/helpers/NetworkPolicy.h +++ b/src/helpers/NetworkPolicy.h @@ -38,7 +38,7 @@ static constexpr MQTTTransitionActions mqttActions(NetworkTransition transition) return { transition == NetworkTransition::Down || transition == NetworkTransition::Switched, transition == NetworkTransition::Up || transition == NetworkTransition::Switched, - transition == NetworkTransition::Up || transition == NetworkTransition::Switched, + transition == NetworkTransition::Switched, }; } @@ -160,7 +160,8 @@ static inline NetworkMedium automaticSelection( if (input.selected == NetworkMedium::WiFi) { if (input.ethernet_connected && - input.ethernet_stable_ms >= kEthernetFailbackStableMs) { + (!input.wifi_connected || + input.ethernet_stable_ms >= kEthernetFailbackStableMs)) { return NetworkMedium::Ethernet; } return NetworkMedium::WiFi; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index cfbe3ad5..427106fa 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1252,7 +1252,10 @@ void MQTTBridge::mqttTaskLoop() { // Wait a bit for WiFi to start connecting vTaskDelay(pdMS_TO_TICKS(1000)); - bool network_was_connected = _network->isConnected(); + // Treat the first post-settle sample as an edge. A fast Wi-Fi association + // may complete during the delay above, before the task has sampled state; + // initializing from the live value would then defer NTP until its retry. + bool network_was_connected = false; unsigned long last_ntp_attempt = 0; // Main task loop diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 37740e8c..1aa638d0 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,24 @@ // so an untouched password field never overwrites the stored value. static const char SECRET_SENTINEL[] = "********"; +// esp_random() has a true entropy source only while RF is active. Ethernet +// LAN mode deliberately keeps Wi-Fi and Bluetooth down, so temporarily enable +// the bootloader RNG source while generating authentication secrets there. +class WebConfigEntropyGuard { + bool _enabled; + + public: + WebConfigEntropyGuard() + : _enabled(activeNetworkInterface().medium() == + NetworkMedium::Ethernet) { + if (_enabled) bootloader_random_enable(); + } + + ~WebConfigEntropyGuard() { + if (_enabled) bootloader_random_disable(); + } +}; + // Key classification (allowlist, secret detection, slot-prefix parsing) lives in // helpers/WebConfigKeys.h so it can be unit-tested on the host. Thin aliases keep // the call sites below readable. @@ -262,8 +281,11 @@ bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[ } _initial_setup = initial_setup; if (_initial_setup) { - for (int i = 0; i < 3; ++i) { - sprintf(&_setup_code[i * 4], "%04X", (unsigned)(esp_random() & 0xffff)); + { + WebConfigEntropyGuard entropy; + for (int i = 0; i < 3; ++i) { + sprintf(&_setup_code[i * 4], "%04X", (unsigned)(esp_random() & 0xffff)); + } } _setup_code[12] = 0; } else { @@ -730,7 +752,13 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { } _login_fails = 0; _login_lock_until = 0; - for (int i = 0; i < 4; i++) sprintf(&_session_token[i * 8], "%08lx", (unsigned long)esp_random()); + { + WebConfigEntropyGuard entropy; + for (int i = 0; i < 4; i++) { + sprintf(&_session_token[i * 8], "%08lx", + (unsigned long)esp_random()); + } + } _session_last_seen = now; AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); diff --git a/src/helpers/ethernet/ch390/CH390Config.h b/src/helpers/ethernet/ch390/CH390Config.h index 656603ba..05723067 100644 --- a/src/helpers/ethernet/ch390/CH390Config.h +++ b/src/helpers/ethernet/ch390/CH390Config.h @@ -17,7 +17,6 @@ static inline bool beginConfiguredCH390(const char* hostname = nullptr) { if (hostname && hostname[0] != '\0') { if (!CH390.setHostname(hostname)) { Serial.printf("Network: invalid Ethernet hostname %s\n", hostname); - return false; } } if (!CH390.begin(config)) return false; diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp index b1131a14..6d5f1f36 100644 --- a/test/test_network_policy/test_network_policy.cpp +++ b/test/test_network_policy/test_network_policy.cpp @@ -9,11 +9,11 @@ TEST(NetworkPolicy, MqttDownDisconnectsSlotsWithoutRequestingImmediateRetry) { EXPECT_FALSE(actions.reset_reconnect_backoff); } -TEST(NetworkPolicy, MqttUpRequestsImmediateRetryWithoutDisconnectingSlots) { +TEST(NetworkPolicy, MqttUpRetriesWithoutClearingBrokerCircuitBreaker) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Up); EXPECT_FALSE(actions.stop_started_slots); EXPECT_TRUE(actions.retry_disconnected_slots_now); - EXPECT_TRUE(actions.reset_reconnect_backoff); + EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, NoTransitionHasNoMqttSideEffects) { @@ -69,6 +69,12 @@ TEST(NetworkPolicy, FailbackRequiresStableEthernet) { EXPECT_EQ(NetworkMedium::Ethernet, NetworkPolicy::automaticSelection(input)); } +TEST(NetworkPolicy, DeadWifiFailsBackToReadyEthernetImmediately) { + const NetworkPolicy::AutomaticSelectionInput input = { + NetworkMedium::WiFi, true, false, true, false, 0, 0}; + EXPECT_EQ(NetworkMedium::Ethernet, NetworkPolicy::automaticSelection(input)); +} + TEST(NetworkPolicy, SwitchingLockPinsTheCurrentMedium) { const NetworkPolicy::AutomaticSelectionInput input = { NetworkMedium::WiFi, true, true, true, true, From 9b739a9b43a74671103f9da817665309ece21b6e Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 4 Sep 2026 08:42:46 -0700 Subject: [PATCH 57/93] fix(network): harden Ethernet-preferred lifecycle --- docs/cli_commands.md | 3 +- examples/simple_repeater/MyMesh.cpp | 11 +- examples/simple_repeater/main.cpp | 2 + examples/simple_room_server/MyMesh.cpp | 11 +- examples/simple_room_server/main.cpp | 2 + src/MeshCore.h | 3 + src/helpers/AlertReporter.cpp | 18 +- src/helpers/CommonCLI.cpp | 23 +- src/helpers/ESP32Board.cpp | 147 ++++++-- src/helpers/ESP32Board.h | 3 + src/helpers/MQTTConnectionPolicy.h | 14 + src/helpers/NetworkHostname.h | 16 +- src/helpers/NetworkInterface.cpp | 355 +++++++++++++----- src/helpers/NetworkInterface.h | 9 + src/helpers/NetworkPolicy.h | 51 ++- src/helpers/SNMPAgent.cpp | 4 +- src/helpers/bridges/MQTTBridge.cpp | 22 +- src/helpers/bridges/MQTTBridge.h | 14 +- src/helpers/esp32/HttpPort80Lease.h | 40 ++ src/helpers/esp32/WebConfigServer.cpp | 117 ++++-- src/helpers/esp32/WebConfigServer.h | 2 + .../test_mqtt_connection_policy.cpp | 13 + .../test_network_hostname.cpp | 46 ++- .../test_network_policy.cpp | 64 +++- variants/thinknode_m7/platformio.ini | 2 +- webui/index.html | 4 +- 26 files changed, 803 insertions(+), 193 deletions(-) create mode 100644 src/helpers/esp32/HttpPort80Lease.h diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 0c4ddf6a..6f9fc715 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -85,8 +85,9 @@ This document provides an overview of CLI commands that can be sent to MeshCore ### Start an Over-The-Air (OTA) firmware update **Usage:** -- `start ota` — serves the ElegantOTA web upload page on the station IP if joined to a Wi-Fi network, otherwise raises the `MeshCore-OTA` Wi-Fi hotspot. +- `start ota` — serves the ElegantOTA web upload page on the selected Ethernet or Wi-Fi IP when connected, otherwise raises the `MeshCore-OTA` Wi-Fi hotspot. The manual-OTA session stops after 15 minutes unless an upload is in progress. - `start ota ap` — always raises the `MeshCore-OTA` Wi-Fi hotspot, even when joined to a network. Use this when the network applies client isolation and the station IP isn't reachable. +- `stop ota` — stops an idle manual-OTA web server, releases port 80, and re-enables automatic network switching. It refuses while a firmware upload is in progress. --- diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 121183c4..9ab4bd99 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1550,15 +1550,17 @@ void MyMesh::onConfigBatchEnd() { // same sources as the REQ_TYPE_GET_STATUS reply and `get mqtt.stats`. void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; - int wifi_rssi = 0; + char wifi_rssi[12] = "null"; NetworkInterface& network = activeNetworkInterface(); + const char* network_medium = network.mediumName(); if (network.isConnected()) { strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); const int signal = network.rssi(); - wifi_rssi = signal == INT_MIN ? 0 : signal; + if (signal != INT_MIN) snprintf(wifi_rssi, sizeof(wifi_rssi), "%d", signal); } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); + network_medium = "wifi-ap"; } int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," @@ -1567,7 +1569,8 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { "\"airtime_s\":%lu,\"rx_airtime_s\":%lu," "\"recv\":%lu,\"sent\":%lu,\"rx_err\":%lu," "\"sent_flood\":%lu,\"sent_direct\":%lu,\"recv_flood\":%lu,\"recv_direct\":%lu," - "\"tx_queue\":%d,\"wifi_rssi\":%d,\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", + "\"tx_queue\":%d,\"wifi_rssi\":%s,\"network_medium\":\"%s\"," + "\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", (unsigned long)(uptime_millis / 1000), (unsigned)board.getBattMilliVolts(), (unsigned long)ESP.getFreeHeap(), (unsigned long)ESP.getMinFreeHeap(), (unsigned long)ESP.getMaxAllocHeap(), @@ -1578,7 +1581,7 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { (unsigned long)radio_driver.getPacketsRecvErrors(), (unsigned long)getNumSentFlood(), (unsigned long)getNumSentDirect(), (unsigned long)getNumRecvFlood(), (unsigned long)getNumRecvDirect(), - (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, ip, + (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, network_medium, ip, bridge ? bridge->getQueueSize() : 0); if (pos < 0 || pos >= (int)buf_size - 3) return; // truncated; snprintf terminated it bool first = true; diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index fa56772a..13b0828c 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -138,6 +138,8 @@ void setup() { } void loop() { + board.maintainOTAUpdate(millis()); + // Handle Serial CLI int len = strlen(command); while (Serial.available() && len < sizeof(command)-1) { diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index c0bd7015..64c61a2e 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1352,15 +1352,17 @@ void MyMesh::onConfigBatchEnd() { // same sources as the stats CLI replies and `get mqtt.stats`. void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; - int wifi_rssi = 0; + char wifi_rssi[12] = "null"; NetworkInterface& network = activeNetworkInterface(); + const char* network_medium = network.mediumName(); if (network.isConnected()) { strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); const int signal = network.rssi(); - wifi_rssi = signal == INT_MIN ? 0 : signal; + if (signal != INT_MIN) snprintf(wifi_rssi, sizeof(wifi_rssi), "%d", signal); } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); + network_medium = "wifi-ap"; } int pos = snprintf(buf, buf_size, "{\"uptime_s\":%lu,\"batt_mv\":%u," @@ -1369,7 +1371,8 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { "\"airtime_s\":%lu,\"rx_airtime_s\":%lu," "\"recv\":%lu,\"sent\":%lu,\"rx_err\":%lu," "\"sent_flood\":%lu,\"sent_direct\":%lu,\"recv_flood\":%lu,\"recv_direct\":%lu," - "\"tx_queue\":%d,\"wifi_rssi\":%d,\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", + "\"tx_queue\":%d,\"wifi_rssi\":%s,\"network_medium\":\"%s\"," + "\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", (unsigned long)(uptime_millis / 1000), (unsigned)board.getBattMilliVolts(), (unsigned long)ESP.getFreeHeap(), (unsigned long)ESP.getMinFreeHeap(), (unsigned long)ESP.getMaxAllocHeap(), @@ -1380,7 +1383,7 @@ void MyMesh::buildStatsJson(char* buf, size_t buf_size) { (unsigned long)radio_driver.getPacketsRecvErrors(), (unsigned long)getNumSentFlood(), (unsigned long)getNumSentDirect(), (unsigned long)getNumRecvFlood(), (unsigned long)getNumRecvDirect(), - (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, ip, + (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, network_medium, ip, bridge ? bridge->getQueueSize() : 0); if (pos < 0 || pos >= (int)buf_size - 3) return; // truncated; snprintf terminated it bool first = true; diff --git a/examples/simple_room_server/main.cpp b/examples/simple_room_server/main.cpp index 48f48799..fe4c17a9 100644 --- a/examples/simple_room_server/main.cpp +++ b/examples/simple_room_server/main.cpp @@ -114,6 +114,8 @@ void setup() { } void loop() { + board.maintainOTAUpdate(millis()); + int len = strlen(command); while (Serial.available() && len < sizeof(command)-1) { char c = Serial.read(); diff --git a/src/MeshCore.h b/src/MeshCore.h index adfc1c9e..f7995c81 100644 --- a/src/MeshCore.h +++ b/src/MeshCore.h @@ -69,6 +69,9 @@ public: virtual bool startOTAUpdate(const char* id, char reply[], bool force_ap) { return startOTAUpdate(id, reply); } + virtual bool stopOTAUpdate(char* reply) { (void)reply; return false; } + virtual bool isOTAUpdateInProgress() const { return false; } + virtual void maintainOTAUpdate(uint32_t now_ms) { (void)now_ms; } // Pull-based OTA: fetch the firmware build for this variant from a baked-in manifest and flash it. // current_ver is the running firmware version string (used to skip if already up to date); when // dry_run is true the build is only reported, not flashed. Observer (ESP32+WiFi) builds only. diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index d9617ca7..c473b274 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -208,10 +208,14 @@ void AlertReporter::onLoop(unsigned long now_ms) { const uint32_t min_interval_ms = AlertFaultPolicy::minIntervalMs(_obs->alert_min_interval_min); - // -------- WiFi fault -------- + // -------- Primary network fault -------- if (_obs->alert_wifi_minutes > 0) { if (_bridge != nullptr) { - const AlertFaultPolicy::OutageSnapshot snap = _bridge->getWifiOutageSnapshot(); + const AlertFaultPolicy::OutageSnapshot snap = + _bridge->getNetworkOutageSnapshot(); + const NetworkMedium alert_medium = _bridge->getNetworkAlertMedium(); + const char* alert_label = alert_medium == NetworkMedium::Ethernet + ? "Ethernet" : "WiFi"; AlertFaultPolicy::TickResult r = AlertFaultPolicy::tick( _wifi, now, snap, AlertFaultPolicy::thresholdMs(_obs->alert_wifi_minutes), @@ -219,20 +223,14 @@ void AlertReporter::onLoop(unsigned long now_ms) { if (r.action == AlertFaultPolicy::Action::FireDown) { char text[80]; AlertFaultPolicy::formatNetworkAlert( - text, sizeof(text), - activeNetworkInterface().medium() == NetworkMedium::Ethernet - ? "Ethernet" : "WiFi", - r, snap); + text, sizeof(text), alert_label, r, snap); if (sendChannel(text)) { AlertFaultPolicy::commitDown(_wifi, now, snap.started_ms); } } else if (r.action == AlertFaultPolicy::Action::FireRecovered) { char text[80]; AlertFaultPolicy::formatNetworkAlert( - text, sizeof(text), - activeNetworkInterface().medium() == NetworkMedium::Ethernet - ? "Ethernet" : "WiFi", - r, snap); + text, sizeof(text), alert_label, r, snap); sendChannel(text); AlertFaultPolicy::commitRecovered(_wifi); } diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 35d1822f..244db25b 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -1276,16 +1276,27 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re (int)mi.fordblks, (int)mi.uordblks, (int)mi.arena, (int)mi.ordblks, _callbacks->getQueueSize()); #endif - } else if (memcmp(command, "start ota", 9) == 0) { + } else if (memcmp(command, "start ota", 9) == 0 && + (command[9] == 0 || command[9] == ' ')) { // Manual OTA: bring up the ElegantOTA web UI for a hand-uploaded binary. // Plain "start ota" serves on the station IP when joined to WiFi, else // raises the MeshCore-OTA SoftAP. "start ota ap" forces the SoftAP even // when connected, so the UI is reachable when the network applies client - // isolation and the station IP can't be reached. (&& short-circuits keep - // the [10]/[11] reads in-bounds when command == "start ota".) - bool force_ap = (command[9] == ' ' && command[10] == 'a' && command[11] == 'p'); - if (!_board->startOTAUpdate(_prefs->node_name, reply, force_ap)) { - strcpy(reply, "Error"); + // isolation and the station IP can't be reached. + bool force_ap = command[9] == ' ' && strcmp(&command[10], "ap") == 0; + if (command[9] == ' ' && !force_ap) { + strcpy(reply, "ERR: usage start ota [ap]"); + } else { + reply[0] = 0; + if (!_board->startOTAUpdate(_prefs->node_name, reply, force_ap) && + reply[0] == 0) { + strcpy(reply, "Error"); + } + } + } else if (strcmp(command, "stop ota") == 0) { + reply[0] = 0; + if (!_board->stopOTAUpdate(reply) && reply[0] == 0) { + strcpy(reply, "ERR: OTA web server not running"); } } else if (memcmp(command, "clock", 5) == 0) { uint32_t now = getRTCClock()->getCurrentTime(); diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 19cdce77..20e079b0 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -9,14 +9,66 @@ #include #include #include +#include #include +#include + +#include "esp32/HttpPort80Lease.h" + +namespace { + +AsyncWebServer* ota_server = nullptr; +bool ota_routes_registered = false; +bool ota_server_running = false; +bool ota_raised_ap = false; +bool ota_network_locked = false; +uint32_t ota_started_at = 0; +char ota_id_buf[60]; +char ota_home_buf[90]; +char ota_url_buf[80]; + +void otaRegisterRoutes() { + ota_server->on("/", HTTP_GET, [](AsyncWebServerRequest* request) { + request->send(200, "text/html", ota_home_buf); + }); + ota_server->on("/log", HTTP_GET, [](AsyncWebServerRequest* request) { + request->send(SPIFFS, "/packet_log", "text/plain"); + }); + AsyncElegantOTA.begin(ota_server); + ota_routes_registered = true; +} + +void otaReleaseTransport() { + if (ota_server) ota_server->end(); + if (ota_raised_ap) WiFi.softAPdisconnect(true); + ota_raised_ap = false; + ota_server_running = false; + ota_started_at = 0; + if (ota_network_locked) { + activeNetworkInterface().unlockSwitching(); + ota_network_locked = false; + } + HttpPort80Lease::release(HttpPort80Lease::Owner::Ota); +} + +} // namespace bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { + if (ota_server_running) { + ota_started_at = millis(); + snprintf(reply, 160, "Started: %s", ota_url_buf); + return true; + } + if (!HttpPort80Lease::acquire(HttpPort80Lease::Owner::Ota)) { + snprintf(reply, 160, "Error: port 80 is in use by %s", + HttpPort80Lease::ownerName()); + return false; + } + inhibit_sleep = true; // prevent sleep during OTA - // Manual ElegantOTA owns port 80 and remains active until a successful upload - // reboots the device, so its route lock is intentionally reboot-scoped. activeNetworkInterface().lockSwitching(); + ota_network_locked = true; // If the device is already on its selected network, serve ElegantOTA on that // address so it is reachable without joining a separate AP. Otherwise raise @@ -29,38 +81,89 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { force_ap, activeNetworkInterface().isConnected())) { ip = activeNetworkInterface().localIP(); } else { - WiFi.softAP("MeshCore-OTA", NULL); + ota_raised_ap = WiFi.softAP("MeshCore-OTA", NULL); + if (!ota_raised_ap) { + otaReleaseTransport(); + inhibit_sleep = false; + strcpy(reply, "Error: failed to start OTA AP"); + return false; + } ip = WiFi.softAPIP(); } - sprintf(reply, "Started: http://%s/update", ip.toString().c_str()); + snprintf(ota_id_buf, sizeof(ota_id_buf), "%s (%s)", id, + getManufacturerName()); + snprintf(ota_home_buf, sizeof(ota_home_buf), + "

Hi! I am a MeshCore Repeater. ID: %s

", id); + snprintf(ota_url_buf, sizeof(ota_url_buf), "http://%s/update", + ip.toString().c_str()); + + if (!ota_server) { + ota_server = new (std::nothrow) AsyncWebServer(80); + if (!ota_server) { + otaReleaseTransport(); + inhibit_sleep = false; + strcpy(reply, "Error: insufficient memory for OTA server"); + return false; + } + } + AsyncElegantOTA.setID(ota_id_buf); + if (!ota_routes_registered) otaRegisterRoutes(); + ota_server->begin(); + if (ota_server->state() != LISTEN) { + otaReleaseTransport(); + inhibit_sleep = false; + strcpy(reply, "Error: failed to bind OTA server to port 80"); + return false; + } + + ota_server_running = true; + ota_started_at = millis(); + snprintf(reply, 160, "Started: %s", ota_url_buf); MESH_DEBUG_PRINTLN("startOTAUpdate: %s", reply); - static char id_buf[60]; - sprintf(id_buf, "%s (%s)", id, getManufacturerName()); - static char home_buf[90]; - sprintf(home_buf, "

Hi! I am a MeshCore Repeater. ID: %s

", id); - - AsyncWebServer* server = new AsyncWebServer(80); - - server->on("/", HTTP_GET, [](AsyncWebServerRequest *request) { - request->send(200, "text/html", home_buf); - }); - server->on("/log", HTTP_GET, [](AsyncWebServerRequest *request) { - request->send(SPIFFS, "/packet_log", "text/plain"); - }); - - AsyncElegantOTA.setID(id_buf); - AsyncElegantOTA.begin(server); // Start ElegantOTA - server->begin(); - return true; } +bool ESP32Board::stopOTAUpdate(char reply[]) { + if (!ota_server_running) return false; + if (Update.isRunning()) { + strcpy(reply, "Error: firmware upload is in progress"); + return false; + } + + otaReleaseTransport(); + inhibit_sleep = false; + strcpy(reply, "OK - OTA web server stopped"); + return true; +} + +bool ESP32Board::isOTAUpdateInProgress() const { + return ota_server_running && Update.isRunning(); +} + +void ESP32Board::maintainOTAUpdate(uint32_t now_ms) { + if (!ota_server_running || Update.isRunning()) return; + if (!NetworkPolicy::manualOtaTimeoutDue(now_ms, ota_started_at, false)) return; + char reply[160]; + if (stopOTAUpdate(reply)) MESH_DEBUG_PRINTLN("OTA: %s", reply); +} + #else bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { + (void)id; (void)reply; (void)force_ap; return false; // not supported } +bool ESP32Board::stopOTAUpdate(char reply[]) { + (void)reply; + return false; +} +bool ESP32Board::isOTAUpdateInProgress() const { + return false; +} +void ESP32Board::maintainOTAUpdate(uint32_t now_ms) { + (void)now_ms; +} #endif // --------------------------------------------------------------------------- diff --git a/src/helpers/ESP32Board.h b/src/helpers/ESP32Board.h index 08e264bc..02d0ac4b 100644 --- a/src/helpers/ESP32Board.h +++ b/src/helpers/ESP32Board.h @@ -158,6 +158,9 @@ public: } bool startOTAUpdate(const char* id, char reply[], bool force_ap = false) override; + bool stopOTAUpdate(char* reply) override; + bool isOTAUpdateInProgress() const override; + void maintainOTAUpdate(uint32_t now_ms) override; bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override; // Heavy body (TLS + JSON / HTTPUpdate). Runs in a dedicated large-stack task // spawned by otaFromManifest() — public only so that task entry point can call diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h index cf459459..6a463ebe 100644 --- a/src/helpers/MQTTConnectionPolicy.h +++ b/src/helpers/MQTTConnectionPolicy.h @@ -91,6 +91,20 @@ static inline bool circuitBreakerProbeDue(uint32_t now, uint32_t last_attempt) { return elapsedMs(now, last_attempt) >= kCircuitBreakerProbeMs; } +// Move a disconnected slot's deadline to "due now" after the physical link +// returns. Preserve its earned backoff and breaker state: a normal reconnect +// gets one immediate attempt at its current rung, while a tripped breaker gets +// one immediate probe. Unsigned subtraction keeps this wrap-safe. +static inline uint32_t immediateRetryLastAttempt(uint32_t now, + bool circuit_breaker_tripped, + uint8_t reconnect_backoff, + uint8_t slot_index) { + const uint32_t delay = circuit_breaker_tripped + ? kCircuitBreakerProbeMs + : reconnectDelayMs(reconnect_backoff, slot_index); + return now - delay; +} + // WiFi station reconnect backoff. The bridge drives its own STA reconnect loop // separate from the per-slot MQTT reconnects, with a slightly longer first rung // (15 s vs the slot ladder's 10 s). Extracted from handleWiFiConnection() so the diff --git a/src/helpers/NetworkHostname.h b/src/helpers/NetworkHostname.h index d4dfe7c4..caf6396b 100644 --- a/src/helpers/NetworkHostname.h +++ b/src/helpers/NetworkHostname.h @@ -25,8 +25,10 @@ static inline char asciiLower(uint8_t ch) { * * The result is lowercase, starts with "meshcore-", contains only letters, * digits and hyphens, and never exceeds ESP-IDF's practical 31-character - * payload limit. If the readable name must be shortened, six hex digits from - * the stable node identity are retained to reduce truncation collisions. + * payload limit. Six hex digits from the stable node identity are retained + * whenever non-ASCII bytes are removed, a fallback is needed, or the readable + * name must be shortened. This keeps lossy sanitization from assigning the + * same DHCP identity to unrelated nodes. */ static inline bool build(char* dest, size_t dest_size, const char* node_name, const uint8_t* stable_id, size_t stable_id_size) { @@ -44,6 +46,7 @@ static inline bool build(char* dest, size_t dest_size, const char* node_name, char slug[64]; size_t slug_length = 0; bool separator_pending = false; + bool removed_non_ascii = false; if (node_name) { for (size_t i = 0; node_name[i] != '\0' && slug_length < sizeof(slug) - 1; ++i) { @@ -57,12 +60,14 @@ static inline bool build(char* dest, size_t dest_size, const char* node_name, slug[slug_length++] = asciiLower(ch); } separator_pending = false; - } else if (slug_length > 0) { - separator_pending = true; + } else { + if (ch >= 0x80) removed_non_ascii = true; + if (slug_length > 0) separator_pending = true; } } } + const bool used_fallback = slug_length == 0; if (slug_length == 0) { for (size_t i = 0; i < sizeof(kFallback) - 1; ++i) { slug[slug_length++] = kFallback[i]; @@ -75,7 +80,8 @@ static inline bool build(char* dest, size_t dest_size, const char* node_name, if (max_length == 0) return false; const bool needs_truncation = kPrefixLength + slug_length > max_length; - const bool can_add_identity = needs_truncation && stable_id && + const bool needs_identity = needs_truncation || removed_non_ascii || used_fallback; + const bool can_add_identity = needs_identity && stable_id && stable_id_size >= 3 && max_length > kPrefixLength + kSuffixLength; const size_t suffix_length = can_add_identity ? kSuffixLength : 0; const size_t prefix_length = kPrefixLength < max_length diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkInterface.cpp index ecf105ad..73f2411d 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkInterface.cpp @@ -236,7 +236,7 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { }; private: - bool _started = false; + std::atomic _started{false}; bool _event_registered = false; char _hostname[32] = {}; std::atomic _event_state{static_cast(EventState::None)}; @@ -256,7 +256,7 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { } bool begin(const char*, const char*) override { - if (_started) return true; + if (_started.load(std::memory_order_acquire)) return true; // DNS and WiFiClientSecure are transport-neutral sockets in this Arduino // core, but their hostname path still uses WiFiGeneric's event group. // Initialize that shared runtime without enabling or associating Wi-Fi. @@ -294,9 +294,21 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { }); _event_registered = true; } - _started = beginConfiguredCH390(_hostname); - if (_started && isConnected()) noteConnected(millis()); - return _started; + _event_state.store(static_cast(EventState::None), + std::memory_order_relaxed); + const bool started = beginConfiguredCH390(_hostname); + _started.store(started, std::memory_order_release); + if (started && isConnected()) noteConnected(millis()); + return started; + } + + bool restart() { + CH390.end(); + _started.store(false, std::memory_order_release); + // Preserve status/outage history across attempts. maintain() observes the + // resulting edge in this same task iteration, so a retry cannot reset a + // prolonged-down alert timer or hide a previously connected transition. + return begin(nullptr, nullptr); } NetworkTransition maintain(uint32_t now_ms, uint8_t) override { @@ -334,7 +346,9 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { return NetworkTransition::Down; } - bool isConnected() const override { return _started && CH390.isConnected(); } + bool isConnected() const override { + return _started.load(std::memory_order_acquire) && CH390.isConnected(); + } IPAddress localIP() const override { return CH390.localIP(); } int rssi() const override { return INT_MIN; } bool resolveHost(const char* hostname, IPAddress& address) const override { @@ -351,9 +365,10 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { return static_cast( _event_state.load(std::memory_order_relaxed)); } + bool started() const { return _started.load(std::memory_order_acquire); } bool sampleLink(bool& known) const { known = false; - if (!_started) return false; + if (!_started.load(std::memory_order_acquire)) return false; // IEEE 802.3 BMSR link status is latch-low. Read it twice so the second // value is the current carrier state rather than a remembered link flap. @@ -390,22 +405,26 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { class AutomaticNetworkInterface final : public NetworkInterface { EthernetNetworkInterface _ethernet; WiFiNetworkInterface _wifi; - NetworkMedium _selected = NetworkMedium::None; - bool _ethernet_started = false; - bool _wifi_started = false; + std::atomic _selected{NetworkMedium::None}; + std::atomic _ethernet_started{false}; + std::atomic _wifi_started{false}; char _wifi_ssid[33] = {}; char _wifi_password[65] = {}; - uint32_t _ethernet_stable_since = 0; - uint32_t _selected_down_since = 0; + std::atomic _ethernet_stable_since{0}; + std::atomic _selected_down_since{0}; + std::atomic _last_ethernet_init_attempt{0}; + std::atomic _ethernet_retry_attempt{0}; + std::atomic _ethernet_no_ip_since{0}; std::atomic _switch_locks{0}; + std::atomic _switch_in_progress{false}; - NetworkInterface& selectedInterface() { - return _selected == NetworkMedium::Ethernet + NetworkInterface& selectedInterface(NetworkMedium selected) { + return selected == NetworkMedium::Ethernet ? static_cast(_ethernet) : static_cast(_wifi); } - const NetworkInterface& selectedInterface() const { - return _selected == NetworkMedium::Ethernet + const NetworkInterface& selectedInterface(NetworkMedium selected) const { + return selected == NetworkMedium::Ethernet ? static_cast(_ethernet) : static_cast(_wifi); } @@ -420,8 +439,40 @@ class AutomaticNetworkInterface final : public NetworkInterface { } void startWifiFallback() { - if (_wifi_started || !wifiConfigured()) return; - _wifi_started = _wifi.begin(_wifi_ssid, _wifi_password); + if (_wifi_started.load(std::memory_order_acquire) || !wifiConfigured()) return; + _wifi_started.store(_wifi.begin(_wifi_ssid, _wifi_password), + std::memory_order_release); + } + + bool beginUnlockedMutation() { + bool expected = false; + if (!_switch_in_progress.compare_exchange_strong( + expected, true, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + return false; + } + if (_switch_locks.load(std::memory_order_acquire) != 0) { + _switch_in_progress.store(false, std::memory_order_release); + return false; + } + return true; + } + + void endUnlockedMutation() { + _switch_in_progress.store(false, std::memory_order_release); + } + + bool selectIfUnlocked(NetworkMedium medium) { + if (!beginUnlockedMutation()) return false; + select(medium); + endUnlockedMutation(); + return true; + } + + void startWifiFallbackIfUnlocked() { + if (!beginUnlockedMutation()) return; + startWifiFallback(); + endUnlockedMutation(); } void select(NetworkMedium medium) { @@ -431,25 +482,46 @@ class AutomaticNetworkInterface final : public NetworkInterface { // on Wi-Fi after the manager has declared Ethernet active. WiFi.setAutoReconnect(false); WiFi.disconnect(false, false); - _wifi_started = false; + _wifi_started.store(false, std::memory_order_release); } - _selected = medium; - _selected_down_since = 0; + _selected_down_since.store(0, std::memory_order_relaxed); + _selected.store(medium, std::memory_order_release); + } + + bool startOrRetryEthernet(uint32_t now_ms, bool restart) { + const bool started = restart ? _ethernet.restart() + : _ethernet.begin(nullptr, nullptr); + _last_ethernet_init_attempt.store(now_ms, std::memory_order_relaxed); + _ethernet_started.store(started, std::memory_order_release); + if (started) { + _ethernet_retry_attempt.store(0, std::memory_order_relaxed); + } else { + uint8_t attempt = _ethernet_retry_attempt.load(std::memory_order_relaxed); + if (attempt != UINT8_MAX) ++attempt; + _ethernet_retry_attempt.store(attempt, std::memory_order_relaxed); + } + return started; } public: const char* mediumName() const override { - if (_selected == NetworkMedium::Ethernet) return "ethernet"; - if (_selected == NetworkMedium::WiFi) return "wifi"; + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + if (selected == NetworkMedium::Ethernet) return "ethernet"; + if (selected == NetworkMedium::WiFi) return "wifi"; return "none"; } - NetworkMedium medium() const override { return _selected; } + NetworkMedium medium() const override { + return _selected.load(std::memory_order_acquire); + } const char* statusName() const override { - return _selected == NetworkMedium::None ? "not_selected" - : selectedInterface().statusName(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None ? "not_selected" + : selectedInterface(selected).statusName(); } int statusCode() const override { - return _selected == NetworkMedium::None ? 0 : selectedInterface().statusCode(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None ? 0 + : selectedInterface(selected).statusCode(); } bool configValid(const char* wifi_ssid) const override { // Hardware availability and stored credentials are configuration. Current @@ -469,36 +541,51 @@ class AutomaticNetworkInterface final : public NetworkInterface { bool begin(const char* wifi_ssid, const char* wifi_password) override { rememberWifi(wifi_ssid, wifi_password); - if (!_ethernet_started) { - _ethernet_started = _ethernet.begin(nullptr, nullptr); + if (!_ethernet_started.load(std::memory_order_acquire)) { + const uint32_t now_ms = millis(); + const uint8_t attempt = + _ethernet_retry_attempt.load(std::memory_order_relaxed); + if (NetworkPolicy::ethernetInitRetryDue( + attempt, now_ms, + _last_ethernet_init_attempt.load(std::memory_order_relaxed))) { + startOrRetryEthernet(now_ms, attempt != 0); + } } // bootstrap() owns the initial choice. MQTT begin() is intentionally // idempotent and cannot demote a boot-selected Ethernet link because of a // momentary status sample between tasks. - if (_selected == NetworkMedium::None) { + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + if (selected == NetworkMedium::None) { if (_ethernet.isConnected()) { select(NetworkMedium::Ethernet); } else if (wifiConfigured()) { startWifiFallback(); - _selected = NetworkMedium::WiFi; + _selected.store(NetworkMedium::WiFi, std::memory_order_release); } - } else if (_selected == NetworkMedium::WiFi) { + } else if (selected == NetworkMedium::WiFi) { startWifiFallback(); } - return _ethernet_started || _wifi_started; + return _ethernet_started.load(std::memory_order_acquire) || + _wifi_started.load(std::memory_order_acquire); } bool bootstrap(const char* wifi_ssid, const char* wifi_password, uint32_t wait_ms) override { rememberWifi(wifi_ssid, wifi_password); - if (!_ethernet_started) { - _ethernet_started = _ethernet.begin(nullptr, nullptr); + if (!_ethernet_started.load(std::memory_order_acquire)) { + startOrRetryEthernet(millis(), false); } const uint32_t started_at = millis(); - while (NetworkPolicy::ethernetBootProbePending( - _ethernet_started, _ethernet.isConnected(), - (uint32_t)(millis() - started_at), wait_ms)) { + for (;;) { + bool link_known = false; + const bool link_up = _ethernet.sampleLink(link_known); + if (!NetworkPolicy::ethernetBootProbePending( + _ethernet_started.load(std::memory_order_acquire), + _ethernet.isConnected(), link_known, link_up, + (uint32_t)(millis() - started_at), wait_ms)) { + break; + } delay(25); } @@ -508,15 +595,60 @@ class AutomaticNetworkInterface final : public NetworkInterface { select(initial); } else { startWifiFallback(); - _selected = initial; + _selected.store(initial, std::memory_order_release); } return initial != NetworkMedium::None; } NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) override { + bool ethernet_started = _ethernet_started.load(std::memory_order_acquire); + const bool switching_locked = + _switch_locks.load(std::memory_order_acquire) != 0; + const bool ethernet_stopped = + ethernet_started && + _ethernet.eventState() == EthernetNetworkInterface::EventState::Stopped; + if (ethernet_stopped) { + _ethernet_started.store(false, std::memory_order_release); + ethernet_started = false; + if (_ethernet_retry_attempt.load(std::memory_order_relaxed) == 0) { + _ethernet_retry_attempt.store(1, std::memory_order_relaxed); + _last_ethernet_init_attempt.store(now_ms, std::memory_order_relaxed); + } + } + const EthernetNetworkInterface::EventState ethernet_event = + _ethernet.eventState(); + const bool ethernet_link_up = + ethernet_event == EthernetNetworkInterface::EventState::LinkUp || + ethernet_event == EthernetNetworkInterface::EventState::GotIp; + if (ethernet_started && ethernet_link_up && !_ethernet.isConnected()) { + if (_ethernet_no_ip_since.load(std::memory_order_relaxed) == 0) { + uint32_t started_at = now_ms; + if (started_at == 0) started_at = 1; + _ethernet_no_ip_since.store(started_at, std::memory_order_relaxed); + } + } else { + _ethernet_no_ip_since.store(0, std::memory_order_relaxed); + } + if (!switching_locked && NetworkPolicy::ethernetNoIpRecoveryDue( + ethernet_started, ethernet_link_up, _ethernet.isConnected(), + now_ms, _ethernet_no_ip_since.load(std::memory_order_relaxed))) { + ethernet_started = startOrRetryEthernet(now_ms, true); + _ethernet_no_ip_since.store(0, std::memory_order_relaxed); + } + if (!ethernet_started && !switching_locked) { + const uint8_t attempt = + _ethernet_retry_attempt.load(std::memory_order_relaxed); + if (NetworkPolicy::ethernetInitRetryDue( + attempt, now_ms, + _last_ethernet_init_attempt.load(std::memory_order_relaxed))) { + ethernet_started = startOrRetryEthernet(now_ms, true); + } + } + const NetworkTransition ethernet_transition = _ethernet.maintain(now_ms, wifi_power_save); - const NetworkTransition wifi_transition = _wifi_started + const bool wifi_started = _wifi_started.load(std::memory_order_acquire); + const NetworkTransition wifi_transition = wifi_started ? _wifi.maintain(now_ms, wifi_power_save) : NetworkTransition::None; @@ -525,56 +657,73 @@ class AutomaticNetworkInterface final : public NetworkInterface { // ESP-IDF's default-route priority once it came up. Tear the unused STA // down even without a selection edge, and force MQTT to reconnect if it // had already become reachable. - if (_selected == NetworkMedium::Ethernet && _ethernet.isConnected() && - _wifi_started) { + NetworkMedium selected = _selected.load(std::memory_order_acquire); + if (selected == NetworkMedium::Ethernet && _ethernet.isConnected() && + wifi_started) { const bool wifi_had_route = _wifi.isConnected(); - select(NetworkMedium::Ethernet); - if (wifi_had_route) return NetworkTransition::Switched; + if (selectIfUnlocked(NetworkMedium::Ethernet) && wifi_had_route) { + return NetworkTransition::Switched; + } } if (_ethernet.isConnected()) { - if (_ethernet_stable_since == 0) _ethernet_stable_since = now_ms; + if (_ethernet_stable_since.load(std::memory_order_relaxed) == 0) { + _ethernet_stable_since.store(now_ms, std::memory_order_relaxed); + } } else { - _ethernet_stable_since = 0; + _ethernet_stable_since.store(0, std::memory_order_relaxed); } const bool selected_connected = isConnected(); if (!selected_connected) { - if (_selected_down_since == 0) _selected_down_since = now_ms; + if (_selected_down_since.load(std::memory_order_relaxed) == 0) { + _selected_down_since.store(now_ms, std::memory_order_relaxed); + } } else { - _selected_down_since = 0; + _selected_down_since.store(0, std::memory_order_relaxed); } - const uint32_t selected_down_ms = _selected_down_since == 0 - ? 0 : (uint32_t)(now_ms - _selected_down_since); - if (_selected == NetworkMedium::Ethernet && !_ethernet.isConnected() && + const uint32_t selected_down_since = + _selected_down_since.load(std::memory_order_relaxed); + const uint32_t selected_down_ms = selected_down_since == 0 + ? 0 : (uint32_t)(now_ms - selected_down_since); + selected = _selected.load(std::memory_order_acquire); + if (selected == NetworkMedium::Ethernet && !_ethernet.isConnected() && selected_down_ms >= NetworkPolicy::kEthernetDownGraceMs) { - startWifiFallback(); + startWifiFallbackIfUnlocked(); } - const uint32_t ethernet_stable_ms = _ethernet_stable_since == 0 - ? 0 : (uint32_t)(now_ms - _ethernet_stable_since); + const uint32_t ethernet_stable_since = + _ethernet_stable_since.load(std::memory_order_relaxed); + const uint32_t ethernet_stable_ms = ethernet_stable_since == 0 + ? 0 : (uint32_t)(now_ms - ethernet_stable_since); + const bool selection_locked = + _switch_locks.load(std::memory_order_acquire) != 0; const NetworkPolicy::AutomaticSelectionInput input = { - _selected, _ethernet.isConnected(), - _wifi_started && _wifi.isConnected(), wifiConfigured(), - _switch_locks.load(std::memory_order_relaxed) != 0, + selected, _ethernet.isConnected(), + _wifi_started.load(std::memory_order_acquire) && _wifi.isConnected(), + wifiConfigured(), selection_locked, ethernet_stable_ms, selected_down_ms}; const NetworkMedium next = NetworkPolicy::automaticSelection(input); - if (next != _selected) { - const NetworkMedium previous = _selected; - select(next); - return previous == NetworkMedium::None ? NetworkTransition::Up - : NetworkTransition::Switched; + if (next != selected) { + if (selectIfUnlocked(next)) { + return selected == NetworkMedium::None ? NetworkTransition::Up + : NetworkTransition::Switched; + } } - if (_selected == NetworkMedium::Ethernet) return ethernet_transition; - if (_selected == NetworkMedium::WiFi) return wifi_transition; + if (selected == NetworkMedium::Ethernet) return ethernet_transition; + if (selected == NetworkMedium::WiFi) return wifi_transition; return NetworkTransition::None; } void lockSwitching() override { - _switch_locks.fetch_add(1, std::memory_order_relaxed); + uint8_t value = _switch_locks.load(std::memory_order_relaxed); + while (value != UINT8_MAX && !_switch_locks.compare_exchange_weak( + value, static_cast(value + 1), + std::memory_order_acq_rel, std::memory_order_relaxed)) {} + while (_switch_in_progress.load(std::memory_order_acquire)) delay(1); } void unlockSwitching() override { uint8_t value = _switch_locks.load(std::memory_order_relaxed); @@ -584,60 +733,96 @@ class AutomaticNetworkInterface final : public NetworkInterface { } bool isConnected() const override { - return _selected != NetworkMedium::None && selectedInterface().isConnected(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected != NetworkMedium::None && + selectedInterface(selected).isConnected(); } IPAddress localIP() const override { - return _selected == NetworkMedium::None ? IPAddress() : selectedInterface().localIP(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None ? IPAddress() + : selectedInterface(selected).localIP(); } int rssi() const override { - return _selected == NetworkMedium::None ? INT_MIN : selectedInterface().rssi(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None ? INT_MIN + : selectedInterface(selected).rssi(); } bool resolveHost(const char* hostname, IPAddress& address) const override { - return _selected != NetworkMedium::None && - selectedInterface().resolveHost(hostname, address); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected != NetworkMedium::None && + selectedInterface(selected).resolveHost(hostname, address); } void formatDiagnostics(char* reply, size_t reply_size) const override { + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + const bool ethernet_started = + _ethernet_started.load(std::memory_order_acquire); + const bool wifi_started = _wifi_started.load(std::memory_order_acquire); const bool ethernet_connected = _ethernet.isConnected(); bool ethernet_link_known = false; bool ethernet_link_up = _ethernet.sampleLink(ethernet_link_known); ethernet_link_known = ethernet_link_known || ethernet_connected; ethernet_link_up = ethernet_link_up || ethernet_connected; const bool switching_locked = - _switch_locks.load(std::memory_order_relaxed) != 0; + _switch_locks.load(std::memory_order_acquire) != 0; const uint32_t now_ms = millis(); - const uint32_t ethernet_stable_ms = _ethernet_stable_since == 0 - ? 0 : (uint32_t)(now_ms - _ethernet_stable_since); + const uint32_t ethernet_stable_since = + _ethernet_stable_since.load(std::memory_order_relaxed); + const uint32_t ethernet_stable_ms = ethernet_stable_since == 0 + ? 0 : (uint32_t)(now_ms - ethernet_stable_since); + const uint8_t retry_attempt = + _ethernet_retry_attempt.load(std::memory_order_relaxed); + const uint32_t retry_delay = + NetworkPolicy::ethernetInitRetryDelayMs(retry_attempt); + const uint32_t since_attempt = (uint32_t)( + now_ms - _last_ethernet_init_attempt.load(std::memory_order_relaxed)); + const uint32_t retry_in = !ethernet_started && retry_delay > since_attempt + ? retry_delay - since_attempt : 0; const NetworkDiagnosticReason reason = NetworkPolicy::automaticDiagnosticReason( - _ethernet_started, ethernet_link_known, ethernet_link_up, - ethernet_connected, _selected, switching_locked, + ethernet_started, ethernet_link_known, ethernet_link_up, + ethernet_connected, selected, switching_locked, ethernet_stable_ms); snprintf(reply, reply_size, "> why:%s selected:%s lock:%s\n" - "eth:init:%s evt:%s link:%s ip:%s\n" + "eth:init:%s retry:%u/%lums evt:%s link:%s ip:%s\n" "wifi:cfg:%s started:%s link:%s", - NetworkPolicy::diagnosticReasonName(reason), mediumName(), + NetworkPolicy::diagnosticReasonName(reason), + NetworkPolicy::mediumName(selected), switching_locked ? "yes" : "no", - _ethernet_started ? "ok" : "failed", _ethernet.eventName(), + ethernet_started ? "ok" : "failed", retry_attempt, + (unsigned long)retry_in, _ethernet.eventName(), ethernet_link_known ? (ethernet_link_up ? "up" : "down") : "unknown", _ethernet.localIP().toString().c_str(), - wifiConfigured() ? "yes" : "no", _wifi_started ? "yes" : "no", - (_wifi_started && _wifi.isConnected()) ? "up" : "down"); + wifiConfigured() ? "yes" : "no", wifi_started ? "yes" : "no", + (wifi_started && _wifi.isConnected()) ? "up" : "down"); } unsigned long connectedAtMillis() const override { - return _selected == NetworkMedium::None ? 0 : selectedInterface().connectedAtMillis(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None + ? 0 : selectedInterface(selected).connectedAtMillis(); } uint8_t lastDisconnectReason() const override { - return _selected == NetworkMedium::None ? 0 : selectedInterface().lastDisconnectReason(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None + ? 0 : selectedInterface(selected).lastDisconnectReason(); } unsigned long lastDisconnectTime() const override { - return _selected == NetworkMedium::None ? 0 : selectedInterface().lastDisconnectTime(); + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None + ? 0 : selectedInterface(selected).lastDisconnectTime(); } AlertFaultPolicy::OutageSnapshot outageSnapshot() const override { - return _selected == NetworkMedium::None + const NetworkMedium selected = _selected.load(std::memory_order_acquire); + return selected == NetworkMedium::None ? AlertFaultPolicy::OutageSnapshot{false, 0, 0} - : selectedInterface().outageSnapshot(); + : selectedInterface(selected).outageSnapshot(); + } + NetworkMedium alertMedium() const override { + return NetworkMedium::Ethernet; + } + AlertFaultPolicy::OutageSnapshot alertOutageSnapshot() const override { + return _ethernet.outageSnapshot(); } }; #endif diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkInterface.h index 5ded0ff9..339c1470 100644 --- a/src/helpers/NetworkInterface.h +++ b/src/helpers/NetworkInterface.h @@ -59,6 +59,15 @@ class NetworkInterface { virtual uint8_t lastDisconnectReason() const = 0; virtual unsigned long lastDisconnectTime() const = 0; virtual AlertFaultPolicy::OutageSnapshot outageSnapshot() const = 0; + + // Fault reporting normally follows the selected interface. Automatic + // Ethernet-preferred builds instead keep reporting the primary Ethernet + // outage while healthy Wi-Fi carries traffic, so prolonged degradation is + // not hidden by a successful fallback. + virtual NetworkMedium alertMedium() const { return medium(); } + virtual AlertFaultPolicy::OutageSnapshot alertOutageSnapshot() const { + return outageSnapshot(); + } }; /** Build-selected singleton. Wi-Fi is the compatibility default. */ diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h index fc16ab2d..10d7da69 100644 --- a/src/helpers/NetworkPolicy.h +++ b/src/helpers/NetworkPolicy.h @@ -63,7 +63,40 @@ struct AutomaticSelectionInput { // preempt a working Wi-Fi fallback. static constexpr uint32_t kEthernetDownGraceMs = 3000; static constexpr uint32_t kEthernetFailbackStableMs = 10000; +static constexpr uint32_t kEthernetNoLinkBootGraceMs = 750; +static constexpr uint32_t kEthernetNoIpRecoveryMs = 120000; +static constexpr uint32_t kEthernetInitRetryMinMs = 5000; +static constexpr uint32_t kEthernetInitRetryMaxMs = 300000; static constexpr uint32_t kNtpRetryMs = 30000; +static constexpr uint32_t kManualOtaSessionTimeoutMs = + 15UL * 60UL * 1000UL; + +static inline uint32_t ethernetInitRetryDelayMs(uint8_t attempt) { + if (attempt == 0) return 0; + uint32_t delay_ms = kEthernetInitRetryMinMs; + for (uint8_t i = 1; i < attempt && delay_ms < kEthernetInitRetryMaxMs; ++i) { + delay_ms = delay_ms > kEthernetInitRetryMaxMs / 2 + ? kEthernetInitRetryMaxMs : delay_ms * 2; + } + return delay_ms; +} + +static inline bool ethernetInitRetryDue(uint8_t attempt, + uint32_t now_ms, + uint32_t last_attempt_ms) { + return attempt == 0 || + (uint32_t)(now_ms - last_attempt_ms) >= + ethernetInitRetryDelayMs(attempt); +} + +static constexpr bool ethernetNoIpRecoveryDue(bool initialized, + bool link_up, + bool connected, + uint32_t now_ms, + uint32_t no_ip_since_ms) { + return initialized && link_up && !connected && no_ip_since_ms != 0 && + (uint32_t)(now_ms - no_ip_since_ms) >= kEthernetNoIpRecoveryMs; +} static inline NetworkDiagnosticReason automaticDiagnosticReason( bool ethernet_initialized, bool ethernet_link_known, @@ -135,14 +168,17 @@ static constexpr NetworkMedium bootSelection(bool ethernet_connected, : NetworkMedium::None; } -// Once the Ethernet controller has initialized, allow the entire boot probe -// window for link negotiation and DHCP. PHY carrier is useful diagnostic data, -// but it must not shorten the advertised deadline when carrier reporting lags. +// Give the PHY a short window to report carrier. Once a definitive link-down +// sample arrives, do not stall mesh startup for the full DHCP deadline. A +// present link (or an unknown PHY state) still receives the entire probe. static constexpr bool ethernetBootProbePending(bool ethernet_initialized, bool ethernet_connected, + bool link_known, + bool link_up, uint32_t elapsed_ms, uint32_t wait_ms) { - return ethernet_initialized && !ethernet_connected && elapsed_ms < wait_ms; + return ethernet_initialized && !ethernet_connected && elapsed_ms < wait_ms && + (!link_known || link_up || elapsed_ms < kEthernetNoLinkBootGraceMs); } static inline NetworkMedium automaticSelection( @@ -179,4 +215,11 @@ static constexpr bool startOtaUsesSelectedNetwork(bool force_ap, return !force_ap && network_connected; } +static constexpr bool manualOtaTimeoutDue(uint32_t now_ms, + uint32_t started_ms, + bool upload_in_progress) { + return !upload_in_progress && + (uint32_t)(now_ms - started_ms) >= kManualOtaSessionTimeoutMs; +} + } // namespace NetworkPolicy diff --git a/src/helpers/SNMPAgent.cpp b/src/helpers/SNMPAgent.cpp index ddfb2fe7..8394a588 100644 --- a/src/helpers/SNMPAgent.cpp +++ b/src/helpers/SNMPAgent.cpp @@ -17,7 +17,7 @@ MeshSNMPAgent::MeshSNMPAgent() _total_air_time_secs(0), _mqtt_connected_slots(0), _mqtt_queue_depth(0), _mqtt_skipped_publishes(0), _free_heap(0), _max_alloc(0), _internal_free(0), _psram_free(0), - _wifi_rssi(0) + _wifi_rssi(-127) { _firmware_version[0] = '\0'; _node_name[0] = '\0'; @@ -80,7 +80,7 @@ void MeshSNMPAgent::loop() { #endif const int signal = activeNetworkInterface().rssi(); - _wifi_rssi = signal == INT_MIN ? 0 : signal; + _wifi_rssi = signal == INT_MIN ? -127 : signal; _snmp.loop(); } diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 427106fa..78273d3a 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2776,18 +2776,20 @@ bool MQTTBridge::handleNetworkConnection(unsigned long now) { updateCachedConnectionStatus(); } - if (actions.reset_reconnect_backoff) { - // A usable route is a new connection epoch. Failures earned on the old - // route must not strand the replacement route on the 5-minute rung or at - // the circuit breaker. Preserve JWTs and slot configuration, but give each - // disconnected active slot one immediate, freshly guarded attempt. + if (actions.retry_disconnected_slots_now) { + // Link recovery gets one immediate attempt without forgiving broker + // failures earned on the same route. A medium switch is a new connection + // epoch, so only that transition clears the old route's ladder/breaker. for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].enabled && _slots[i].initial_connect_done && !_slots[i].connected) { - _slots[i].reconnect_backoff = 0; - _slots[i].max_backoff_failures = 0; - _slots[i].circuit_breaker_tripped = false; - _slots[i].last_reconnect_attempt = - now - MQTTConnectionPolicy::reconnectDelayMs(0, static_cast(i)); + if (actions.reset_reconnect_backoff) { + _slots[i].reconnect_backoff = 0; + _slots[i].max_backoff_failures = 0; + _slots[i].circuit_breaker_tripped = false; + } + _slots[i].last_reconnect_attempt = MQTTConnectionPolicy::immediateRetryLastAttempt( + static_cast(now), _slots[i].circuit_breaker_tripped, + _slots[i].reconnect_backoff, static_cast(i)); } } _last_slot_reconnect_ms = now - MQTTConnectionPolicy::kReconnectGuardMs; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 5038db18..04e27649 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -634,17 +634,21 @@ public: static unsigned long getWifiConnectedAtMillis(); /** - * Current selected-network outage snapshot for AlertReporter: down, - * started_ms, and the - * initiating disconnect reason. Distinct from getLastWifiDisconnectTime() / + * Current alert-network outage snapshot for AlertReporter: down, started_ms, + * and the initiating disconnect reason. Ethernet-preferred builds continue + * tracking Ethernet after a Wi-Fi fallback. Distinct from + * getLastWifiDisconnectTime() / * getLastWifiDisconnectReason(), which follow the most recent ESP-IDF * DISCONNECTED event and are overwritten by STA-backoff WiFi.disconnect() * (reason 8 / ASSOC_LEAVE). */ - AlertFaultPolicy::OutageSnapshot getWifiOutageSnapshot() const { - return _network ? _network->outageSnapshot() + AlertFaultPolicy::OutageSnapshot getNetworkOutageSnapshot() const { + return _network ? _network->alertOutageSnapshot() : AlertFaultPolicy::OutageSnapshot{false, 0, 0}; } + NetworkMedium getNetworkAlertMedium() const { + return _network ? _network->alertMedium() : NetworkMedium::None; + } /** * Per-slot outage accessors used by AlertReporter to detect prolonged diff --git a/src/helpers/esp32/HttpPort80Lease.h b/src/helpers/esp32/HttpPort80Lease.h new file mode 100644 index 00000000..464cf39a --- /dev/null +++ b/src/helpers/esp32/HttpPort80Lease.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +// Ownership guard for the two AsyncWebServer users. It prevents `start ota` +// from claiming port 80 while WebConfig is listening (and vice versa), even +// when the requests arrive from different tasks. +namespace HttpPort80Lease { + +enum class Owner : uint8_t { None = 0, WebConfig, Ota }; + +inline std::atomic& current() { + static std::atomic owner{Owner::None}; + return owner; +} + +inline bool acquire(Owner requested) { + if (requested == Owner::None) return false; + Owner expected = Owner::None; + return current().compare_exchange_strong( + expected, requested, std::memory_order_acq_rel, + std::memory_order_relaxed); +} + +inline void release(Owner expected) { + current().compare_exchange_strong( + expected, Owner::None, std::memory_order_acq_rel, + std::memory_order_relaxed); +} + +inline const char* ownerName() { + switch (current().load(std::memory_order_acquire)) { + case Owner::WebConfig: return "webconfig"; + case Owner::Ota: return "ota"; + default: return "none"; + } +} + +} // namespace HttpPort80Lease diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 1aa638d0..5c75a1f1 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -7,9 +7,10 @@ #include #include #include -#include +#include #include #include +#include #include #include @@ -18,30 +19,58 @@ #include #include +#include "HttpPort80Lease.h" + #include "WebConfigHtml.h" // Placeholder sent instead of stored secrets; POSTs carrying it are dropped // so an untouched password field never overwrites the stored value. static const char SECRET_SENTINEL[] = "********"; -// esp_random() has a true entropy source only while RF is active. Ethernet -// LAN mode deliberately keeps Wi-Fi and Bluetooth down, so temporarily enable -// the bootloader RNG source while generating authentication secrets there. +// The ESP hardware RNG has a continuous entropy source only while an RF block +// is running. Ethernet LAN mode deliberately keeps Wi-Fi and Bluetooth down, +// so briefly start an unassociated station while creating login secrets. The +// bootloader RNG helper is intentionally not used here: ESP-IDF limits it to +// early boot, before normal RF/ADC/I2S operation begins. class WebConfigEntropyGuard { - bool _enabled; + wifi_mode_t _previous_mode; + bool _started_wifi; + bool _ready; public: WebConfigEntropyGuard() - : _enabled(activeNetworkInterface().medium() == - NetworkMedium::Ethernet) { - if (_enabled) bootloader_random_enable(); + : _previous_mode(WiFi.getMode()), _started_wifi(false), _ready(true) { + if (_previous_mode == WIFI_MODE_NULL) { + _ready = WiFi.mode(WIFI_MODE_STA); + _started_wifi = _ready; + } } ~WebConfigEntropyGuard() { - if (_enabled) bootloader_random_disable(); + if (_started_wifi) WiFi.mode(_previous_mode); } + + bool ready() const { return _ready; } }; +static bool fillRandomBytes(uint8_t* output, size_t byte_count) { + if (!output || byte_count == 0) return false; + WebConfigEntropyGuard entropy; + if (!entropy.ready()) return false; + + esp_fill_random(output, byte_count); + return true; +} + +static void bytesToHex(char* output, const uint8_t* bytes, size_t byte_count) { + static const char HEX_DIGITS[] = "0123456789ABCDEF"; + for (size_t i = 0; i < byte_count; ++i) { + output[i * 2] = HEX_DIGITS[bytes[i] >> 4]; + output[i * 2 + 1] = HEX_DIGITS[bytes[i] & 0x0f]; + } + output[byte_count * 2] = '\0'; +} + // Key classification (allowlist, secret detection, slot-prefix parsing) lives in // helpers/WebConfigKeys.h so it can be unit-tested on the host. Thin aliases keep // the call sites below readable. @@ -223,6 +252,11 @@ bool WebConfigServer::startSetupMode(char reply[]) { strcpy(reply, "Err: webconfig busy"); return false; } + if (!HttpPort80Lease::acquire(HttpPort80Lease::Owner::WebConfig)) { + snprintf(reply, 160, "Err: port 80 is in use by %s", + HttpPort80Lease::ownerName()); + return false; + } // AP_STA (not pure AP) so the WiFi scan for the SSID picker works while // the AP is up. STA stays unconnected - the bridge won't touch WiFi // while wifi_ssid is empty, and `start webconfig ap` requires it stopped. @@ -244,6 +278,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { #endif if (!ap_ok) { WiFi.mode(WIFI_OFF); + HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); strcpy(reply, "Err: failed to start AP"); return false; } @@ -271,26 +306,38 @@ bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[ strcpy(reply, "Err: webconfig busy"); return false; } + if (!HttpPort80Lease::acquire(HttpPort80Lease::Owner::WebConfig)) { + snprintf(reply, 160, "Err: port 80 is in use by %s", + HttpPort80Lease::ownerName()); + return false; + } activeNetworkInterface().lockSwitching(); _network_locked = true; if (!activeNetworkInterface().isConnected() || ip == IPAddress()) { activeNetworkInterface().unlockSwitching(); _network_locked = false; + HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); strcpy(reply, "Err: selected network not connected"); return false; } _initial_setup = initial_setup; + uint8_t session_entropy[sizeof(_session_secret) + 6]; + const size_t entropy_size = sizeof(_session_secret) + (_initial_setup ? 6 : 0); + if (!fillRandomBytes(session_entropy, entropy_size)) { + activeNetworkInterface().unlockSwitching(); + _network_locked = false; + HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); + strcpy(reply, "Err: secure random source unavailable"); + return false; + } + memcpy(_session_secret, session_entropy, sizeof(_session_secret)); + _session_generation = 0; if (_initial_setup) { - { - WebConfigEntropyGuard entropy; - for (int i = 0; i < 3; ++i) { - sprintf(&_setup_code[i * 4], "%04X", (unsigned)(esp_random() & 0xffff)); - } - } - _setup_code[12] = 0; + bytesToHex(_setup_code, session_entropy + sizeof(_session_secret), 6); } else { _setup_code[0] = 0; } + memset(session_entropy, 0, sizeof(session_entropy)); _mode = MODE_LAN; createServer(); _last_activity = millis(); @@ -391,6 +438,8 @@ void WebConfigServer::finalizeTeardown() { _batch_state = BATCH_IDLE; _batch_next = 0; _batch_reboot_armed = false; + memset(_session_secret, 0, sizeof(_session_secret)); + _session_generation = 0; _session_token[0] = 0; _stats_json[0] = 0; _setup_code[0] = 0; @@ -399,6 +448,7 @@ void WebConfigServer::finalizeTeardown() { activeNetworkInterface().unlockSwitching(); _network_locked = false; } + HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); if (_cb) _cb->onWebConfigStopped(); } @@ -653,13 +703,14 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { _last_activity = millis(); if (_mode == MODE_SETUP) return true; // physical proximity implied, nothing configured if (_mode != MODE_LAN) return false; - if (_session_token[0] == 0) return false; if (!req->hasHeader("Cookie")) return false; const String& cookies = req->getHeader("Cookie")->value(); int idx = cookies.indexOf("wcs="); if (idx < 0 || (int)cookies.length() < idx + 4 + 32) return false; String token = cookies.substring(idx + 4, idx + 4 + 32); uint32_t now = millis(); + WCLock lock(_mux); + if (_session_token[0] == 0) return false; if ((uint32_t)(now - _session_last_seen) > WEBCONFIG_SESSION_TTL_MS) return false; if (!fixedTimeEquals(token.c_str(), _session_token, 32)) return false; _session_last_seen = now; // sliding expiry @@ -752,25 +803,41 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { } _login_fails = 0; _login_lock_until = 0; + uint32_t generation; { - WebConfigEntropyGuard entropy; - for (int i = 0; i < 4; i++) { - sprintf(&_session_token[i * 8], "%08lx", - (unsigned long)esp_random()); - } + WCLock lock(_mux); + generation = ++_session_generation; + if (generation == 0) generation = ++_session_generation; + } + uint8_t token_bytes[16]; + SHA256 token_hmac; + token_hmac.resetHMAC(_session_secret, sizeof(_session_secret)); + token_hmac.update(reinterpret_cast(&generation), + sizeof(generation)); + token_hmac.finalizeHMAC(_session_secret, sizeof(_session_secret), + token_bytes, sizeof(token_bytes)); + char session_token[33]; + bytesToHex(session_token, token_bytes, sizeof(token_bytes)); + memset(token_bytes, 0, sizeof(token_bytes)); + { + WCLock lock(_mux); + memcpy(_session_token, session_token, sizeof(_session_token)); + _session_last_seen = now; } - _session_last_seen = now; AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); char cookie[80]; - sprintf(cookie, "wcs=%s; HttpOnly; SameSite=Lax; Path=/", _session_token); + sprintf(cookie, "wcs=%s; HttpOnly; SameSite=Lax; Path=/", session_token); res->addHeader("Set-Cookie", cookie); req->send(res); } void WebConfigServer::handleLogout(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } - _session_token[0] = 0; + { + WCLock lock(_mux); + _session_token[0] = 0; + } AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); res->addHeader("Set-Cookie", "wcs=; Max-Age=0; Path=/"); req->send(res); diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index 79cb167b..b19ca810 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -171,6 +171,8 @@ private: BatchEntry _batch[MAX_BATCH]; // LAN-mode session (single slot; new login evicts the old session) + uint8_t _session_secret[32] = {0}; + uint32_t _session_generation = 0; char _session_token[33] = {0}; uint32_t _session_last_seen = 0; uint32_t _setup_reminder_at = 0; diff --git a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp index e5fef6d0..e9d1ff22 100644 --- a/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp +++ b/test/test_mqtt_connection_policy/test_mqtt_connection_policy.cpp @@ -96,6 +96,19 @@ TEST(MQTTConnectionPolicy, CircuitBreakerProbeHasExactThirtyMinuteBoundary) { EXPECT_TRUE(Policy::circuitBreakerProbeDue(900000U, last)); } +TEST(MQTTConnectionPolicy, LinkRecoveryMakesCurrentBackoffDueImmediately) { + const uint32_t now = 1234U; + const uint32_t last = Policy::immediateRetryLastAttempt(now, false, 4, 2); + EXPECT_TRUE(Policy::reconnectDue(now, last, 4, 2)); + EXPECT_FALSE(Policy::circuitBreakerProbeDue(now, last)); +} + +TEST(MQTTConnectionPolicy, LinkRecoveryMakesBreakerProbeDueWithoutClearingIt) { + const uint32_t now = 1234U; + const uint32_t last = Policy::immediateRetryLastAttempt(now, true, 5, 2); + EXPECT_TRUE(Policy::circuitBreakerProbeDue(now, last)); +} + TEST(MQTTConnectionPolicy, JwtLifetimeUsesCappedPerSlotStagger) { EXPECT_EQ(86400U, Policy::jwtLifetimeSecs(86400U, 0)); EXPECT_EQ(86100U, Policy::jwtLifetimeSecs(86400U, 1)); diff --git a/test/test_network_hostname/test_network_hostname.cpp b/test/test_network_hostname/test_network_hostname.cpp index 8835c7d9..37814861 100644 --- a/test/test_network_hostname/test_network_hostname.cpp +++ b/test/test_network_hostname/test_network_hostname.cpp @@ -47,7 +47,51 @@ TEST(NetworkHostname, UsesDocumentedFallbackForEmptySanitizedName) { char hostname[NetworkHostname::kBufferSize]; ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), "___ !!!", kStableId, sizeof(kStableId))); - EXPECT_STREQ("meshcore-node", hostname); + EXPECT_STREQ("meshcore-node-abcd01", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, RemovesUtf8AndRetainsStableIdentity) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + "M\xC3\xBCnchen", kStableId, + sizeof(kStableId))); + EXPECT_STREQ("meshcore-m-nchen-abcd01", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, MarksCombiningUnicodeAsLossy) { + char hostname[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), + "Cafe\xCC\x81", kStableId, + sizeof(kStableId))); + EXPECT_STREQ("meshcore-cafe-abcd01", hostname); + expectDhcpSafe(hostname); +} + +TEST(NetworkHostname, UnicodeOnlyNamesUseDistinctIdentityFallbacks) { + const uint8_t other_id[] = {0x12, 0x34, 0x56}; + const char unicode_only[] = "\xE7\xBD\x91\xE6\xA0\xBC\xF0\x9F\x8C\x90"; + char first[NetworkHostname::kBufferSize]; + char second[NetworkHostname::kBufferSize]; + ASSERT_TRUE(NetworkHostname::build(first, sizeof(first), unicode_only, + kStableId, sizeof(kStableId))); + ASSERT_TRUE(NetworkHostname::build(second, sizeof(second), unicode_only, + other_id, sizeof(other_id))); + EXPECT_STREQ("meshcore-node-abcd01", first); + EXPECT_STREQ("meshcore-node-123456", second); + EXPECT_STRNE(first, second); + expectDhcpSafe(first); + expectDhcpSafe(second); +} + +TEST(NetworkHostname, MalformedUtf8CannotReachDhcpHostname) { + char hostname[NetworkHostname::kBufferSize]; + const char malformed[] = {'b', 'a', 'd', static_cast(0xff), + 'n', 'a', 'm', 'e', '\0'}; + ASSERT_TRUE(NetworkHostname::build(hostname, sizeof(hostname), malformed, + kStableId, sizeof(kStableId))); + EXPECT_STREQ("meshcore-bad-name-abcd01", hostname); expectDhcpSafe(hostname); } diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp index 6d5f1f36..7a057918 100644 --- a/test/test_network_policy/test_network_policy.cpp +++ b/test/test_network_policy/test_network_policy.cpp @@ -42,13 +42,54 @@ TEST(NetworkPolicy, BootPrefersEthernetAndOtherwiseUsesConfiguredWifi) { EXPECT_EQ(NetworkMedium::None, NetworkPolicy::bootSelection(false, false)); } -TEST(NetworkPolicy, EthernetBootProbeHonorsFullDeadlineUntilConnected) { - EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 0, 8000)); - EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 1500, 8000)); - EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending(true, false, 7999, 8000)); - EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(true, false, 8000, 8000)); - EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(true, true, 100, 8000)); - EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending(false, false, 100, 8000)); +TEST(NetworkPolicy, EthernetBootProbeUsesFullDeadlineForLinkOrUnknownState) { + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending( + true, false, false, false, 1500, 8000)); + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending( + true, false, true, true, 7999, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending( + true, false, true, true, 8000, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending( + true, true, true, true, 100, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending( + false, false, false, false, 100, 8000)); +} + +TEST(NetworkPolicy, EthernetBootProbeStopsEarlyOnKnownCableDown) { + EXPECT_TRUE(NetworkPolicy::ethernetBootProbePending( + true, false, true, false, + NetworkPolicy::kEthernetNoLinkBootGraceMs - 1, 8000)); + EXPECT_FALSE(NetworkPolicy::ethernetBootProbePending( + true, false, true, false, + NetworkPolicy::kEthernetNoLinkBootGraceMs, 8000)); +} + +TEST(NetworkPolicy, EthernetInitRetryUsesBoundedExponentialBackoff) { + EXPECT_EQ(0u, NetworkPolicy::ethernetInitRetryDelayMs(0)); + EXPECT_EQ(5000u, NetworkPolicy::ethernetInitRetryDelayMs(1)); + EXPECT_EQ(10000u, NetworkPolicy::ethernetInitRetryDelayMs(2)); + EXPECT_EQ(300000u, NetworkPolicy::ethernetInitRetryDelayMs(8)); + EXPECT_EQ(300000u, NetworkPolicy::ethernetInitRetryDelayMs(255)); + + EXPECT_FALSE(NetworkPolicy::ethernetInitRetryDue(1, 4999, 0)); + EXPECT_TRUE(NetworkPolicy::ethernetInitRetryDue(1, 5000, 0)); + EXPECT_TRUE(NetworkPolicy::ethernetInitRetryDue(1, 10, 10u - 5000u)); +} + +TEST(NetworkPolicy, EthernetNoIpRecoveryRequiresSustainedCarrier) { + const uint32_t timeout = NetworkPolicy::kEthernetNoIpRecoveryMs; + EXPECT_FALSE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, true, false, timeout, 0)); + EXPECT_FALSE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, true, false, timeout, 1)); + EXPECT_TRUE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, true, false, timeout + 1, 1)); + EXPECT_FALSE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, false, false, timeout + 1, 1)); + EXPECT_FALSE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, true, true, timeout + 1, 1)); + EXPECT_TRUE(NetworkPolicy::ethernetNoIpRecoveryDue( + true, true, false, 100, 100u - timeout)); } TEST(NetworkPolicy, EthernetFailureWaitsForGraceAndConnectedWifi) { @@ -146,6 +187,15 @@ TEST(NetworkPolicy, StartOtaForceApOverridesAReachableSelectedNetwork) { EXPECT_FALSE(NetworkPolicy::startOtaUsesSelectedNetwork(true, false)); } +TEST(NetworkPolicy, ManualOtaTimeoutIsUploadSafeAndWrapSafe) { + const uint32_t timeout = NetworkPolicy::kManualOtaSessionTimeoutMs; + EXPECT_FALSE(NetworkPolicy::manualOtaTimeoutDue(timeout - 1, 0, false)); + EXPECT_TRUE(NetworkPolicy::manualOtaTimeoutDue(timeout, 0, false)); + EXPECT_FALSE(NetworkPolicy::manualOtaTimeoutDue(timeout, 0, true)); + EXPECT_TRUE(NetworkPolicy::manualOtaTimeoutDue( + 100, 100 - timeout, false)); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index fa9a0140..2888f585 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -47,7 +47,7 @@ build_flags = -D ETH_INT_PIN=45 -D ETHERNET_DEBUG_LOGGING=1 lib_deps = - https://github.com/agessaman/ESP32-CH390.git#55c425705d9b21df11f95782ae6408707b2a91a9 + https://github.com/agessaman/ESP32-CH390.git#a9367ff8ba400880270615b54da9c239cfd68eca ; Existing companion/CLI transport overlay. MQTT Ethernet observers consume the ; CH390 fragment above without ETHERNET_ENABLED, whose meaning in the simple diff --git a/webui/index.html b/webui/index.html index 6a851c1f..9452b241 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1156,7 +1156,9 @@ function pollStats(){ tile("Direct RX/TX",s.recv_direct+" / "+s.sent_direct)+ tile("TX queue",s.tx_queue)+ tile("MQTT queue",s.mqtt_queue)+ - tile("WiFi RSSI",s.wifi_rssi,"dBm")+ + (s.wifi_rssi==null + ? tile("Network",s.network_medium||"—","RSSI unavailable") + : tile("WiFi RSSI",s.wifi_rssi,"dBm"))+ tile("IP",s.ip||"—"); push(st.hist.heap,kb);push(st.hist.noise,s.noise); spark($("#spark-heap"),st.hist.heap);spark($("#spark-noise"),st.hist.noise); From d8d72b783346b2d775a87dbda87588e48eacc51a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 5 Sep 2026 12:14:29 -0700 Subject: [PATCH 58/93] feat(station-g3): add second RF slot (r2) build targets The motherboard has two RF daughterboard slots on separate GPIOs, mirrored by its "LNA P" / "LNA S" jumpers, but the variant only ever pinned slot 1. The shared configuration moves into [Station_G3_ESP32_common], with the SPI and LNA lines split into per-slot sections. [Station_G3_ESP32] (slot 1) and [Station_G3_ESP32_r2] (slot 2) each compose common + their own pins, so neither set of -D flags has to shadow the other on the compile line - appending an override would work only by last-wins, and esp32_base's -w hides the resulting redefinition warning. Every pre-existing env keeps its name and resolves to byte-identical options; verified by diffing pio project config --json-output. PA PL1 is one board-level jumper shared by both slots, so P_PA1_EN stays in the common section along with the TX power calibration. Adds r2 twins of the repeater, room server and both observer envs, each an exact mirror of its slot-1 sibling apart from the base it extends. Slot 2 pins come from on-hardware testing in agessaman/MeshCore#49 and are not documented publicly; the vendor wiki serves no content to fetchers and Meshtastic implements slot 1 only. Known limitation, unchanged from that PR: GPIO 42 and 43 fall outside the ESP32-S3 RTC GPIO range (0-21), so the rtc_gpio_hold_en() calls in ESP32Board::enterDeepSleep() and StationG3Board::powerOff() return ESP_ERR_INVALID_ARG on slot 2 and the NSS and LNA pins are not latched through deep sleep. Fixing it means gpio_hold_en() plus gpio_deep_sleep_hold_en() in shared code. --- MQTT_IMPLEMENTATION.md | 4 + variants/station_g3_esp32/platformio.ini | 177 +++++++++++++++++++++-- 2 files changed, 171 insertions(+), 10 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index ac74485d..96b7340e 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -211,6 +211,8 @@ pio run -e Station_G2_repeater_observer_mqtt # Station G3 (ESP32) pio run -e Station_G3_ESP32_repeater_observer_mqtt pio run -e Station_G3_ESP32_room_server_observer_mqtt +pio run -e Station_G3_ESP32_r2_repeater_observer_mqtt # second RF slot +pio run -e Station_G3_ESP32_r2_room_server_observer_mqtt # second RF slot # LilyGo T-LoRa V2.1-1.6 (TTGO LoRa32 V1.0) pio run -e LilyGo_TLora_V2_1_1_6_repeater_observer_mqtt @@ -245,6 +247,8 @@ Some MQTT observer builds use a non-default partition table to accommodate the l | `Station_G2_room_server_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | 16 MB flash board | | `Station_G3_ESP32_repeater_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | 16 MB flash board | | `Station_G3_ESP32_room_server_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | 16 MB flash board | +| `Station_G3_ESP32_r2_repeater_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | Second RF slot; same board and layout as the slot-1 env | +| `Station_G3_ESP32_r2_room_server_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | same | | `LilyGo_TBeam_1W_repeater_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | Set in `boards/t_beam_1w.json`; required vs implicit `default.csv` | | `LilyGo_TBeam_1W_room_server_observer_mqtt` | `default_16MB.csv` | 16 MB | 6.25 MB | same | diff --git a/variants/station_g3_esp32/platformio.ini b/variants/station_g3_esp32/platformio.ini index bca80963..d6f179c1 100644 --- a/variants/station_g3_esp32/platformio.ini +++ b/variants/station_g3_esp32/platformio.ini @@ -1,4 +1,7 @@ -[Station_G3_ESP32] +; The motherboard has two RF daughterboard slots on separate GPIOs, mirrored by its +; "LNA P" / "LNA S" jumpers. Slot 1 is [Station_G3_ESP32], slot 2 is [Station_G3_ESP32_r2]; +; the slot pins sit in their own sections so neither set of -D flags shadows the other. +[Station_G3_ESP32_common] extends = esp32_base board = station-g3-esp32 build_flags = @@ -10,17 +13,8 @@ build_flags = -D USE_SX1262 -D RADIO_CLASS=CustomSX1262 -D WRAPPER_CLASS=CustomSX1262Wrapper - -D P_LORA_DIO_1=48 - -D P_LORA_NSS=11 - -D P_LORA_RESET=21 - -D P_LORA_BUSY=47 - -D P_LORA_SCLK=12 - -D P_LORA_MISO=14 - -D P_LORA_MOSI=13 -D P_PA1_EN=9 ; PA PL1 Mode: LOW/open selects low level, HIGH/short selects high level. -D P_PA1_EN_ACTIVE=HIGH - -D P_PRIMARY_LNA_EN=10 ; Primary Slot LNA Mode: LOW/open is LNA on, HIGH/short is LNA off. - -D P_PRIMARY_LNA_EN_ACTIVE=LOW -D LORA_TX_POWER=7 ; SX1262 input power to the Station G3 PA; final output depends on PA PL1/PL2 level. -D MAX_LORA_TX_POWER=22 ; -D P_LORA_TX_LED=35 @@ -45,6 +39,48 @@ lib_deps = adafruit/Adafruit SH110X @ ~2.1.13 adafruit/Adafruit GFX Library @ ^1.12.1 +; PA PL1 is one board-level jumper shared by both slots, so P_PA1_EN stays in the base above. +[station_g3_esp32_r1_pins] +build_flags = + -D P_LORA_DIO_1=48 + -D P_LORA_NSS=11 + -D P_LORA_RESET=21 + -D P_LORA_BUSY=47 + -D P_LORA_SCLK=12 + -D P_LORA_MISO=14 + -D P_LORA_MOSI=13 + -D P_PRIMARY_LNA_EN=10 ; Primary Slot LNA Mode: LOW/open is LNA on, HIGH/short is LNA off. + -D P_PRIMARY_LNA_EN_ACTIVE=LOW + +; Slot 2 ("LNA S"). P_PRIMARY_LNA_EN keeps its name because LoRaFEMControl knows only one LNA pin. +[station_g3_esp32_r2_pins] +build_flags = + -D P_LORA_DIO_1=2 + -D P_LORA_NSS=43 + -D P_LORA_RESET=44 + -D P_LORA_BUSY=1 + -D P_LORA_SCLK=39 + -D P_LORA_MISO=41 + -D P_LORA_MOSI=40 + -D P_PRIMARY_LNA_EN=42 ; Secondary Slot LNA Mode, same polarity as the primary slot. + -D P_PRIMARY_LNA_EN_ACTIVE=LOW + +[Station_G3_ESP32] +extends = Station_G3_ESP32_common +build_flags = + ${Station_G3_ESP32_common.build_flags} + ${station_g3_esp32_r1_pins.build_flags} +build_src_filter = ${Station_G3_ESP32_common.build_src_filter} +lib_deps = ${Station_G3_ESP32_common.lib_deps} + +[Station_G3_ESP32_r2] +extends = Station_G3_ESP32_common +build_flags = + ${Station_G3_ESP32_common.build_flags} + ${station_g3_esp32_r2_pins.build_flags} +build_src_filter = ${Station_G3_ESP32_common.build_src_filter} +lib_deps = ${Station_G3_ESP32_common.lib_deps} + [env:Station_G3_ESP32_repeater] extends = Station_G3_ESP32 build_flags = @@ -246,3 +282,124 @@ lib_deps = JChristensen/Timezone paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent + +[env:Station_G3_ESP32_r2_repeater] +extends = Station_G3_ESP32_r2 +build_flags = + ${Station_G3_ESP32_r2.build_flags} + -D ADVERT_NAME='"Station G3 ESP32 Repeater"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 +build_src_filter = ${Station_G3_ESP32_r2.build_src_filter} + +<../examples/simple_repeater> +lib_deps = + ${Station_G3_ESP32_r2.lib_deps} + ${esp32_ota.lib_deps} + +[env:Station_G3_ESP32_r2_room_server] +extends = Station_G3_ESP32_r2 +build_src_filter = ${Station_G3_ESP32_r2.build_src_filter} + +<../examples/simple_room_server> +build_flags = + ${Station_G3_ESP32_r2.build_flags} + -D ADVERT_NAME='"Station G3 ESP32 Room"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 +lib_deps = + ${Station_G3_ESP32_r2.lib_deps} + ${esp32_ota.lib_deps} + +[env:Station_G3_ESP32_r2_repeater_observer_mqtt] +extends = Station_G3_ESP32_r2 +board_build.partitions = default_16MB.csv ; standard 16MB partition table +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Station_G3_ESP32_r2.build_flags} + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +# -D MESH_PACKET_LOGGING=1 +# -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +# -D WIFI_SSID='"ssid"' +# -D WIFI_PWD='"password"' +# -D MQTT_SERVER='"your-mqtt-broker.com"' +# -D MQTT_PORT=1883 +# -D MQTT_USERNAME='"your-username"' +# -D MQTT_PASSWORD='"your-password"' +build_src_filter = ${Station_G3_ESP32_r2.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Station_G3_ESP32_r2.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Station_G3_ESP32_r2_room_server_observer_mqtt] +extends = Station_G3_ESP32_r2 +board_build.partitions = default_16MB.csv ; standard 16MB partition table +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Station_G3_ESP32_r2.build_flags} + -D ADVERT_NAME='"Station G3 ESP32 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D WITH_MQTT_BRIDGE=1 + -D MAX_NEIGHBOURS=50 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${Station_G3_ESP32_r2.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Station_G3_ESP32_r2.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent From e28bf78b7f81a151d63fcf5241511f8cbb43abb4 Mon Sep 17 00:00:00 2001 From: Stephan Rodemeier Date: Sun, 6 Sep 2026 20:09:08 +0200 Subject: [PATCH 59/93] Add BSmesh.de preset --- MQTT_IMPLEMENTATION.md | 1 + src/helpers/MQTTPresets.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 96b7340e..baf39ed6 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -162,6 +162,7 @@ below documents the current build. | `gomesh` | `wss://mqtt.gomesh.dev:443` | JWT | — | | `idahomesh` | `wss://mqtt.idahomesh.org:443/mqtt` | JWT | — | | `ntxmesh` | `wss://ntxmesh.dhovin.me:8883` | JWT | — | +| `bsmesh` | `wss://mqtt.bsmesh.de:8885` | JWT | — | | `custom` | your own broker | User/pass, or JWT when `mqttN.audience` is set | `set mqttN.server` (see [custom broker setup](#custom-brokers)) | | `none` | (slot disabled) | — | — | diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 176413cb..8b9d4b1f 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -148,7 +148,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 35; +static const int MQTT_PRESET_COUNT = 36; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -196,6 +196,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "gomesh", "wss://mqtt.gomesh.dev:443", "mqtt.gomesh.dev", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "idahomesh", "wss://mqtt.idahomesh.org:443/mqtt", "mqtt.idahomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "ntxmesh", "wss://ntxmesh.dhovin.me:8883", "ntxmesh.dhovin.me", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "bsmesh", "wss://mqtt.bsmesh.de:8885", "mqtt.bsmesh.de", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, }; // Find a preset by name, returns nullptr if not found From d8ffae3230951db35c0f112adc8517ccaf691caf Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 14:42:39 -0700 Subject: [PATCH 60/93] fix(dispatcher): pick radio-watchdog activity by age, not largest timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchdog took max(last_recv, last_irq, last_tx) of three raw millis() timestamps. Around the 32-bit wrap a transmit stamped in the upper half of the range is numerically larger than every fresh post-wrap receive, so it kept winning for up to ~24 days: the watchdog measured silence from that stale transmit, concluded the radio had gone deaf, and reset a radio that was receiving normally — once per watchdog interval, until the old value stopped winning. Extract the decision into RadioWatchdog (pure, host-testable). It compares each timestamp only against its own previous value, so any change is fresh activity, and stamps it against a monotonic 64-bit clock extended from millis(). Evidence older than one 32-bit cycle now stays old instead of aliasing back to "recent", which the previous unsigned-subtraction age could not do either. Covered in test/test_radio_watchdog: pre-wrap TX with continuous post-wrap RX, genuine silence across the wrap, silence past a full millis() cycle, no activity since boot, disabled watchdog, TX-only traffic, and a stalled mesh loop. --- src/Dispatcher.cpp | 34 +-- src/Dispatcher.h | 8 +- src/helpers/RadioWatchdog.h | 116 ++++++++++ .../test_radio_watchdog.cpp | 204 ++++++++++++++++++ 4 files changed, 343 insertions(+), 19 deletions(-) create mode 100644 src/helpers/RadioWatchdog.h create mode 100644 test/test_radio_watchdog/test_radio_watchdog.cpp diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index 2a491a61..0c9c4750 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -29,6 +29,9 @@ void Dispatcher::begin() { _radio->begin(); prev_isrecv_mode = _radio->isInRecvMode(); +#ifdef WITH_MQTT_BRIDGE + radio_watchdog.reset((uint32_t)_ms->getMillis()); +#endif } float Dispatcher::getAirtimeBudgetFactor() const { @@ -94,23 +97,20 @@ void Dispatcher::loop() { // MQTTPrefs radio_watchdog_minutes setting. #ifdef WITH_MQTT_BRIDGE { - const uint32_t watchdog_ms = getRadioWatchdogMillis(); - if (watchdog_ms > 0) { - unsigned long last_recv = _radio->getLastRecvMillis(); - unsigned long last_irq = _radio->getLastRadioInterruptMillis(); - unsigned long last_active = (last_recv > last_irq ? last_recv : last_irq); - if (last_radio_active_ms > last_active) last_active = last_radio_active_ms; - if (is_recv && last_active > 0) { - unsigned long silent_ms = _ms->getMillis() - last_active; - unsigned long since_recovery = _ms->getMillis() - last_watchdog_recovery; - if (silent_ms > watchdog_ms && since_recovery > watchdog_ms) { - _err_flags |= ERR_EVENT_RADIO_WATCHDOG; - MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState()); - _radio->idle(); - _radio->startRecv(); - last_watchdog_recovery = _ms->getMillis(); - } - } + // Selecting the most recent event by age rather than by largest timestamp is + // what keeps this correct across the millis() wrap — see RadioWatchdog. + RadioWatchdogDecision wd = radio_watchdog.update( + (uint32_t)_ms->getMillis(), is_recv, getRadioWatchdogMillis(), + (uint32_t)_radio->getLastRecvMillis(), + (uint32_t)_radio->getLastRadioInterruptMillis(), + (uint32_t)last_radio_active_ms); + if (wd.recover) { + _err_flags |= ERR_EVENT_RADIO_WATCHDOG; + MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", + (unsigned long)wd.silent_ms, _radio->getRadioState()); + _radio->idle(); + _radio->startRecv(); + radio_watchdog.noteRecovery(); } } #endif // WITH_MQTT_BRIDGE (radio watchdog) diff --git a/src/Dispatcher.h b/src/Dispatcher.h index 7edb7c7f..25f1c307 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -5,6 +5,9 @@ #include #include #include +#ifdef WITH_MQTT_BRIDGE + #include "helpers/RadioWatchdog.h" +#endif namespace mesh { @@ -135,8 +138,10 @@ typedef uint32_t DispatcherAction; class Dispatcher { Packet* outbound; // current outbound packet unsigned long outbound_expiry, outbound_start, total_air_time, rx_air_time; - unsigned long last_watchdog_recovery; unsigned long last_radio_active_ms; // updated on any TX or RX event; used by watchdog +#ifdef WITH_MQTT_BRIDGE + RadioWatchdog radio_watchdog; +#endif unsigned long next_tx_time; unsigned long cad_busy_start; unsigned long radio_nonrx_start; @@ -171,7 +176,6 @@ protected: tx_budget_ms = 0; last_budget_update = 0; duty_cycle_window_ms = 3600000; - last_watchdog_recovery = 0; last_radio_active_ms = 0; } diff --git a/src/helpers/RadioWatchdog.h b/src/helpers/RadioWatchdog.h new file mode 100644 index 00000000..1642c60b --- /dev/null +++ b/src/helpers/RadioWatchdog.h @@ -0,0 +1,116 @@ +#pragma once + +#include + +// Decides when the observer radio watchdog should kick a radio that is parked in +// RX but hearing nothing (see Dispatcher::loop()). +// +// Pure logic: no Arduino, radio or mesh headers. The caller feeds the raw 32-bit +// activity timestamps it can see (last receive, last radio interrupt, last +// successful transmit) plus its millis() counter. +// +// Why this is not a max() of those three timestamps: millis() wraps every ~49.7 +// days, and a pre-wrap timestamp is numerically larger than every fresh +// post-wrap one. Picking the largest therefore latches onto an old transmit and +// reports the radio silent while it is receiving continuously, resetting a +// healthy radio once per watchdog interval until that timestamp stops winning. +// +// Instead the caller's timestamps are only ever compared with their own previous +// values: any change is fresh activity, stamped against a monotonic 64-bit clock +// extended from millis(). Nothing downstream has a rollover case, and evidence +// older than one 32-bit cycle stays old instead of aliasing back to "recent". +// +// How far a timestamp may run behind the previous one and still count as an +// out-of-order reading rather than a very long forward gap (same rule as +// RadioActivityWindow). +#define RADIO_WATCHDOG_BACKSTEP_TOLERANCE_MS 5000UL + +struct RadioWatchdogDecision { + bool recover; // trip now: idle + restart receive + bool measurable; // false until some activity has been observed + uint64_t silent_ms; // age of the most recent activity (0 when !measurable) +}; + +class RadioWatchdog { +public: + RadioWatchdog() { reset(0); } + + void reset(uint32_t now_ms) { + _now_ms = now_ms; + _last_input_ms = now_ms; + _seeded = false; + _has_activity = false; + _last_recv = _last_irq = _last_tx = 0; + _last_activity_ms = now_ms; + _recovered = false; + _last_recovery_ms = 0; + } + + // Call once per loop with the current activity timestamps. `in_recv_mode` and + // `watchdog_ms` (0 disables) gate the decision; the silence measurement is + // updated either way so a disabled or transmitting radio does not accumulate + // phantom silence. + RadioWatchdogDecision update(uint32_t now_ms, bool in_recv_mode, uint32_t watchdog_ms, + uint32_t last_recv, uint32_t last_irq, uint32_t last_tx) { + tick(now_ms); + + if (!_seeded) { + // First observation: the timestamps carry activity from before this + // tracker existed, so date it to now rather than trusting a raw value we + // cannot place on our own clock. Worst case that delays the first + // possible trip by one watchdog interval. + _seeded = true; + _has_activity = (last_recv != 0 || last_irq != 0 || last_tx != 0); + _last_activity_ms = _now_ms; + } else if (last_recv != _last_recv || last_irq != _last_irq || last_tx != _last_tx) { + _has_activity = true; + _last_activity_ms = _now_ms; + } + _last_recv = last_recv; + _last_irq = last_irq; + _last_tx = last_tx; + + RadioWatchdogDecision d; + d.recover = false; + d.measurable = _has_activity; + d.silent_ms = _has_activity ? (_now_ms - _last_activity_ms) : 0; + + if (!_has_activity || !in_recv_mode || watchdog_ms == 0) return d; + if (d.silent_ms <= watchdog_ms) return d; + // One recovery per watchdog interval: a radio that stays silent through a + // reset must not be reset every loop. + if (_recovered && (_now_ms - _last_recovery_ms) <= watchdog_ms) return d; + + d.recover = true; + return d; + } + + // Call after acting on a decision with recover set. + void noteRecovery() { + _recovered = true; + _last_recovery_ms = _now_ms; + } + + uint64_t nowMs() const { return _now_ms; } + +private: + // Extends the caller's 32-bit millis() to a monotonic 64-bit clock. + void tick(uint32_t now_ms) { + uint32_t delta = now_ms - _last_input_ms; + if (delta > 0x80000000u && + (uint32_t)(_last_input_ms - now_ms) <= RADIO_WATCHDOG_BACKSTEP_TOLERANCE_MS) { + return; // out-of-order reading: no time has passed + } + _now_ms += delta; + _last_input_ms = now_ms; + } + + uint64_t _now_ms; + uint32_t _last_input_ms; + bool _seeded; + bool _has_activity; + uint32_t _last_recv, _last_irq, _last_tx; + uint64_t _last_activity_ms; + bool _recovered; + uint64_t _last_recovery_ms; +}; diff --git a/test/test_radio_watchdog/test_radio_watchdog.cpp b/test/test_radio_watchdog/test_radio_watchdog.cpp new file mode 100644 index 00000000..fea7e720 --- /dev/null +++ b/test/test_radio_watchdog/test_radio_watchdog.cpp @@ -0,0 +1,204 @@ +#include "helpers/RadioWatchdog.h" + +#include + +namespace { + +const uint32_t WATCHDOG_MS = 300000UL; // RADIO_WATCHDOG_MS default: 5 minutes + +// Mirrors Dispatcher::loop(): the radio's three timestamps are sampled every +// pass, and a recovery is acknowledged as soon as it is decided. +struct Radio { + uint32_t last_recv = 0; + uint32_t last_irq = 0; + uint32_t last_tx = 0; +}; + +RadioWatchdogDecision run(RadioWatchdog& w, const Radio& r, uint32_t now, + bool in_recv_mode = true, uint32_t watchdog_ms = WATCHDOG_MS) { + RadioWatchdogDecision d = w.update(now, in_recv_mode, watchdog_ms, + r.last_recv, r.last_irq, r.last_tx); + if (d.recover) w.noteRecovery(); + return d; +} + +} // namespace + +TEST(RadioWatchdog, NoActivitySinceBootNeverTrips) { + RadioWatchdog w; + w.reset(0); + Radio r; // radio has never received, interrupted or transmitted + + for (uint32_t t = 0; t <= 4 * WATCHDOG_MS; t += 10000) { + RadioWatchdogDecision d = run(w, r, t); + EXPECT_FALSE(d.recover) << "tripped at t=" << t; + EXPECT_FALSE(d.measurable); + } +} + +TEST(RadioWatchdog, ContinuousReceiveNeverTrips) { + RadioWatchdog w; + w.reset(0); + Radio r; + + for (uint32_t t = 1000; t <= 3 * WATCHDOG_MS; t += 1000) { + r.last_recv = t; + r.last_irq = t; + RadioWatchdogDecision d = run(w, r, t); + EXPECT_FALSE(d.recover) << "tripped at t=" << t; + EXPECT_LE(d.silent_ms, 1000u); + } +} + +TEST(RadioWatchdog, SilentRadioTripsOncePerInterval) { + RadioWatchdog w; + w.reset(0); + Radio r; + r.last_recv = 1000; + run(w, r, 1000); + + int trips = 0; + uint32_t first_trip = 0; + for (uint32_t t = 2000; t <= 1000000; t += 1000) { + if (run(w, r, t).recover) { + if (trips == 0) first_trip = t; + trips++; + } + } + + EXPECT_EQ(302000u, first_trip); // strictly past the interval, from the last receive + EXPECT_EQ(3, trips); // ~1 M ms / 300 s, never per-loop +} + +// F13: a transmit timestamp from just before the millis() wrap is numerically +// larger than every fresh post-wrap receive. Selecting the largest timestamp +// reported ~24 days of silence and reset a radio that was receiving normally. +TEST(RadioWatchdog, PreWrapTransmitDoesNotSilenceFreshPostWrapReceives) { + const uint32_t WRAP = 0xFFFFFFFFu; + const uint32_t TX_AT = 0x80000000u; // upper half: beats any post-wrap value + + RadioWatchdog w; + w.reset(TX_AT - 60000); + Radio r; + r.last_recv = TX_AT - 60000; + r.last_irq = TX_AT - 60000; + run(w, r, TX_AT - 60000); + + r.last_tx = TX_AT; + run(w, r, TX_AT); + + // Receive steadily right up to the wrap... + for (uint32_t t = TX_AT + 1000; t < WRAP - 1000; t += 1000) { + r.last_recv = t; + r.last_irq = t; + ASSERT_FALSE(run(w, r, t).recover) << "tripped before wrap at t=" << t; + } + + // ...and straight through it. The stale transmit is still the numerically + // largest of the three timestamps for the next ~24 days. + for (uint32_t t = 1000; t < 2 * WATCHDOG_MS; t += 1000) { + r.last_recv = t; + r.last_irq = t; + RadioWatchdogDecision d = run(w, r, t); + ASSERT_FALSE(d.recover) << "tripped after wrap at t=" << t; + ASSERT_LE(d.silent_ms, 1000u); + } +} + +// The wrap must not hide a genuinely silent radio either. +TEST(RadioWatchdog, SilenceAcrossTheWrapIsStillDetected) { + const uint32_t LAST_RX = 0xFFFF0000u; // ~65 s before the wrap + + RadioWatchdog w; + w.reset(LAST_RX - 1000); + Radio r; + r.last_recv = LAST_RX; + r.last_irq = LAST_RX; + run(w, r, LAST_RX); + + bool tripped = false; + uint64_t silent_at_trip = 0; + for (uint32_t step = 1000; step <= WATCHDOG_MS + 5000; step += 1000) { + uint32_t t = LAST_RX + step; // wraps on its own + RadioWatchdogDecision d = run(w, r, t); + if (d.recover) { + tripped = true; + silent_at_trip = d.silent_ms; + break; + } + } + + EXPECT_TRUE(tripped); + EXPECT_GT(silent_at_trip, (uint64_t)WATCHDOG_MS); +} + +// Uptime past 2^32 ms must not alias a long silence back to "recent". +TEST(RadioWatchdog, SilenceBeyondOneMillisCycleKeepsGrowing) { + RadioWatchdog w; + w.reset(0); + Radio r; + r.last_recv = 1000; + run(w, r, 1000); + + // Walk a full cycle in 1-minute steps; the raw timestamp aliases back to its + // starting value, the tracked age does not. + uint64_t last_silent = 0; + for (uint64_t step = 60000; step <= 0x100000000ull + 600000ull; step += 60000) { + uint32_t t = (uint32_t)((1000ull + step) & 0xFFFFFFFFull); + RadioWatchdogDecision d = w.update(t, true, WATCHDOG_MS, r.last_recv, r.last_irq, r.last_tx); + if (d.recover) w.noteRecovery(); + ASSERT_GT(d.silent_ms, last_silent); + last_silent = d.silent_ms; + } + + EXPECT_GT(last_silent, 0x100000000ull); +} + +TEST(RadioWatchdog, DisabledWatchdogAndNonRecvModeNeverTrip) { + RadioWatchdog w; + w.reset(0); + Radio r; + r.last_recv = 1000; + run(w, r, 1000); + + for (uint32_t t = 2000; t <= 1000000; t += 10000) { + EXPECT_FALSE(run(w, r, t, /*in_recv_mode=*/true, /*watchdog_ms=*/0).recover); + EXPECT_FALSE(run(w, r, t, /*in_recv_mode=*/false).recover); + } +} + +// Transmit-only traffic still counts as a live radio. +TEST(RadioWatchdog, TransmitActivityRefreshesSilence) { + RadioWatchdog w; + w.reset(0); + Radio r; + r.last_recv = 1000; + run(w, r, 1000); + + for (uint32_t t = 100000; t <= 900000; t += 100000) { + r.last_tx = t; + EXPECT_FALSE(run(w, r, t).recover) << "tripped at t=" << t; + } +} + +// A blocked mesh loop (e.g. a 30 s NTP probe) collapses several events into one +// observation. That may only shorten measured silence, never lengthen it. +TEST(RadioWatchdog, LoopStallDoesNotManufactureSilence) { + RadioWatchdog w; + w.reset(0); + Radio r; + r.last_recv = 1000; + run(w, r, 1000); + + r.last_recv = 20000; // arrived during the stall + r.last_irq = 20000; + RadioWatchdogDecision d = run(w, r, 31000); + + EXPECT_FALSE(d.recover); + EXPECT_EQ(0u, d.silent_ms); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From dfdcc25b50de07b40312431c6582ddbfb1dabe2e Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 14:44:14 -0700 Subject: [PATCH 61/93] fix(wifi): one power-save mapping for CLI, startup and reconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `set wifi.powersave min` stored 0 and applied WIFI_PS_MIN_MODEM, but the bridge's post-reconnect mapping read the same 0 as WIFI_PS_NONE. A node explicitly configured for minimum modem sleep therefore ran with power save off after its first reconnect, while `get wifi.powersave` still reported min. The stored default is already 1 (`none`), so nothing here changes the product default — it stops an operator's explicit choice from being reinterpreted. Move the value/name/mode table into WifiPowerSavePolicy (pure, host-tested) and use it from the CLI setter and getter, the web config snapshot and the bridge. The bridge now also applies the mode when it finds the STA already associated at start — that path has no connect transition to carry the setting, so a bridge restart used to leave whatever mode was set before. --- src/helpers/CommonCLI_Observer.cpp | 30 ++------ src/helpers/WifiPowerSavePolicy.h | 73 +++++++++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 31 +++++--- src/helpers/bridges/MQTTBridge.h | 1 + src/helpers/esp32/WebConfigServer.cpp | 4 +- .../test_wifi_power_save_policy.cpp | 58 +++++++++++++++ 6 files changed, 163 insertions(+), 34 deletions(-) create mode 100644 src/helpers/WifiPowerSavePolicy.h create mode 100644 test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 9b2b2d30..dbb6e021 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -15,6 +15,7 @@ #include "TxtDataHelpers.h" #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) +#include "WifiPowerSavePolicy.h" // one powersave value->mode/name mapping #include #include #ifdef ESP_PLATFORM @@ -438,41 +439,28 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf if (!persistObserverPrefs(reply)) return true; strcpy(reply, "OK"); } else if (memcmp(config, "wifi.powersave ", 15) == 0) { - const char* value = &config[15]; uint8_t ps_value; - bool valid = false; - if (memcmp(value, "min", 3) == 0 && (value[3] == 0 || value[3] == ' ')) { - ps_value = 0; - valid = true; - } else if (memcmp(value, "none", 4) == 0 && (value[4] == 0 || value[4] == ' ')) { - ps_value = 1; - valid = true; - } else if (memcmp(value, "max", 3) == 0 && (value[3] == 0 || value[3] == ' ')) { - ps_value = 2; - valid = true; - } - if (!valid) { + if (!WifiPowerSavePolicy::parseName(&config[15], &ps_value)) { strcpy(reply, "Error: must be none, min, or max"); } else { _mqtt_prefs.wifi_power_save = ps_value; if (!persistObserverPrefs(reply)) return true; + const char* ps_name = WifiPowerSavePolicy::nameFor(ps_value); #ifdef ESP_PLATFORM if (WiFi.status() == WL_CONNECTED) { - wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : - (ps_value == 2) ? WIFI_PS_MAX_MODEM : WIFI_PS_MIN_MODEM; - esp_err_t ps_result = esp_wifi_set_ps(ps_mode); + // Same mapping the bridge applies on every association, so this cannot + // drift back apart (see WifiPowerSavePolicy). + esp_err_t ps_result = + esp_wifi_set_ps((wifi_ps_type_t)WifiPowerSavePolicy::modeFor(ps_value)); if (ps_result == ESP_OK) { - const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; sprintf(reply, "OK - power save set to %s", ps_name); } else { sprintf(reply, "OK - saved, but failed to apply: %d", ps_result); } } else { - const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; sprintf(reply, "OK - saved as %s (will apply on next WiFi connection)", ps_name); } #else - const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; sprintf(reply, "OK - saved as %s", ps_name); #endif } @@ -1090,9 +1078,7 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf #endif } } else if (memcmp(config, "wifi.powersave", 14) == 0) { - uint8_t ps = _mqtt_prefs.wifi_power_save; - const char* ps_name = (ps == 1) ? "none" : (ps == 2) ? "max" : "min"; - sprintf(reply, "> %s", ps_name); + sprintf(reply, "> %s", WifiPowerSavePolicy::nameFor(_mqtt_prefs.wifi_power_save)); } else if (memcmp(config, "timezone.offset", 15) == 0) { // Must precede the "timezone" (8-byte) check below — that prefix-matches // "timezone.offset" too, so the more-specific key has to come first or diff --git a/src/helpers/WifiPowerSavePolicy.h b/src/helpers/WifiPowerSavePolicy.h new file mode 100644 index 00000000..bdebd57e --- /dev/null +++ b/src/helpers/WifiPowerSavePolicy.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +// The single mapping between the stored `wifi.powersave` preference, its CLI +// name, and the IDF power-save mode. +// +// Pure lookup so host tests can hold startup, reconnect, CLI and web config to +// one table: they used to map the same stored value differently, so a node set +// to `min` silently ran with power save off after its first reconnect while +// `get wifi.powersave` still said min. +// +// Stored values are fleet state — never renumber them. The product default is +// `none`, which is a *default* (MQTTDefaults.h), not a reinterpretation of an +// operator's explicit `min`. +namespace WifiPowerSavePolicy { + +enum StoredValue : uint8_t { + kMin = 0, // WIFI_PS_MIN_MODEM + kNone = 1, // WIFI_PS_NONE (default) + kMax = 2, // WIFI_PS_MAX_MODEM +}; + +// Mirrors wifi_ps_type_t. MQTTBridge.cpp static_asserts these against the SDK. +enum Mode : uint8_t { + kModeNone = 0, + kModeMinModem = 1, + kModeMaxModem = 2, +}; + +// Anything outside the known range reads as the default rather than as the +// lowest-numbered mode, so a corrupt byte cannot silently enable modem sleep. +static inline Mode modeFor(uint8_t stored) { + switch (stored) { + case kMin: return kModeMinModem; + case kMax: return kModeMaxModem; + case kNone: return kModeNone; + default: return kModeNone; + } +} + +static inline const char* nameFor(uint8_t stored) { + switch (stored) { + case kMin: return "min"; + case kMax: return "max"; + case kNone: return "none"; + default: return "none"; + } +} + +// Parses a CLI argument, which may be followed by trailing text (the observer +// setters take the rest of the command line). Returns false and leaves *out +// untouched for anything else. +static inline bool parseName(const char* value, uint8_t* out) { + if (value == nullptr || out == nullptr) return false; + static const struct { const char* name; uint8_t stored; } kNames[] = { + { "min", kMin }, + { "none", kNone }, + { "max", kMax }, + }; + for (unsigned i = 0; i < sizeof(kNames) / sizeof(kNames[0]); i++) { + const size_t len = strlen(kNames[i].name); + if (strncmp(value, kNames[i].name, len) == 0 && + (value[len] == '\0' || value[len] == ' ')) { + *out = kNames[i].stored; + return true; + } + } + return false; +} + +} // namespace WifiPowerSavePolicy diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 39c8f05b..a45dd612 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1,4 +1,5 @@ #include "MQTTBridge.h" +#include "../WifiPowerSavePolicy.h" #include "../MQTTConnectionPolicy.h" #include "../MQTTMessageBuilder.h" #include "../MQTTPacketQueuePolicy.h" @@ -427,6 +428,19 @@ int MQTTBridge::getMaxActiveSlots() { #endif } +// One mapping for startup, reconnect and CLI (see WifiPowerSavePolicy). The +// stored default is `none`; `min` means MIN_MODEM here exactly as the CLI says +// it does. +void MQTTBridge::applyWifiPowerSave() { + #ifdef ESP_PLATFORM + static_assert((int)WifiPowerSavePolicy::kModeNone == (int)WIFI_PS_NONE, "wifi_ps_type_t drift"); + static_assert((int)WifiPowerSavePolicy::kModeMinModem == (int)WIFI_PS_MIN_MODEM, "wifi_ps_type_t drift"); + static_assert((int)WifiPowerSavePolicy::kModeMaxModem == (int)WIFI_PS_MAX_MODEM, "wifi_ps_type_t drift"); + if (!_obs) return; + esp_wifi_set_ps((wifi_ps_type_t)WifiPowerSavePolicy::modeFor(_obs->wifi_power_save)); + #endif +} + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } @@ -2804,6 +2818,12 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { _wifi_status_initialized = true; setWifiOutage(AlertFaultPolicy::applyWifiStatus( (uint32_t)now, current_wifi_status == WL_CONNECTED, wifiOutage(), false)); + #ifdef ESP_PLATFORM + // Already associated at bridge start (end()/begin() leaves STA up): there is + // no connect transition below to carry the setting, so apply it here or the + // node runs on whatever the previous mode was. + if (current_wifi_status == WL_CONNECTED) applyWifiPowerSave(); + #endif } if (now - _last_wifi_check <= 10000) { // Events own the snapshot between 10 s polls. If STA is associated again @@ -2827,16 +2847,7 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { s_wifi_connected_at = now; _wifi_reconnect_backoff_attempt = 0; #ifdef ESP_PLATFORM - wifi_ps_type_t ps_mode; - uint8_t ps_pref = _obs->wifi_power_save; - if (ps_pref == 1) { - ps_mode = WIFI_PS_NONE; - } else if (ps_pref == 2) { - ps_mode = WIFI_PS_MAX_MODEM; - } else { - ps_mode = WIFI_PS_NONE; // default: no power save; eliminates DTIM wake latency on mains-powered bridges - } - esp_wifi_set_ps(ps_mode); + applyWifiPowerSave(); #ifdef MQTT_WIFI_TX_POWER WiFi.setTxPower(MQTT_WIFI_TX_POWER); #else diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index e2d27253..b4ee7b99 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -517,6 +517,7 @@ private: void getClientVersion(char* buffer, size_t buffer_size) const; void logMemoryStatus(); void refreshOriginFromPrefs(); + void applyWifiPowerSave(); // one mapping, applied on every association // begin()/end()-scoped PSRAM buffers. Each allocation is independent so a // transient heap shortage degrades to the existing stack fallback instead // of making the bridge unusable. diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 34bdca4b..d62a07f1 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "WebConfigHtml.h" @@ -722,8 +723,7 @@ void WebConfigServer::handleConfigGet(AsyncWebServerRequest* req) { JsonObject wifi = doc.createNestedObject("wifi"); wifi["ssid"] = (const char*)_obs->wifi_ssid; wifi["pwd"] = _obs->wifi_password[0] ? SECRET_SENTINEL : ""; - wifi["powersave"] = _obs->wifi_power_save == 0 ? "min" - : _obs->wifi_power_save == 2 ? "max" : "none"; + wifi["powersave"] = WifiPowerSavePolicy::nameFor(_obs->wifi_power_save); JsonObject mqtt = doc.createNestedObject("mqtt"); mqtt["origin"] = (const char*)_obs->mqtt_origin; diff --git a/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp b/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp new file mode 100644 index 00000000..66803aa5 --- /dev/null +++ b/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp @@ -0,0 +1,58 @@ +#include "helpers/WifiPowerSavePolicy.h" + +#include + +using namespace WifiPowerSavePolicy; + +// F11: the CLI applied stored 0 as MIN_MODEM while the reconnect path applied it +// as NONE, so a node configured for `min` changed behaviour after every +// reconnect and `get wifi.powersave` still reported min. One table, one answer. +TEST(WifiPowerSavePolicy, StoredValuesMapToOneModeEach) { + EXPECT_EQ(kModeMinModem, modeFor(kMin)); + EXPECT_EQ(kModeNone, modeFor(kNone)); + EXPECT_EQ(kModeMaxModem, modeFor(kMax)); +} + +TEST(WifiPowerSavePolicy, NamesRoundTripWithStoredValues) { + for (uint8_t stored = 0; stored <= 2; stored++) { + uint8_t parsed = 0xFF; + ASSERT_TRUE(parseName(nameFor(stored), &parsed)) << "stored " << (int)stored; + EXPECT_EQ(stored, parsed); + EXPECT_EQ(modeFor(stored), modeFor(parsed)); + } +} + +// A byte outside the stored range must read as the product default, not as +// whatever mode happens to sit at that index. +TEST(WifiPowerSavePolicy, OutOfRangeStoredValueReadsAsDefault) { + for (int stored = 3; stored <= 255; stored++) { + EXPECT_EQ(kModeNone, modeFor((uint8_t)stored)) << "stored " << stored; + EXPECT_STREQ("none", nameFor((uint8_t)stored)); + } +} + +// The setters take the remainder of the command line, so a trailing argument +// must still parse — and a longer word starting with a valid name must not. +TEST(WifiPowerSavePolicy, ParsesCliArgumentsExactly) { + uint8_t stored = 0xFF; + + EXPECT_TRUE(parseName("min", &stored)); + EXPECT_EQ(kMin, stored); + EXPECT_TRUE(parseName("none extra", &stored)); + EXPECT_EQ(kNone, stored); + EXPECT_TRUE(parseName("max ", &stored)); + EXPECT_EQ(kMax, stored); + + stored = 0xFF; + EXPECT_FALSE(parseName("minimum", &stored)); + EXPECT_FALSE(parseName("", &stored)); + EXPECT_FALSE(parseName("off", &stored)); + EXPECT_FALSE(parseName("MIN", &stored)); + EXPECT_FALSE(parseName(nullptr, &stored)); + EXPECT_EQ(0xFF, stored); // rejected input leaves the caller's value alone +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 5ce284f0d9e89ac96a2392b08bf6a2c5894816d4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 14:44:37 -0700 Subject: [PATCH 62/93] fix(mqtt): label SDK errors from named constants; report broker refusals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four of the hand-written error labels named the wrong failure against the installed SDK: Wi-Fi reason 201 is NO_AP_FOUND (reported as "security mismatch"), 202 is AUTH_FAIL (reported as "auth mode rejected"), 39 is TIMEOUT (reported as "SSID not found") and 34 is MISSING_ACKS (reported as an "AP state mismatch"). esp-tls 0x8008 is TCP_CLOSED_FIN, not a timeout, and 0x8010 is CERT_PARTLY_OK, not a generic mbedTLS error. 0x800B, labelled "cert verify failed", is not an esp-tls error at all. So the one line an operator reads to find out why a node is offline could name the wrong cause. Move the tables into MQTTErrorLabels.h (pure, host-tested) with every value static_asserted against the SDK enum/#define actually being compiled, so a framework bump fails the build instead of relabelling errors in the field. Codes the SDK does not define — including 61/88/168, which had labels no header supports — now return no label and the caller prints the raw number. Also keep the CONNACK return code from a broker refusal and show it in `get mqttN.diag` ("refused: bad user/password (4)"). A refusal is not a transport failure, so the TLS/socket fields are empty and a slot with wrong credentials previously showed no useful detail at all. --- src/helpers/bridges/MQTTBridge.cpp | 91 ++++++++----- src/helpers/bridges/MQTTBridge.h | 4 + src/helpers/bridges/MQTTErrorLabels.h | 125 ++++++++++++++++++ .../test_mqtt_error_labels.cpp | 78 +++++++++++ 4 files changed, 267 insertions(+), 31 deletions(-) create mode 100644 src/helpers/bridges/MQTTErrorLabels.h create mode 100644 test/test_mqtt_error_labels/test_mqtt_error_labels.cpp diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index a45dd612..056af62a 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1,4 +1,5 @@ #include "MQTTBridge.h" +#include "MQTTErrorLabels.h" #include "../WifiPowerSavePolicy.h" #include "../MQTTConnectionPolicy.h" #include "../MQTTMessageBuilder.h" @@ -22,6 +23,7 @@ #ifdef ESP_PLATFORM #include +#include #include #include #include @@ -463,41 +465,52 @@ const char* MQTTBridge::getSlotPresetName(int slot_index) const { return MQTT_PRESET_CUSTOM; } +// The label tables live in MQTTErrorLabels.h so host tests can exercise them. +// These asserts are the contract between that pure table and the SDK actually +// being compiled against: a framework bump that renumbers a reason or an +// esp-tls error fails here instead of silently mislabelling it in the field. +#ifdef ESP_PLATFORM +static_assert(MQTTErrorLabels::kWifiNoApFound == WIFI_REASON_NO_AP_FOUND, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiAuthFail == WIFI_REASON_AUTH_FAIL, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiAssocFail == WIFI_REASON_ASSOC_FAIL, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiHandshakeTimeout == WIFI_REASON_HANDSHAKE_TIMEOUT, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiConnectionFail == WIFI_REASON_CONNECTION_FAIL, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiBeaconTimeout == WIFI_REASON_BEACON_TIMEOUT, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiApTsfReset == WIFI_REASON_AP_TSF_RESET, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiRoaming == WIFI_REASON_ROAMING, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiSaQueryTimeout == WIFI_REASON_SA_QUERY_TIMEOUT, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiAuthExpire == WIFI_REASON_AUTH_EXPIRE, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiAssocExpire == WIFI_REASON_ASSOC_EXPIRE, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiAssocLeave == WIFI_REASON_ASSOC_LEAVE, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiBssTransitionDisassoc == WIFI_REASON_BSS_TRANSITION_DISASSOC, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifi4WayHandshakeTimeout == WIFI_REASON_4WAY_HANDSHAKE_TIMEOUT, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiGroupCipherInvalid == WIFI_REASON_GROUP_CIPHER_INVALID, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiCipherSuiteRejected == WIFI_REASON_CIPHER_SUITE_REJECTED, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiMissingAcks == WIFI_REASON_MISSING_ACKS, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiTimeout == WIFI_REASON_TIMEOUT, "wifi reason drift"); +static_assert(MQTTErrorLabels::kWifiInvalidPmkid == WIFI_REASON_INVALID_PMKID, "wifi reason drift"); + +static_assert(MQTTErrorLabels::kTlsCannotResolveHostname == ESP_ERR_ESP_TLS_CANNOT_RESOLVE_HOSTNAME, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsCannotCreateSocket == ESP_ERR_ESP_TLS_CANNOT_CREATE_SOCKET, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsUnsupportedProtoFamily == ESP_ERR_ESP_TLS_UNSUPPORTED_PROTOCOL_FAMILY, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsFailedConnectToHost == ESP_ERR_ESP_TLS_FAILED_CONNECT_TO_HOST, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsSocketSetoptFailed == ESP_ERR_ESP_TLS_SOCKET_SETOPT_FAILED, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsConnectionTimeout == ESP_ERR_ESP_TLS_CONNECTION_TIMEOUT, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsTcpClosedFin == ESP_ERR_ESP_TLS_TCP_CLOSED_FIN, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsCertPartlyOk == ESP_ERR_MBEDTLS_CERT_PARTLY_OK, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsSetHostname == ESP_ERR_MBEDTLS_SSL_SET_HOSTNAME_FAILED, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsX509ParseFailed == ESP_ERR_MBEDTLS_X509_CRT_PARSE_FAILED, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsSslSetupFailed == ESP_ERR_MBEDTLS_SSL_SETUP_FAILED, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsSslWriteFailed == ESP_ERR_MBEDTLS_SSL_WRITE_FAILED, "esp-tls error drift"); +static_assert(MQTTErrorLabels::kTlsMbedtlsHandshakeFailed == ESP_ERR_MBEDTLS_SSL_HANDSHAKE_FAILED, "esp-tls error drift"); +#endif + const char* MQTTBridge::wifiReasonStr(uint8_t reason) { - switch (reason) { - case 2: return "auth expired"; - case 4: return "assoc timeout"; - case 8: return "AP disconnected"; - case 15: return "4-way handshake timeout"; - case 18: return "group cipher mismatch"; - case 40: return "cipher suite rejected"; - case 49: return "invalid PMKID"; - case 61: return "AP BSS management"; - case 88: return "AP BSS management"; - case 168: return "AP band-steering kick"; - case 34: return "AP state mismatch (class 3 frame)"; - case 39: return "SSID not found"; - case 63: return "SA query timeout (PMF)"; - case 200: return "signal lost"; - case 201: return "security mismatch"; - case 202: return "auth mode rejected"; - case 204: return "handshake timeout"; - default: return nullptr; - } + return MQTTErrorLabels::wifiReason(reason); } const char* MQTTBridge::tlsErrorStr(int32_t err) { - switch (err) { - case 0x8001: return "DNS failed"; - case 0x8002: return "socket error"; - case 0x8004: return "connect refused"; - case 0x8006: return "TLS timeout"; - case 0x8008: return "connection timeout"; - case 0x800B: return "cert verify failed"; - case 0x8010: return "mbedTLS error"; - case 0x801A: return "TLS handshake failed"; - default: return nullptr; - } + return MQTTErrorLabels::tlsError(err); } void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) { @@ -564,6 +577,16 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) if (slot.connected && slot.last_error_time == 0) { replyAppendf(buf, bufsize, &pos, ", no errors"); } else if (slot.last_error_time > 0) { + // Broker refusal: the CONNECT reached a broker that answered "no". Reported + // first because the transport fields below are empty or irrelevant then. + if (slot.last_connack_code != 0) { + const char* why = MQTTErrorLabels::connackReason(slot.last_connack_code); + if (why) { + replyAppendf(buf, bufsize, &pos, ", refused: %s (%u)", why, (unsigned)slot.last_connack_code); + } else { + replyAppendf(buf, bufsize, &pos, ", refused: code %u", (unsigned)slot.last_connack_code); + } + } // TLS error with human-friendly description if (slot.last_tls_err != 0) { const char* desc = tlsErrorStr(slot.last_tls_err); @@ -1660,6 +1683,7 @@ bool MQTTBridge::ensureSlotClient(int index) { _slots[index].last_tls_err = 0; _slots[index].last_tls_stack_err = 0; _slots[index].last_sock_errno = 0; + _slots[index].last_connack_code = 0; _slots[index].last_error_time = 0; _slots[index].current_outage_started_ms = 0; // clear current-outage timer for AlertReporter updateCachedConnectionStatus(); // bool store — safe from this (esp-mqtt) task @@ -1689,6 +1713,11 @@ bool MQTTBridge::ensureSlotClient(int index) { _slots[index].last_tls_stack_err = error.esp_tls_stack_err; _slots[index].last_sock_errno = error.esp_transport_sock_errno; _slots[index].last_error_time = millis(); + // Cleared on any other error type so the diag describes the latest failure + // rather than pairing a fresh transport error with an old refusal. + _slots[index].last_connack_code = + (error.error_type == MQTT_ERROR_TYPE_CONNECTION_REFUSED) + ? (uint8_t)error.connect_return_code : 0; if (error.error_type == MQTT_ERROR_TYPE_CONNECTION_REFUSED) { _slot_force_jwt_mint[index] = true; // Broker rejected the MQTT CONNECT itself — not a transport failure. diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index b4ee7b99..9dd3f29c 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -135,6 +135,10 @@ private: int32_t last_tls_err; // esp_tls_last_esp_err (0 = no error) int32_t last_tls_stack_err; // mbedTLS stack error int last_sock_errno; // socket errno + // CONNACK return code from the last broker refusal (0 = none). A refusal is + // not a transport failure, and without this the diag shows a slot with wrong + // credentials as an unexplained disconnect. + uint8_t last_connack_code; unsigned long last_error_time; // millis() of last error uint32_t disconnect_count; // Number of disconnect callbacks since boot unsigned long first_disconnect_time; // millis() of first disconnect after boot diff --git a/src/helpers/bridges/MQTTErrorLabels.h b/src/helpers/bridges/MQTTErrorLabels.h new file mode 100644 index 00000000..cee1b688 --- /dev/null +++ b/src/helpers/bridges/MQTTErrorLabels.h @@ -0,0 +1,125 @@ +#pragma once + +#include + +// Operator-facing labels for the SDK error codes the observer reports through +// `get wifi.status`, `get mqttN.diag` and the web config panel. +// +// Pure lookup: no Arduino or ESP headers, so the table is host-testable. The +// numeric values below are the installed SDK's; MQTTBridge.cpp static_asserts +// every one of them against the real enum/#define, so a framework bump either +// keeps these labels honest or fails the build. Codes with no entry return +// nullptr and the caller prints the raw number — an unknown code must never be +// given a neighbouring code's name. +namespace MQTTErrorLabels { + +// esp_wifi_types.h wifi_err_reason_t. Values below 200 are 802.11 reason codes +// forwarded from the AP; 200+ are Espressif's own local failures. +enum WifiReason : uint16_t { + kWifiAuthExpire = 2, + kWifiAssocExpire = 4, + kWifiAssocLeave = 8, + kWifi4WayHandshakeTimeout = 15, + kWifiGroupCipherInvalid = 18, + kWifiCipherSuiteRejected = 24, + kWifiMissingAcks = 34, + kWifiTimeout = 39, + kWifiInvalidPmkid = 49, + kWifiBssTransitionDisassoc = 12, + kWifiBeaconTimeout = 200, + kWifiNoApFound = 201, + kWifiAuthFail = 202, + kWifiAssocFail = 203, + kWifiHandshakeTimeout = 204, + kWifiConnectionFail = 205, + kWifiApTsfReset = 206, + kWifiRoaming = 207, + kWifiSaQueryTimeout = 209, +}; + +// esp_tls_errors.h, ESP_ERR_ESP_TLS_BASE (0x8000) + offset. +enum TlsError : int32_t { + kTlsCannotResolveHostname = 0x8001, + kTlsCannotCreateSocket = 0x8002, + kTlsUnsupportedProtoFamily= 0x8003, + kTlsFailedConnectToHost = 0x8004, + kTlsSocketSetoptFailed = 0x8005, + kTlsConnectionTimeout = 0x8006, + kTlsTcpClosedFin = 0x8008, + kTlsMbedtlsCertPartlyOk = 0x8010, + kTlsMbedtlsSetHostname = 0x8012, + kTlsMbedtlsX509ParseFailed= 0x8015, + kTlsMbedtlsSslSetupFailed = 0x8017, + kTlsMbedtlsSslWriteFailed = 0x8018, + kTlsMbedtlsHandshakeFailed= 0x801A, +}; + +// MQTT 3.1.1 CONNACK return codes (esp_mqtt_error_codes.connect_return_code). +enum ConnackCode : uint8_t { + kConnackAccepted = 0, + kConnackBadProtocol = 1, + kConnackIdRejected = 2, + kConnackServerUnavail = 3, + kConnackBadCredentials = 4, + kConnackNotAuthorized = 5, +}; + +// Short enough to sit inside a LoRa-bounded diagnostic reply next to the raw code. +static inline const char* wifiReason(uint8_t reason) { + switch (reason) { + case kWifiAuthExpire: return "auth expired"; + case kWifiAssocExpire: return "assoc expired"; + case kWifiAssocLeave: return "AP disconnected"; + case kWifiBssTransitionDisassoc: return "AP steered us away"; + case kWifi4WayHandshakeTimeout: return "4-way handshake timeout"; + case kWifiGroupCipherInvalid: return "group cipher mismatch"; + case kWifiCipherSuiteRejected: return "cipher suite rejected"; + case kWifiMissingAcks: return "AP saw no acks"; + case kWifiTimeout: return "802.11 timeout"; + case kWifiInvalidPmkid: return "invalid PMKID"; + case kWifiBeaconTimeout: return "beacon lost"; + case kWifiNoApFound: return "SSID not found"; + case kWifiAuthFail: return "auth failed (check password)"; + case kWifiAssocFail: return "association failed"; + case kWifiHandshakeTimeout: return "handshake timeout"; + case kWifiConnectionFail: return "connect failed"; + case kWifiApTsfReset: return "AP restarted"; + case kWifiRoaming: return "roaming"; + case kWifiSaQueryTimeout: return "SA query timeout (PMF)"; + default: return nullptr; + } +} + +static inline const char* tlsError(int32_t err) { + switch (err) { + case kTlsCannotResolveHostname: return "DNS failed"; + case kTlsCannotCreateSocket: return "socket error"; + case kTlsUnsupportedProtoFamily: return "unsupported protocol family"; + case kTlsFailedConnectToHost: return "connect failed"; + case kTlsSocketSetoptFailed: return "socket setopt failed"; + case kTlsConnectionTimeout: return "connection timeout"; + case kTlsTcpClosedFin: return "server closed connection"; + case kTlsMbedtlsCertPartlyOk: return "cert chain partly parsed"; + case kTlsMbedtlsSetHostname: return "SNI hostname rejected"; + case kTlsMbedtlsX509ParseFailed: return "cert parse failed"; + case kTlsMbedtlsSslSetupFailed: return "TLS setup failed"; + case kTlsMbedtlsSslWriteFailed: return "TLS write failed"; + case kTlsMbedtlsHandshakeFailed: return "TLS handshake failed"; + default: return nullptr; + } +} + +// The broker answered and refused: this is what an operator needs instead of a +// transport error, which is why the bridge keeps the CONNACK code separately. +static inline const char* connackReason(uint8_t code) { + switch (code) { + case kConnackBadProtocol: return "protocol rejected"; + case kConnackIdRejected: return "client id rejected"; + case kConnackServerUnavail: return "server unavailable"; + case kConnackBadCredentials: return "bad user/password"; + case kConnackNotAuthorized: return "not authorized"; + default: return nullptr; + } +} + +} // namespace MQTTErrorLabels diff --git a/test/test_mqtt_error_labels/test_mqtt_error_labels.cpp b/test/test_mqtt_error_labels/test_mqtt_error_labels.cpp new file mode 100644 index 00000000..604789bc --- /dev/null +++ b/test/test_mqtt_error_labels/test_mqtt_error_labels.cpp @@ -0,0 +1,78 @@ +#include "helpers/bridges/MQTTErrorLabels.h" + +#include + +#include + +using namespace MQTTErrorLabels; + +namespace { + +bool labelIs(const char* actual, const char* expected) { + return actual != nullptr && strcmp(actual, expected) == 0; +} + +} // namespace + +// F14: these four were the mislabels found in review. 201 is "no AP found" +// (usually a wrong/hidden SSID), not a security mismatch; 202 is an auth +// failure (usually a wrong password), not a rejected auth mode. +TEST(MQTTErrorLabels, WifiReasonsMatchTheSdkMeaning) { + EXPECT_TRUE(labelIs(wifiReason(201), "SSID not found")); + EXPECT_TRUE(labelIs(wifiReason(202), "auth failed (check password)")); + EXPECT_TRUE(labelIs(wifiReason(39), "802.11 timeout")); // was "SSID not found" + EXPECT_TRUE(labelIs(wifiReason(34), "AP saw no acks")); // was "AP state mismatch" +} + +TEST(MQTTErrorLabels, TlsErrorsMatchTheSdkMeaning) { + EXPECT_TRUE(labelIs(tlsError(0x8008), "server closed connection")); // was "connection timeout" + EXPECT_TRUE(labelIs(tlsError(0x8010), "cert chain partly parsed")); // was "mbedTLS error" + EXPECT_TRUE(labelIs(tlsError(0x8006), "connection timeout")); + EXPECT_TRUE(labelIs(tlsError(0x8001), "DNS failed")); + EXPECT_TRUE(labelIs(tlsError(0x801A), "TLS handshake failed")); +} + +// An unknown code must fall through to the caller's raw-number formatting +// rather than borrow a neighbouring code's label. 0x800B is the one the review +// caught: no such esp-tls error exists, but it was labelled "cert verify failed". +TEST(MQTTErrorLabels, UnknownCodesHaveNoLabel) { + EXPECT_EQ(nullptr, tlsError(0x800B)); + EXPECT_EQ(nullptr, tlsError(0)); + EXPECT_EQ(nullptr, tlsError(-1)); + EXPECT_EQ(nullptr, tlsError(0x8099)); + + EXPECT_EQ(nullptr, wifiReason(0)); + EXPECT_EQ(nullptr, wifiReason(61)); // not defined by the installed SDK + EXPECT_EQ(nullptr, wifiReason(88)); + EXPECT_EQ(nullptr, wifiReason(168)); + EXPECT_EQ(nullptr, wifiReason(255)); +} + +TEST(MQTTErrorLabels, ConnackReasonsCoverTheRefusalCodes) { + EXPECT_TRUE(labelIs(connackReason(1), "protocol rejected")); + EXPECT_TRUE(labelIs(connackReason(2), "client id rejected")); + EXPECT_TRUE(labelIs(connackReason(3), "server unavailable")); + EXPECT_TRUE(labelIs(connackReason(4), "bad user/password")); + EXPECT_TRUE(labelIs(connackReason(5), "not authorized")); + EXPECT_EQ(nullptr, connackReason(0)); // accepted: not an error to report + EXPECT_EQ(nullptr, connackReason(6)); +} + +// Labels share a bounded LoRa reply with the rest of the diagnostic line. +TEST(MQTTErrorLabels, LabelsStayShortEnoughForABoundedReply) { + for (int i = 0; i <= 255; i++) { + const char* w = wifiReason((uint8_t)i); + if (w) EXPECT_LE(strlen(w), 30u) << "wifi reason " << i; + const char* c = connackReason((uint8_t)i); + if (c) EXPECT_LE(strlen(c), 30u) << "connack code " << i; + } + for (int32_t e = 0x8000; e <= 0x8040; e++) { + const char* t = tlsError(e); + if (t) EXPECT_LE(strlen(t), 30u) << "tls error " << e; + } +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From a728d7542cb8ebef68f0cc69bf61ce815f64751f Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 14:45:06 -0700 Subject: [PATCH 63/93] fix(mqtt): honour `mqtt.status off` for the on-connect status message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the periodic status path consulted the setting. The connect callback armed a status publish unconditionally and publishStatusToSlot() never checked it, so every boot and every reconnect published a status message — including the metadata an operator turned the setting off to suppress. The documented contract is "Enable/disable status messages". Read the toggle live from prefs at publish time, matching the periodic path, and also skip a slot the operator has disabled: teardown only stops a client that reports connected, so a slot switched off mid-connect can still complete its handshake and arm this publish. --- MQTT_IMPLEMENTATION.md | 2 +- src/helpers/bridges/MQTTBridge.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index baf39ed6..9d4d16bc 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -531,7 +531,7 @@ These settings apply across all MQTT slots: #### Set Commands - `set mqtt.origin ` - Set device origin name - `set mqtt.iata ` - Set IATA code (auto-uppercased) -- `set mqtt.status on|off` - Enable/disable status messages +- `set mqtt.status on|off` - Enable/disable status messages (periodic *and* the one sent on each broker connect) - `set mqtt.packets on|off` - Enable/disable packet messages - `set mqtt.raw on|off` - Enable/disable raw messages - `set mqtt.rx on|off` - Enable/disable RX (received) packet uplinking diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 056af62a..7fa5d69e 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2604,6 +2604,16 @@ void MQTTBridge::publishStatusToSlot(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; if (!slot.client || !slot.connected) return; + // `set mqtt.status off` disables status messages, on-connect ones included + // (MQTT_IMPLEMENTATION.md: "Enable/disable status messages"). Read live from + // prefs like the periodic path, and checked here rather than at the pending + // flag so the setting that counts is the one in force when we publish. + if (!_obs->mqtt_status_enabled) return; + // A disabled slot can still hold a connection that was established before it + // was switched off (teardown only stops a client reporting connected), and + // its callback arms this publish. Do not speak for a slot the operator + // turned off. + if (!slot.enabled) return; refreshOriginFromPrefs(); From 4c7b6450d74113ae612e534aebba80133a8a9c76 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 14:45:22 -0700 Subject: [PATCH 64/93] fix(mqtt): validate NTP replies before trusting them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NTPClient::forceUpdate() treated any non-empty datagram arriving on its fixed local port 1337 as a time response: it ignored the read length, the version, the mode, the stratum, the leap indicator and any request/response correlation, then handed the bytes at offset 40 to the bridge. The bridge only checked that the derived epoch was at least 2026-01-01 before calling settimeofday() and writing the RTC. A host probe against the installed library source accepted a one-byte non-NTP datagram and produced epoch 2085978496. Anyone able to land a UDP datagram during a query window could set a bogus clock, which then feeds JWT issuance, certificate validity and packet timestamps. Replace that path with probeNtpServer(): one exchange per server on a fresh ephemeral socket, closed again on every exit. A reply is accepted only if it is a full 48 bytes, from the address and port queried, NTPv3/v4 mode 4, from a synchronised server (no leap alarm, stratum 1-15), echoing the random transmit timestamp of the request, with an epoch inside a plausible range. Rejected datagrams leave the clock alone and do not end the wait, so an early bogus packet cannot pre-empt the real answer. Era-1 (post-2036) timestamps convert forward instead of wrapping into 1900. The diagnostic now runs the same validated probe, so `get mqtt.ntp.diag` answers the question it is asked — would this server be trusted? — instead of reporting a datagram nothing checked. It also no longer leaves its UDP socket open after the probe, and each server's per-probe result carries the reason it failed ("DNS failed", "unsolicited reply", "server unsynced", ...). The diagnostic still blocks its caller; making it asynchronous is a separate change. The acceptance rules live in NtpValidation.h (pure, host-tested), including the review's one-byte-datagram reproduction. --- MQTT_IMPLEMENTATION.md | 3 +- src/helpers/NtpValidation.h | 141 ++++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 168 ++++++++++++---- src/helpers/bridges/MQTTBridge.h | 20 +- .../test_ntp_validation.cpp | 184 ++++++++++++++++++ 5 files changed, 474 insertions(+), 42 deletions(-) create mode 100644 src/helpers/NtpValidation.h create mode 100644 test/test_ntp_validation/test_ntp_validation.cpp diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 9d4d16bc..56f43569 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -524,7 +524,7 @@ These settings apply across all MQTT slots: - `get mqtt.neighbors` - Get periodic neighbors publishing setting (on/off; neighbors-enabled builds) - `get mqtt.neighbors.interval` - Get neighbors publish interval in hours (neighbors-enabled builds) - `get mqtt.ntp` - Get effective NTP server hostname -- `get mqtt.ntp.diag` - Probe every configured NTP server for connectivity (does not change the clock; serial console shows each server's reported time, LoRa shows a compact ` ok|fail` list) +- `get mqtt.ntp.diag` - Probe every configured NTP server for connectivity (does not change the clock; serial console shows each server's reported time, or why a probe was rejected — `DNS failed`, `unsolicited reply`, `server unsynced`, ... — and LoRa shows a compact ` ok|fail` list) - `get mqtt.owner` - Get owner public key (serial console only) - `get mqtt.email` - Get owner email address (serial console only) @@ -884,6 +884,7 @@ the radio actually performs in that case. - Automatic time synchronization with NTP servers (required for JWT authentication) - Default primary: `pool.ntp.org`; built-in fallbacks (tried sequentially on failure): `time.google.com`, `time.cloudflare.com`, `time.aws.com`, `time.nist.gov` - Periodic time updates (every hour) on the effective primary only; system time is kept in UTC +- Replies are validated before they are trusted: the datagram must be a full-length NTPv3/v4 server reply from the queried address and port, from a synchronised server (stratum 1-15, no leap alarm), echoing the random transmit timestamp of the request, with a plausible epoch. Anything else is discarded and the clock, the RTC and JWT issuance are left alone - Configure and diagnose with `set mqtt.ntp` / `get mqtt.ntp` / `get mqtt.ntp.diag` — see [MQTT Shared Commands](#mqtt-shared-commands) ### Authentication diff --git a/src/helpers/NtpValidation.h b/src/helpers/NtpValidation.h new file mode 100644 index 00000000..e81510b7 --- /dev/null +++ b/src/helpers/NtpValidation.h @@ -0,0 +1,141 @@ +#pragma once + +#include +#include + +// Client-side NTPv4 request building and response validation. +// +// Pure logic: no Arduino, WiFi or UDP headers, so every rejection case is +// host-testable. The caller owns the socket and supplies the datagram plus +// whether it arrived from the server it asked (see MQTTBridge::probeNtpServer). +// +// This exists because the NTPClient library accepted *any* non-empty datagram +// on its fixed local port as time: it ignored the read length, the mode, the +// stratum, the leap indicator and the request/response correlation, so a +// one-byte packet from anywhere set the clock — which then feeds JWT issuance, +// certificate validity and the RTC. Everything below is the validation that +// path never had. +namespace NtpValidation { + +static const size_t kPacketSize = 48; + +// Seconds between the NTP epoch (1900-01-01) and the Unix epoch (1970-01-01). +static const uint32_t kUnixEpochOffset = 2208988800UL; +// Wrap distance for NTP era 1, which starts 2036-02-07: era-1 seconds count +// from there, so a small NTP seconds field is a *future* time, not a past one. +static const uint32_t kEra1UnixOffset = 2085978496UL; + +enum Reject : uint8_t { + kAccepted = 0, + kShortPacket, // fewer than 48 bytes: not an NTP response at all + kWrongSource, // not from the address/port we queried + kBadVersion, // NTP version outside 3..4 + kBadMode, // not mode 4 (server) + kLeapAlarm, // LI=3: the server itself is unsynchronised + kBadStratum, // 0 (kiss-of-death) or >= 16 (unsynchronised) + kOriginMismatch, // did not echo the transmit timestamp we sent + kZeroTransmit, // no transmit timestamp + kEpochTooEarly, // before the caller's floor + kEpochTooLate, // implausibly far in the future +}; + +// The transmit timestamp we send and the server must echo back in its originate +// field. Random, so an off-path sender cannot answer a query it never saw. +struct Nonce { + uint32_t seconds; + uint32_t fraction; +}; + +struct Result { + Reject reject; + uint32_t epoch; // Unix seconds; only meaningful when reject == kAccepted +}; + +static inline uint32_t readU32(const uint8_t* p) { + return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | + ((uint32_t)p[2] << 8) | (uint32_t)p[3]; +} + +static inline void writeU32(uint8_t* p, uint32_t v) { + p[0] = (uint8_t)(v >> 24); + p[1] = (uint8_t)(v >> 16); + p[2] = (uint8_t)(v >> 8); + p[3] = (uint8_t)v; +} + +// LI=0 (no warning), VN=4, Mode=3 (client). The NTPClient request claimed LI=3, +// advertising an alarm condition it had no business asserting. +static inline void buildRequest(uint8_t out[kPacketSize], Nonce nonce) { + memset(out, 0, kPacketSize); + out[0] = 0x23; + out[1] = 0; // stratum: unspecified + out[2] = 6; // poll interval + out[3] = 0xEC; // precision + writeU32(out + 40, nonce.seconds); // transmit timestamp = our nonce + writeU32(out + 44, nonce.fraction); +} + +// Unix time for an NTP seconds field, handling the 2036 era rollover instead of +// wrapping into 1900 the way an unchecked subtraction does. +static inline uint32_t unixFromNtpSeconds(uint32_t ntp_seconds) { + return ntp_seconds >= kUnixEpochOffset ? (ntp_seconds - kUnixEpochOffset) + : (ntp_seconds + kEra1UnixOffset); +} + +// `source_matches` is the caller's check that the datagram came from the +// address and port it queried; it is a parameter rather than a lookup so this +// stays free of socket types. +static inline Result validate(const uint8_t* data, size_t len, bool source_matches, + Nonce nonce, uint32_t min_epoch, uint32_t max_epoch) { + Result r; + r.epoch = 0; + + if (data == nullptr || len < kPacketSize) { r.reject = kShortPacket; return r; } + if (!source_matches) { r.reject = kWrongSource; return r; } + + const uint8_t li = (uint8_t)((data[0] >> 6) & 0x03); + const uint8_t version = (uint8_t)((data[0] >> 3) & 0x07); + const uint8_t mode = (uint8_t)(data[0] & 0x07); + const uint8_t stratum = data[1]; + + if (version < 3 || version > 4) { r.reject = kBadVersion; return r; } + if (mode != 4) { r.reject = kBadMode; return r; } + if (li == 3) { r.reject = kLeapAlarm; return r; } + // Stratum 0 carries a kiss-of-death code (RATE/DENY/RSTR) rather than time; + // 16 and above means the server has no time to give. + if (stratum == 0 || stratum >= 16) { r.reject = kBadStratum; return r; } + + if (readU32(data + 24) != nonce.seconds || + readU32(data + 28) != nonce.fraction) { r.reject = kOriginMismatch; return r; } + + const uint32_t transmit_seconds = readU32(data + 40); + if (transmit_seconds == 0) { r.reject = kZeroTransmit; return r; } + + const uint32_t epoch = unixFromNtpSeconds(transmit_seconds); + if (epoch < min_epoch) { r.reject = kEpochTooEarly; return r; } + if (epoch > max_epoch) { r.reject = kEpochTooLate; return r; } + + r.reject = kAccepted; + r.epoch = epoch; + return r; +} + +// Short enough for a LoRa-bounded diagnostic reply. +static inline const char* rejectReason(Reject reject) { + switch (reject) { + case kAccepted: return "ok"; + case kShortPacket: return "short packet"; + case kWrongSource: return "wrong source"; + case kBadVersion: return "bad version"; + case kBadMode: return "not a server reply"; + case kLeapAlarm: return "server unsynced"; + case kBadStratum: return "bad stratum"; + case kOriginMismatch: return "unsolicited reply"; + case kZeroTransmit: return "no timestamp"; + case kEpochTooEarly: return "time too old"; + case kEpochTooLate: return "time too far ahead"; + default: return "invalid"; + } +} + +} // namespace NtpValidation diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 7fa5d69e..c2deeaed 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2,13 +2,13 @@ #include "MQTTErrorLabels.h" #include "../WifiPowerSavePolicy.h" #include "../MQTTConnectionPolicy.h" +#include "../NtpValidation.h" #include "../MQTTMessageBuilder.h" #include "../MQTTPacketQueuePolicy.h" #include "../MQTTReplyFormat.h" #include "../MQTTRuntimeBufferLifecycle.h" #include "../MQTTTopicRouter.h" #include "../TxtDataHelpers.h" -#include #include #include #include @@ -24,6 +24,7 @@ #ifdef ESP_PLATFORM #include #include +#include #include #include #include @@ -57,6 +58,10 @@ static constexpr size_t kNtpBuiltinFallbackCount = static_assert(MQTTBridge::kMaxNtpServers >= 1 + (int)kNtpBuiltinFallbackCount, "kMaxNtpServers must hold the custom primary plus all built-in fallbacks"); +// Shared so the retry loop can recognise "this name did not resolve" by pointer +// and stop retrying: nothing was sent, so a second attempt changes nothing. +static const char* const kNtpDnsFailedReason = "DNS failed"; + static bool ntpHostnameEquals(const char* a, const char* b) { if (!a || !b) return false; return strcasecmp(a, b) == 0; @@ -674,7 +679,7 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg _obs(obs), _queue_count(0), _last_status_publish(0), _last_status_retry(0), _status_interval(300000), - _ntp_client(_ntp_udp, effectiveNtpPrimary(obs), 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), + _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), _ntp_force_requested(false), _ntp_force_done(false), _ntp_force_result(false), _ntp_diag_requested(false), _ntp_diag_done(false), _ntp_diag_count(0), // Default to UTC; setRules() will be called from syncTimeWithNTP when a @@ -4026,6 +4031,106 @@ void MQTTBridge::refreshNTP() { MQTT_DEBUG_PRINTLN("NTP refresh triggered (async SNTP)"); } +// One validated NTP exchange with one named server, on a fresh ephemeral socket. +// +// Replaces NTPClient, which accepted any datagram that arrived on its fixed +// local port 1337 as time — no length, mode, stratum, leap or request/response +// check — and fed it straight into the system clock, the RTC and JWT issuance +// (F07). Everything that makes a reply trustworthy is checked here or in +// NtpValidation; a rejected reply leaves the clock alone and the loop keeps +// listening until the deadline, so an early bogus datagram cannot cancel the +// real answer. +// +// Returns false with *why set to a short reason for the diagnostic. The socket +// is opened and closed inside this call: no listener outlives the probe, and +// each server gets its own local port, so no reply can be credited to the wrong +// name (F08). +bool MQTTBridge::probeNtpServer(const char* server, uint32_t min_epoch, + uint32_t* epoch_out, const char** why) { + if (why) *why = "no reply"; + if (!server || server[0] == '\0') { if (why) *why = "no server"; return false; } + + bool have_expected_ip = false; + IPAddress expected_ip; + #ifdef ESP_PLATFORM + // Authoritative, not advisory. WiFiUDP leaves remote_ip/remote_port at the + // previous destination when a name fails to resolve, so sending anyway asks + // whichever server resolved last and credits its genuine reply to this name. + // Observed on d4: `set mqtt.ntp bogus.invalid` reported success with a correct + // epoch, answered by the pool address left over from boot. + if (!WiFi.hostByName(server, expected_ip)) { + if (why) *why = kNtpDnsFailedReason; + return false; + } + have_expected_ip = true; + #endif + + // Ephemeral local port: nothing to aim unsolicited traffic at between probes. + if (!_ntp_udp.begin(0)) { + if (why) *why = "no socket"; + return false; + } + + NtpValidation::Nonce nonce; + #ifdef ESP_PLATFORM + nonce.seconds = esp_random(); + nonce.fraction = esp_random(); + #else + nonce.seconds = (uint32_t)millis() * 2654435761UL; + nonce.fraction = (uint32_t)random(0, 0x7FFFFFFF); + #endif + + uint8_t packet[NtpValidation::kPacketSize]; + NtpValidation::buildRequest(packet, nonce); + + bool sent; + if (have_expected_ip) { + sent = _ntp_udp.beginPacket(expected_ip, kNtpPort) != 0; + } else { + sent = _ntp_udp.beginPacket(server, kNtpPort) != 0; + } + if (sent) { + _ntp_udp.write(packet, sizeof(packet)); + sent = _ntp_udp.endPacket() != 0; + } + if (!sent) { + _ntp_udp.stop(); + if (why) *why = "send failed"; + return false; + } + + bool accepted = false; + const unsigned long started = millis(); + while (millis() - started < kNtpProbeTimeoutMs) { + delay(10); + int len = _ntp_udp.parsePacket(); + if (len <= 0) continue; + + uint8_t reply[NtpValidation::kPacketSize]; + const int read_len = _ntp_udp.read(reply, sizeof(reply)); + // Discard any tail: WiFiUDP::parsePacket() refuses to read the next datagram + // while an unread one is still buffered, so an oversized reply would + // otherwise block the rest of this wait. + _ntp_udp.flush(); + const bool source_ok = (!have_expected_ip || _ntp_udp.remoteIP() == expected_ip) && + _ntp_udp.remotePort() == kNtpPort; + NtpValidation::Result r = NtpValidation::validate( + reply, read_len > 0 ? (size_t)read_len : 0, source_ok, nonce, + min_epoch, kNtpMaxValidEpoch); + if (r.reject == NtpValidation::kAccepted) { + if (epoch_out) *epoch_out = r.epoch; + if (why) *why = "ok"; + accepted = true; + break; + } + // Keep waiting: a rejected datagram must not consume this server's chance. + if (why) *why = NtpValidation::rejectReason(r.reject); + } + + _ntp_udp.stop(); + return accepted; +} + bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { if (!WiFi.isConnected()) { MQTT_DEBUG_PRINTLN("Cannot sync time - WiFi not connected"); @@ -4058,49 +4163,32 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { bool ntp_ok = false; unsigned long epochTime = 0; - const unsigned long kMinValidEpoch = 1767225600; // 2026-01-01 00:00:00 UTC + const uint32_t kMinValidEpoch = kNtpMinValidEpoch; const char* ntp_server_used = nullptr; - _ntp_client.begin(); const int kMaxNtpRetriesPerServer = 2; for (int s = 0; s < server_count && !ntp_ok; s++) { const char* server = servers[s]; - #ifdef ESP_PLATFORM - // Authoritative, not advisory. NTPClient::sendNTPPacket() ignores what - // beginPacket() returns, and WiFiUDP leaves remote_ip/remote_port at the previous - // destination when a name fails to resolve — so asking an unresolvable host sends - // the request to whichever server resolved last, and that server's genuine reply - // gets credited to this name. Observed on d4: `set mqtt.ntp bogus.invalid` reported - // success with a correct epoch, answered by the pool address left over from boot. - // Skipping is what keeps the credit honest; the name that answered is the name - // recorded. - IPAddress resolved_ip; - if (!WiFi.hostByName(server, resolved_ip)) { - MQTT_DEBUG_PRINTLN("NTP: %s does not resolve — skipping, not attempting a send", server); - continue; - } - #endif - - _ntp_client.setPoolServerName(server); - for (int attempt = 1; attempt <= kMaxNtpRetriesPerServer && !ntp_ok; attempt++) { if (attempt > 1) { MQTT_DEBUG_PRINTLN("NTP retry %d/%d on %s...", attempt, kMaxNtpRetriesPerServer, server); delay(1000); } - if (_ntp_client.forceUpdate()) { - epochTime = _ntp_client.getEpochTime(); - if (epochTime >= kMinValidEpoch) { - ntp_ok = true; - ntp_server_used = server; - } + uint32_t probed_epoch = 0; + const char* why = nullptr; + if (probeNtpServer(server, kMinValidEpoch, &probed_epoch, &why)) { + epochTime = probed_epoch; + ntp_ok = true; + ntp_server_used = server; + } else { + MQTT_DEBUG_PRINTLN("NTP: %s rejected (%s)", server, why ? why : "no reply"); + if (why == kNtpDnsFailedReason) break; // no send happened; try the next server } } } - _ntp_client.end(); - // Fallback: use ESP32 built-in SNTP (configTime) when NTPClient fails + // Fallback: use ESP32 built-in SNTP (configTime) when no server passed validation #ifdef ESP_PLATFORM if (!ntp_ok) { MQTT_DEBUG_PRINTLN("NTP client failed, trying SNTP fallback..."); @@ -4304,15 +4392,18 @@ void MQTTBridge::runNtpDiagProbe() { int count = 0; fillNtpServerList(_obs, servers, count); - _ntp_client.begin(); for (int i = 0; i < count; i++) { - _ntp_client.setPoolServerName(servers[i]); - bool ok = _ntp_client.forceUpdate(); + // Same validated probe the real sync uses, so the diagnostic answers the + // question an operator is actually asking: would this server be trusted? + uint32_t epoch = 0; + const char* why = nullptr; + bool ok = probeNtpServer(servers[i], kNtpMinValidEpoch, &epoch, &why); NtpDiagResult& r = _ntp_diag_results[i]; strncpy(r.server, servers[i], sizeof(r.server) - 1); r.server[sizeof(r.server) - 1] = '\0'; r.ok = ok; - r.epoch = ok ? (uint32_t)_ntp_client.getEpochTime() : 0; + r.epoch = ok ? epoch : 0; + r.why = why; // static literal from NtpValidation/probeNtpServer } _ntp_diag_count = count; } @@ -4346,12 +4437,13 @@ bool MQTTBridge::ntpDiag(char* reply, size_t reply_size, bool verbose) { const NtpDiagResult& r = _ntp_diag_results[i]; if (r.ok) { time_t t = (time_t)r.epoch; - struct tm* tmv = gmtime(&t); + struct tm tmv; + gmtime_r(&t, &tmv); // caller-owned storage: gmtime()'s buffer is shared Serial.printf(" %-20s OK %04d-%02d-%02d %02d:%02d:%02d UTC\r\n", - r.server, tmv->tm_year + 1900, tmv->tm_mon + 1, tmv->tm_mday, - tmv->tm_hour, tmv->tm_min, tmv->tm_sec); + r.server, tmv.tm_year + 1900, tmv.tm_mon + 1, tmv.tm_mday, + tmv.tm_hour, tmv.tm_min, tmv.tm_sec); } else { - Serial.printf(" %-20s FAIL\r\n", r.server); + Serial.printf(" %-20s FAIL %s\r\n", r.server, r.why ? r.why : "no reply"); } } snprintf(reply, reply_size, "> NTP diag: %d/%d OK (see console)", ok_count, _ntp_diag_count); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 9dd3f29c..39d169e2 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -4,7 +4,6 @@ #include "helpers/bridges/BridgeBase.h" #include #include -#include #include #include #include "helpers/JWTHelper.h" @@ -92,6 +91,13 @@ public: private: static const size_t AUTH_TOKEN_SIZE = 768; + // NTP acceptance bounds. A reply outside them is rejected outright rather + // than allowed to set the clock, the RTC and every JWT minted afterwards. + static const uint16_t kNtpPort = 123; + static const uint32_t kNtpProbeTimeoutMs = 1000; + static const uint32_t kNtpMinValidEpoch = 1767225600UL; // 2026-01-01 UTC + static const uint32_t kNtpMaxValidEpoch = 4102444800UL; // 2100-01-01 UTC + // Connection slot - each slot holds one MQTT connection struct MQTTSlot { PsychicMqttClient* client; @@ -216,9 +222,9 @@ private: #endif int _queue_count; // Protected by queue operations or mutex - // NTP time sync + // NTP time sync. The socket is opened per probe and closed again (see + // probeNtpServer); nothing listens between syncs. WiFiUDP _ntp_udp; - NTPClient _ntp_client; unsigned long _last_ntp_sync; bool _ntp_synced; bool _ntp_sync_pending; // Flag to trigger NTP sync from loop() instead of event handler @@ -266,6 +272,10 @@ private: char server[64]; bool ok; uint32_t epoch; // server-reported UTC epoch when ok + // Why a probe failed, as a static literal ("DNS failed", "unsolicited + // reply", ...). Previously every failure looked alike, and a name that never + // resolved could be credited with another server's reply. + const char* why; }; NtpDiagResult _ntp_diag_results[kMaxNtpServers]; int _ntp_diag_count; @@ -509,6 +519,10 @@ private: bool isAnySlotConnected(); void refreshNTP(); // Lightweight periodic NTP refresh (non-blocking) void runNtpDiagProbe(); // Probe every server for connectivity; never sets the clock. Core 0 only. + // One validated NTP exchange with one server on a fresh ephemeral socket. + // Core 0 only; never touches the clock. *why receives a static reason literal. + bool probeNtpServer(const char* server, uint32_t min_epoch, + uint32_t* epoch_out, const char** why); // Populates dst_out/std_out with TimeChangeRules for the given IANA or // abbreviation string. Returns false if the string is not recognized // (callers should fall back to UTC). Zero-allocation. diff --git a/test/test_ntp_validation/test_ntp_validation.cpp b/test/test_ntp_validation/test_ntp_validation.cpp new file mode 100644 index 00000000..d2222dd8 --- /dev/null +++ b/test/test_ntp_validation/test_ntp_validation.cpp @@ -0,0 +1,184 @@ +#include "helpers/NtpValidation.h" + +#include + +#include + +using namespace NtpValidation; + +namespace { + +const uint32_t MIN_EPOCH = 1767225600UL; // 2026-01-01, the bridge's floor +const uint32_t MAX_EPOCH = 4102444800UL; // 2100-01-01 +const Nonce NONCE = { 0xA1B2C3D4UL, 0x0F1E2D3CUL }; + +// A well-formed reply to buildRequest(): stratum 2, mode 4, our nonce echoed. +void makeReply(uint8_t out[48], uint32_t unix_epoch = 1789000000UL) { + memset(out, 0, 48); + out[0] = (0 << 6) | (4 << 3) | 4; // LI=0, VN=4, mode=4 (server) + out[1] = 2; // stratum + writeU32(out + 24, NONCE.seconds); // originate = our transmit timestamp + writeU32(out + 28, NONCE.fraction); + writeU32(out + 32, unix_epoch + kUnixEpochOffset); // receive + writeU32(out + 40, unix_epoch + kUnixEpochOffset); // transmit +} + +Result check(const uint8_t* data, size_t len, bool source_matches = true) { + return validate(data, len, source_matches, NONCE, MIN_EPOCH, MAX_EPOCH); +} + +} // namespace + +TEST(NtpValidation, RequestIsAClientPacketCarryingTheNonce) { + uint8_t req[48]; + memset(req, 0xFF, sizeof(req)); + buildRequest(req, NONCE); + + EXPECT_EQ(0u, (req[0] >> 6) & 0x03); // LI = 0, not the library's bogus alarm + EXPECT_EQ(4u, (req[0] >> 3) & 0x07); // version 4 + EXPECT_EQ(3u, req[0] & 0x07); // mode 3 = client + EXPECT_EQ(NONCE.seconds, readU32(req + 40)); + EXPECT_EQ(NONCE.fraction, readU32(req + 44)); +} + +TEST(NtpValidation, ValidReplyIsAccepted) { + uint8_t pkt[48]; + makeReply(pkt, 1789000000UL); + + Result r = check(pkt, sizeof(pkt)); + + EXPECT_EQ(kAccepted, r.reject); + EXPECT_EQ(1789000000UL, r.epoch); +} + +// F07's reproduction: the installed NTPClient accepted a one-byte non-NTP +// datagram and derived epoch 2085978496 from the uninitialised buffer. +TEST(NtpValidation, OneByteNonNtpDatagramIsRejected) { + const uint8_t junk[1] = { 0x00 }; + + Result r = check(junk, sizeof(junk)); + + EXPECT_EQ(kShortPacket, r.reject); + EXPECT_EQ(0u, r.epoch); +} + +TEST(NtpValidation, ShortAndEmptyDatagramsAreRejected) { + uint8_t pkt[48]; + makeReply(pkt); + + for (size_t len = 0; len < 48; len++) { + EXPECT_EQ(kShortPacket, check(pkt, len).reject) << "len " << len; + } + EXPECT_EQ(kShortPacket, check(nullptr, 48).reject); +} + +TEST(NtpValidation, ReplyFromAnotherSourceIsRejected) { + uint8_t pkt[48]; + makeReply(pkt); + + EXPECT_EQ(kWrongSource, check(pkt, sizeof(pkt), /*source_matches=*/false).reject); +} + +// An off-path sender that never saw the query cannot echo the nonce. +TEST(NtpValidation, UnsolicitedReplyIsRejected) { + uint8_t pkt[48]; + makeReply(pkt); + writeU32(pkt + 24, 0); + writeU32(pkt + 28, 0); + + EXPECT_EQ(kOriginMismatch, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + writeU32(pkt + 28, NONCE.fraction ^ 1u); // one bit off is still not ours + EXPECT_EQ(kOriginMismatch, check(pkt, sizeof(pkt)).reject); +} + +TEST(NtpValidation, ClientModeAndWrongVersionAreRejected) { + uint8_t pkt[48]; + + makeReply(pkt); + pkt[0] = (0 << 6) | (4 << 3) | 3; // our own request echoed back + EXPECT_EQ(kBadMode, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + pkt[0] = (0 << 6) | (5 << 3) | 4; // version 5 does not exist + EXPECT_EQ(kBadVersion, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + pkt[0] = (0 << 6) | (3 << 3) | 4; // NTPv3 servers are fine + EXPECT_EQ(kAccepted, check(pkt, sizeof(pkt)).reject); +} + +TEST(NtpValidation, UnsynchronisedServersAreRejected) { + uint8_t pkt[48]; + + makeReply(pkt); + pkt[0] |= (3 << 6); // LI = 3: alarm, clock not set + EXPECT_EQ(kLeapAlarm, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + pkt[1] = 0; // stratum 0: kiss-of-death + EXPECT_EQ(kBadStratum, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + pkt[1] = 16; // unsynchronised + EXPECT_EQ(kBadStratum, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt); + pkt[1] = 15; // last usable stratum + EXPECT_EQ(kAccepted, check(pkt, sizeof(pkt)).reject); +} + +TEST(NtpValidation, MissingTransmitTimestampIsRejected) { + uint8_t pkt[48]; + makeReply(pkt); + writeU32(pkt + 40, 0); + + EXPECT_EQ(kZeroTransmit, check(pkt, sizeof(pkt)).reject); +} + +// The bridge's floor is the only defence against a stale or rolled-back clock, +// and nothing previously stopped an absurd forward jump. +TEST(NtpValidation, ImplausibleTimesAreRejected) { + uint8_t pkt[48]; + + makeReply(pkt, MIN_EPOCH - 1); + EXPECT_EQ(kEpochTooEarly, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt, MIN_EPOCH); + EXPECT_EQ(kAccepted, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt, MAX_EPOCH + 1); + EXPECT_EQ(kEpochTooLate, check(pkt, sizeof(pkt)).reject); + + makeReply(pkt, MAX_EPOCH); + EXPECT_EQ(kAccepted, check(pkt, sizeof(pkt)).reject); +} + +// Era 1 begins 2036-02-07; a naive subtraction turns those seconds into 1900. +TEST(NtpValidation, Era1TimestampsConvertForward) { + EXPECT_EQ(0u, unixFromNtpSeconds(kUnixEpochOffset)); + EXPECT_EQ(2085978496UL, unixFromNtpSeconds(0)); // 2036-02-07 + EXPECT_GT(unixFromNtpSeconds(1000), unixFromNtpSeconds(0xFFFFFFFFu)); + + uint8_t pkt[48]; + makeReply(pkt); + writeU32(pkt + 40, 1000); // 1000 s into era 1 + Result r = check(pkt, sizeof(pkt)); + EXPECT_EQ(kAccepted, r.reject); + EXPECT_EQ(2085979496UL, r.epoch); +} + +TEST(NtpValidation, EveryRejectionHasAShortReason) { + for (int i = 0; i <= (int)kEpochTooLate; i++) { + const char* reason = rejectReason((Reject)i); + ASSERT_NE(nullptr, reason); + EXPECT_GT(strlen(reason), 0u); + EXPECT_LE(strlen(reason), 20u) << "reject " << i; + } +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From d52192f79b917e506e82dd7f6fac74c2244d563c Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 16:26:54 -0700 Subject: [PATCH 65/93] feat(mqtt): log the Wi-Fi power-save mode read back from the SDK The applied mode had no observable trace on a running node, which is how the CLI and the reconnect path could disagree about what `min` means for as long as they did. Log it at the one place that applies it, reading the value back with esp_wifi_get_ps() rather than printing what was requested. Used to confirm F11 on hardware: with `min` stored, a Heltec V4 now reports "WiFi power save: min (mode=1)" on association, where the old mapping applied WIFI_PS_NONE (mode=0). --- src/helpers/bridges/MQTTBridge.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index c2deeaed..0f8e1e33 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -444,7 +444,15 @@ void MQTTBridge::applyWifiPowerSave() { static_assert((int)WifiPowerSavePolicy::kModeMinModem == (int)WIFI_PS_MIN_MODEM, "wifi_ps_type_t drift"); static_assert((int)WifiPowerSavePolicy::kModeMaxModem == (int)WIFI_PS_MAX_MODEM, "wifi_ps_type_t drift"); if (!_obs) return; - esp_wifi_set_ps((wifi_ps_type_t)WifiPowerSavePolicy::modeFor(_obs->wifi_power_save)); + const uint8_t stored = _obs->wifi_power_save; + esp_wifi_set_ps((wifi_ps_type_t)WifiPowerSavePolicy::modeFor(stored)); + // Read back rather than logging what we asked for: this is the only place the + // mode is observable on a running node, and the setting used to change + // meaning between the CLI and this path. + wifi_ps_type_t applied = WIFI_PS_NONE; + esp_wifi_get_ps(&applied); + MQTT_DEBUG_PRINTLN("WiFi power save: %s (mode=%d)", + WifiPowerSavePolicy::nameFor(stored), (int)applied); #endif } From 2d4490bcaa8e7e4c89bccefe098582ae731d918b Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 17:33:46 -0700 Subject: [PATCH 66/93] fix(mqtt): release nothing after a stop the MQTT task never acknowledged (F01) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeout fallback did the two things that cannot be done safely to a task that may still be inside mbedTLS or holding esp-mqtt's API mutex: it deleted that task from the loop task, then force-stopped and deleted the clients it owned, then freed the queue and buffers it can reach. `esp_mqtt_client_stop()` waits on the API mutex and the task's stopped event with no bound, so killing its caller strands the next one; a second stop sees the run flag already clear and returns failure without joining. Nothing in that path established quiescence, and `canFlashAfterStop()` could only withhold OTA afterwards. Changes, all of them about who owns what: - `MQTTLifecycle` gains a `StopUnproven` state (appended, so existing values keep their numbering). `StopTimedOut` now leads there and releases NOTHING; it still fires `ota_release` so the OTA barrier aborts. `acceptsNewWork`, `mayRestart` and `isStopInProgress` are false there, `mayTouchOwnedState` is true, and `end()`'s wait loop still terminates. - `begin()` refuses to start while a stop is unproven. A new start was never proof that the previous clients stopped, and it used to clear the dirty latch that withheld OTA. Because a start is now impossible from `StopUnproven`, clearing that latch on an accepted start is sound. - A late acknowledgement is honoured. The task publishes it only after tearing its clients down, so it proves the same thing whenever it arrives; the deadline is an availability policy, not a statement about what the ack means. `pollLateStopAck()` releases the withheld resources and makes the bridge restartable — a slow stop (a blackholed WSS broker can hold the SDK well past the budget) no longer costs a reboot. - The handshake flags become `std::atomic` with release/acquire, and the ack moves into the task trampoline immediately before `vTaskDelete(nullptr)`, gated on a `_teardown_complete` flag the loop sets. Published from inside the loop it proved teardown returned, not that the task had stopped executing — and `volatile` ordered none of it. This is the soak campaign's blocker #20, the one item on its list that can corrupt memory. - The force-stop paths are gone with their only caller: `destroySlotClients()` and `teardownSlot()` no longer take a `force` flag. - `get mqtt.status` reports "previous stop unproven, reboot to recover" instead of a bare "not running", latched so it survives end() clearing the singleton. Two lifecycle tests asserted the old contract and are re-encoded, not deleted: a timed-out stop is no longer `Stopped`-and-released, and "repeated failed stops leave the bridge restartable" becomes "repeated *slow* stops recover on their late ack" plus a new case pinning that a never-acknowledged stop stays unusable for the boot. That availability trade is the point of the change. --- src/helpers/MQTTLifecycle.h | 51 +++++- src/helpers/bridges/MQTTBridge.cpp | 151 +++++++++++------- src/helpers/bridges/MQTTBridge.h | 42 +++-- .../test_mqtt_lifecycle.cpp | 131 +++++++++++++-- 4 files changed, 285 insertions(+), 90 deletions(-) diff --git a/src/helpers/MQTTLifecycle.h b/src/helpers/MQTTLifecycle.h index d85aafa9..099d2ee7 100644 --- a/src/helpers/MQTTLifecycle.h +++ b/src/helpers/MQTTLifecycle.h @@ -40,6 +40,13 @@ enum class State : uint8_t { Running, StopRequested, Stopping, + // A requested stop passed its deadline without an acknowledgement. The MQTT + // task may still be alive inside mbedTLS or holding the SDK's API mutex, so + // NOTHING it can reach may be released and the bridge may not restart. This + // is deliberately not Stopped: releasing on a stop we cannot prove is the + // reviewed teardown-heap-panic path (F01). Appended so the values above keep + // their numbering. + StopUnproven, }; // Events driven either by the owner (loop task) or by the MQTT task reporting @@ -135,12 +142,17 @@ inline Result apply(State s, Event e) { r.accepted = true; break; case Event::StopAcknowledged: - case Event::StopTimedOut: r.next = State::Stopped; r.effects.release_resources = true; r.effects.ota_release = true; r.accepted = true; break; + case Event::StopTimedOut: + // Unblock the OTA barrier (so it aborts) but release nothing. + r.next = State::StopUnproven; + r.effects.ota_release = true; + r.accepted = true; + break; default: // Duplicate StopRequested is a no-op (idempotent stop). break; @@ -150,16 +162,33 @@ inline Result apply(State s, Event e) { case State::Stopping: switch (e) { case Event::StopAcknowledged: - case Event::StopTimedOut: r.next = State::Stopped; r.effects.release_resources = true; r.effects.ota_release = true; r.accepted = true; break; + case Event::StopTimedOut: + r.next = State::StopUnproven; + r.effects.ota_release = true; + r.accepted = true; + break; default: break; } break; + + case State::StopUnproven: + // A late acknowledgement proves exactly what a timely one proves: the + // task published it only after tearing its clients down. The deadline was + // an availability policy, not a statement about what the ack means — so + // honour it, release, and let the bridge be restartable again. A start + // before that proof is refused (see mayRestart). + if (e == Event::StopAcknowledged) { + r.next = State::Stopped; + r.effects.release_resources = true; + r.accepted = true; + } + break; } return r; } @@ -182,6 +211,11 @@ inline bool mayTouchOwnedState(State s) { return s != State::Stopped; } // A restart (begin()) is safe only from a completed stop. inline bool mayRestart(State s) { return s == State::Stopped; } +// True while a stop has passed its deadline unproven: the owner must keep every +// resource the MQTT task can reach (queue, buffers, clients, its own task) and +// must not restart. Cleared only by a late StopAcknowledged. +inline bool isStopUnproven(State s) { return s == State::StopUnproven; } + inline bool isStopInProgress(State s) { return s == State::StopRequested || s == State::Stopping; } @@ -193,6 +227,7 @@ inline const char* stateName(State s) { case State::Running: return "Running"; case State::StopRequested: return "StopRequested"; case State::Stopping: return "Stopping"; + case State::StopUnproven: return "StopUnproven"; } return "?"; } @@ -264,10 +299,16 @@ class Coordinator { bool isStopInProgress() const { return MQTTLifecycle::isStopInProgress(_state); } - // A restart is safe from a completed stop. A stop that reached Stopped via - // the timeout fallback still allows restart (the bridge is down); only OTA - // flashing is withheld after a dirty stop. + // A restart is safe only from a PROVEN stop. An unproven stop keeps the + // bridge down until the task acknowledges late (or the node reboots): a new + // start is not proof that the previous clients stopped, and starting on top + // of them is what the review refused to approve. Because a start is + // impossible while unproven, clearing the dirty latch on start (below) can no + // longer erase an unproven stop's OTA block. bool mayRestart() const { return MQTTLifecycle::mayRestart(_state); } + // The owner polls this to decide whether it may release/restart, and calls + // onTaskStopped() again if the task acknowledges late. + bool isStopUnproven() const { return MQTTLifecycle::isStopUnproven(_state); } // OTA erase/write is permitted only after a CLEAN stop. A timed-out stop // leaves ownership uncertain, so flashing stays blocked until a clean // start/stop cycle clears the latch. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 0f8e1e33..1c5ce4ef 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -223,6 +223,11 @@ static unsigned long s_wifi_connected_at = 0; // Last WiFi disconnect reason (from ESP-IDF event). Used for get wifi.status diagnostics. static uint8_t s_wifi_disconnect_reason = 0; +// Latched by end() when a stop passes its deadline unacknowledged. end() clears +// the diagnostic singleton, so without this `get mqtt.status` could only say +// "not running" and an operator would have no way to learn that the bridge is +// down for the rest of the boot and why. +static bool s_stop_unproven = false; static unsigned long s_wifi_disconnect_time = 0; #ifdef MQTT_MEMORY_DEBUG @@ -267,7 +272,9 @@ void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPref if (buf == nullptr || bufsize == 0) return; const char* msgs = (obs && obs->mqtt_status_enabled) ? "on" : "off"; if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) { - snprintf(buf, bufsize, "> msgs: %s (bridge not running)", msgs); + snprintf(buf, bufsize, "> msgs: %s (bridge %s)", msgs, + s_stop_unproven ? "stopped: previous stop unproven, reboot to recover" + : "not running"); return; } MQTTBridge* b = s_mqtt_bridge_instance; @@ -456,6 +463,8 @@ void MQTTBridge::applyWifiPowerSave() { #endif } +bool MQTTBridge::stopUnprovenLatched() { return s_stop_unproven; } + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } @@ -871,6 +880,18 @@ void MQTTBridge::begin() { return; } + // A stop that never acknowledged leaves its task possibly still running and + // everything it can reach still owned. Starting on top of that is the failure + // F01 describes: a new start is not proof the previous clients stopped. Check + // once for a late acknowledgement (which releases and clears this), then + // refuse. Recovery is the task finishing, or a reboot. + pollLateStopAck(); + if (!_lifecycle.mayRestart()) { + MQTT_DEBUG_PRINTLN("MQTT Bridge start refused: previous stop unproven (%s) - reboot to recover", + MQTTLifecycle::stateName(_lifecycle.state())); + return; + } + // PSRAM diagnostic - helps debug memory fragmentation on boards with external RAM #ifdef BOARD_HAS_PSRAM { @@ -1067,8 +1088,9 @@ void MQTTBridge::begin() { // Clear the cooperative-stop handshake before the new task starts reading it. // deliverStop() leaves _stop_requested latched true after a stop cycle, so a // restart must reset it or the fresh task would self-terminate immediately. - _stop_requested = false; - _stop_acked = false; + _stop_requested.store(false, std::memory_order_relaxed); + _stop_acked.store(false, std::memory_order_relaxed); + _teardown_complete.store(false, std::memory_order_relaxed); BaseType_t create_result = xTaskCreatePinnedToCore( mqttTask, "MQTTBridge", @@ -1161,14 +1183,14 @@ void MQTTBridge::end() { #ifdef ESP_PLATFORM // Wait (bounded) for the task to acknowledge. tick() synthesizes the timeout - // fallback if the task never acks. Checking the ack first each iteration means - // a stop that completes right as the timeout expires is still treated as clean. + // if the task never acks. Checking the ack first each iteration means a stop + // that completes right as the timeout expires is still treated as clean. while (_lifecycle.isStopInProgress()) { - if (_stop_acked) { - _lifecycle.onTaskStopped(); // StopRequested -> Stopped (clean): releaseResources() + if (_stop_acked.load(std::memory_order_acquire)) { + _lifecycle.onTaskStopped(); // -> Stopped (proven): releaseResources() break; } - _lifecycle.tick(); // may fire StopTimedOut -> Stopped (dirty): releaseResources() + _lifecycle.tick(); // may fire StopTimedOut -> StopUnproven: releases NOTHING if (!_lifecycle.isStopInProgress()) break; vTaskDelay(pdMS_TO_TICKS(20)); } @@ -1182,10 +1204,38 @@ void MQTTBridge::end() { // Timezone is inline class storage (_timezone_storage) — nothing to delete. // The shared JSON document's pools were freed by releaseRuntimeBuffers() above. + // Not running either way, so diagnostics and publishing stop. What differs is + // ownership: after an unproven stop the task may still be alive and every + // resource it can reach is deliberately still allocated (nothing was freed + // above). begin() refuses until the task acknowledges, so _initialized == false + // cannot be turned into a second task over the same state. _initialized = false; _slots_setup_done = false; // Reset so deferred setup runs again on next begin() - MQTT_DEBUG_PRINTLN("MQTT Bridge stopped (%s)", - _lifecycle.stopTimedOut() ? "forced/timeout - OTA blocked" : "clean"); + s_stop_unproven = _lifecycle.isStopUnproven(); + if (_lifecycle.isStopUnproven()) { + MQTT_DEBUG_PRINTLN("MQTT Bridge stop UNPROVEN after %lu ms: task did not acknowledge. " + "Nothing released, restart refused, OTA blocked - reboot to recover.", + (unsigned long)_lifecycle.stopTimeoutMs()); + } else { + MQTT_DEBUG_PRINTLN("MQTT Bridge stopped (clean)"); + } +} + +// A stop whose deadline passed unproven is not necessarily wedged forever: the +// MQTT task may simply have been slow (a blackholed WSS broker can hold +// esp_mqtt_client_stop() well past the budget). Its acknowledgement means the +// same thing whenever it arrives — teardown finished and the task is about to +// stop executing — so honour it late: release the resources that were withheld +// and let the bridge be restartable again. Called from begin() (the moment it +// matters) and from the diagnostics path, both on the loop task. +void MQTTBridge::pollLateStopAck() { + if (!_lifecycle.isStopUnproven()) return; +#ifdef ESP_PLATFORM + if (!_stop_acked.load(std::memory_order_acquire)) return; + MQTT_DEBUG_PRINTLN("MQTT task acknowledged its stop late - releasing withheld resources"); + _lifecycle.onTaskStopped(); // StopUnproven -> Stopped: releaseResources() + s_stop_unproven = false; +#endif } // --------------------------------------------------------------------------- @@ -1203,38 +1253,27 @@ void MQTTBridge::LifecycleOps::startTask() { } void MQTTBridge::LifecycleOps::deliverStop() { - // Clear any stale ack before raising the request (same ordering as the NTP - // handshake: clear the done-flag, then set the request). The MQTT task polls - // _stop_requested at the top of mqttTaskLoop(). - _b->_stop_acked = false; - _b->_stop_requested = true; + // Clear the completion flags before raising the request, so the task cannot + // observe a stale ack from a previous cycle. Release on the request store + // publishes those clears to the MQTT task. + _b->_stop_acked.store(false, std::memory_order_relaxed); + _b->_teardown_complete.store(false, std::memory_order_relaxed); + _b->_stop_requested.store(true, std::memory_order_release); } void MQTTBridge::LifecycleOps::releaseResources() { MQTTBridge* b = _b; #ifdef ESP_PLATFORM - // stopTimedOut() is set before this effect fires (Coordinator::dispatch), so - // it reliably distinguishes a clean ack from the timeout fallback. - const bool dirty = b->_lifecycle.stopTimedOut(); - if (dirty && !b->_stop_acked) { - // Reviewed fallback: the task never acknowledged (likely wedged in mbedTLS). - // Force-kill it and tear down clients here on Core 1 — the pre-cooperative - // behavior — accepting the heap risk. The dirty latch keeps OTA flashing - // blocked (canFlashAfterStop() == false) so firmware is never written after - // this path. - if (b->_mqtt_task_handle != nullptr) { - vTaskDelete(b->_mqtt_task_handle); - } - // force: the task is already gone and the client is presumed wedged, so waiting on a - // DISCONNECTED event that may never arrive would hang this task (the app loop) forever. - for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) b->teardownSlot(i, /*force=*/true); - b->destroySlotClients(/*force=*/true); - } - // Clean path (or a task that acked right at the deadline): the MQTT task - // already disconnected/deleted its clients on Core 0 and self-terminated, so - // we must NOT touch slots here (that would be a cross-core double-delete). - // Just drop our handle reference; FreeRTOS reclaims the self-deleted task's - // dynamically-allocated stack/TCB in the idle task. + // This effect now fires ONLY after an acknowledged stop — timely or late (see + // MQTTLifecycle: StopTimedOut leads to StopUnproven, which releases nothing). + // So there is no force-kill branch here any more, and there must not be one: + // the task published the ack from its trampoline immediately before + // vTaskDelete(nullptr), after destroying its own clients on Core 0. Deleting + // that task from here, or tearing its clients down a second time, was the + // reviewed use-after-free (F01, soak blocker #20). + // + // The MQTT task self-terminates; FreeRTOS reclaims its stack/TCB in the idle + // task. We only drop our handle reference. b->_mqtt_task_handle = nullptr; // Drain and delete the FreeRTOS packet queue (value-copied packets, no @@ -1282,8 +1321,14 @@ void MQTTBridge::mqttTask(void* parameter) { MQTTBridge* bridge = static_cast(parameter); if (bridge) { bridge->mqttTaskLoop(); + // Last act before ceasing to execute: publish the stop acknowledgement, but + // only if the loop actually completed its ordered teardown. An unexpected + // return (mqttTaskLoop() has no other exit) must not tell the owner it is + // safe to free the queue, the buffers and the clients. + if (bridge->_teardown_complete.load(std::memory_order_acquire)) { + bridge->_stop_acked.store(true, std::memory_order_release); + } } - // Task should never return, but if it does, delete itself vTaskDelete(nullptr); } @@ -1371,13 +1416,17 @@ void MQTTBridge::mqttTaskLoop() { // vTaskDelete. Acknowledge LAST so end() only frees the queue/buffers once // this teardown has completed, then self-terminate via the mqttTask() // trampoline (vTaskDelete(nullptr)). - if (_stop_requested) { + if (_stop_requested.load(std::memory_order_acquire)) { MQTT_DEBUG_PRINTLN("MQTT task: cooperative stop - tearing down clients on Core 0"); for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { teardownSlot(i); } destroySlotClients(); - _stop_acked = true; // release semantics: set only after teardown is done + // Record that the ordered teardown finished, but do NOT publish the ack + // here: the owner treats the ack as permission to free everything this + // task can reach, and between this point and vTaskDelete(nullptr) the task + // is still executing. mqttTask() publishes it as its last act. + _teardown_complete.store(true, std::memory_order_release); return; } @@ -1789,17 +1838,11 @@ void MQTTBridge::releaseSlotAuthToken(int index) { slot.last_token_renewal = 0; } -void MQTTBridge::destroySlotClients(bool force) { +void MQTTBridge::destroySlotClients() { for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { MQTTSlot& slot = _slots[i]; if (slot.client != nullptr) { - // force is deliberately NOT gated on connected(). The state it exists for — a client - // that already took its DISCONNECTED callback and is now stuck inside - // esp_mqtt_client_stop() — reports not-connected, so gating skipped the stop exactly - // when it mattered and left the object to be deleted from under a live IDF task. - if (force) { - slot.client->forceStop(); - } else if (slot.client->connected()) { + if (slot.client->connected()) { slot.client->disconnect(); } #ifdef ESP_PLATFORM @@ -2057,18 +2100,12 @@ bool MQTTBridge::setupSlot(int index) { // the client object alive so a subsequent setupSlot() can reuse its mbedTLS // context. This is called both on reconfigure (preset change) and at shutdown; // destruction of the underlying client happens once in destroySlotClients(). -void MQTTBridge::teardownSlot(int index, bool force) { +void MQTTBridge::teardownSlot(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; - // As in destroySlotClients(): force is not gated on connected(), because the wedged - // mid-stop state it exists for already reports not-connected. - if (slot.client && (force || slot.client->connected())) { - if (force) { - slot.client->forceStop(); - } else { - slot.client->disconnect(); - } + if (slot.client && slot.client->connected()) { + slot.client->disconnect(); #ifdef ESP_PLATFORM vTaskDelay(pdMS_TO_TICKS(50)); #else diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 39d169e2..0ec28ccc 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -280,15 +280,22 @@ private: NtpDiagResult _ntp_diag_results[kMaxNtpServers]; int _ntp_diag_count; - // Cooperative-shutdown handshake (Phase 5). The loop task (Core 1) raises + // Cooperative-shutdown handshake. The loop task (Core 1) raises // _stop_requested through the lifecycle Coordinator; the MQTT task (Core 0) // sees it, tears down its own clients on Core 0 (where the mbedTLS contexts - // live), sets _stop_acked LAST, and self-terminates. end() waits for the ack - // before freeing the queue/buffers. Plain volatile matches the existing - // NTP/reconfigure handshake idiom above; replacing all of these with a command - // channel / task notifications is explicitly deferred (see MQTT_OWNERSHIP.md). - volatile bool _stop_requested = false; - volatile bool _stop_acked = false; + // live), records _teardown_complete, and publishes _stop_acked from the task + // trampoline immediately before vTaskDelete(nullptr). end() waits for that ack + // before anything is freed. + // + // std::atomic, not volatile: this is a two-flag release/acquire handshake + // across cores, and the owner acts on it by freeing memory the other task can + // reach. `volatile` orders nothing and was the soak campaign's blocker #20. + // Publishing the ack from the trampoline is what makes it mean "this task is + // about to cease executing" rather than "teardown returned"; _teardown_complete + // gates it so an unexpected return from mqttTaskLoop() cannot claim a clean stop. + std::atomic _stop_requested{false}; + std::atomic _stop_acked{false}; + std::atomic _teardown_complete{false}; // Timezone handling. // _timezone_storage is inline class storage (zero heap) that is reconfigured @@ -467,10 +474,11 @@ private: bool ensureSlotClient(int index); // Allocate this slot's persistent client + callbacks on first use bool ensureSlotAuthToken(int index); // Allocate this slot's JWT token buffer on first token creation void releaseSlotAuthToken(int index);// Free the token buffer (only with the client — see MQTTSlot) - // force=true stops each client without waiting for its DISCONNECTED event. Only the - // dirty-stop fallback passes it: disconnect()'s wait is unbounded, so a client already - // wedged in mbedTLS would block the caller — MyMesh::loop() — indefinitely. - void destroySlotClients(bool force = false); // Delete all persistent clients (shutdown only) + // No force variant: the only caller that ever passed one was the dirty-stop + // fallback, and that fallback is gone (F01). A client whose stop cannot be + // proven is now left alone rather than force-stopped and deleted under a + // possibly-live SDK task. + void destroySlotClients(); // Delete all persistent clients (shutdown only) bool setupSlot(int index); // Configure and connect the slot; false = not activated // Single definition of "this slot holds one of the _max_active_slots positions": // it is enabled and has been through a successful setupSlot(). Startup, the @@ -479,7 +487,7 @@ private: int activatedSlotCount() const; bool canActivateSlot(int index) const; // force as in destroySlotClients(): skip the unbounded wait, dirty-stop path only. - void teardownSlot(int index, bool force = false); // Disconnect the slot's client (keeps the object alive) + void teardownSlot(int index); // Disconnect the slot's client (keeps the object alive) // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a // no-op on a stopped client). See the definition. void reconnectSlotClient(int index); @@ -536,6 +544,9 @@ private: void logMemoryStatus(); void refreshOriginFromPrefs(); void applyWifiPowerSave(); // one mapping, applied on every association + // Honours a stop acknowledgement that arrived after the deadline: releases the + // withheld resources and makes the bridge restartable. Loop task only. + void pollLateStopAck(); // begin()/end()-scoped PSRAM buffers. Each allocation is independent so a // transient heap shortage degrades to the existing stack fallback instead // of making the bridge unusable. @@ -670,6 +681,13 @@ public: * OTA flashing is withheld until a clean start/stop cycle. Mirrors * MQTTLifecycle::mayBeginFlash(); read on the loop task (Core 1). */ bool canFlashAfterStop() const { return _lifecycle.mayBeginFlash(); } + // True when a stop passed its deadline without the MQTT task acknowledging: + // the bridge is down, nothing was released, and it will not restart until the + // task acknowledges late (pollLateStopAck) or the node reboots. + bool isStopUnproven() const { return _lifecycle.isStopUnproven(); } + // Survives end() clearing the diagnostic singleton, so `get mqtt.status` can + // still explain why a stopped bridge will not come back without a reboot. + static bool stopUnprovenLatched(); static unsigned long getWifiConnectedAtMillis(); diff --git a/test/test_mqtt_lifecycle/test_mqtt_lifecycle.cpp b/test/test_mqtt_lifecycle/test_mqtt_lifecycle.cpp index b5dec2f3..7b3f05e9 100644 --- a/test/test_mqtt_lifecycle/test_mqtt_lifecycle.cpp +++ b/test/test_mqtt_lifecycle/test_mqtt_lifecycle.cpp @@ -174,9 +174,12 @@ TEST(MQTTLifecycle, RestartAfterStop) { // --- Timeout / fallback ---------------------------------------------------- -// Handoff: "Timeout/fallback behavior when the MQTT task or client does not -// acknowledge." Models the reviewed fallback replacing the abrupt vTaskDelete. -TEST(MQTTLifecycle, StopTimeoutFiresReviewedFallback) { +// F01: a stop that passes its deadline unacknowledged releases NOTHING. The +// MQTT task may still be inside mbedTLS or holding the SDK's API mutex, so the +// queue, buffers, clients and the task itself all stay owned. The OTA barrier +// is still unblocked — so it can abort — but nothing is freed and the bridge +// may not restart. +TEST(MQTTLifecycle, StopTimeoutReleasesNothingAndBlocksRestart) { FakeOps ops; L::Coordinator c(ops, kStopTimeoutMs); bringUpToRunning(c); @@ -191,18 +194,70 @@ TEST(MQTTLifecycle, StopTimeoutFiresReviewedFallback) { EXPECT_EQ(L::State::StopRequested, c.state()); EXPECT_EQ(0, ops.release_calls); - // At the deadline the fallback fires: forced release, dirty OTA signal. ops.now = 1000 + kStopTimeoutMs; c.tick(); - EXPECT_EQ(L::State::Stopped, c.state()); + + EXPECT_EQ(L::State::StopUnproven, c.state()); + EXPECT_TRUE(c.isStopUnproven()); EXPECT_TRUE(c.stopTimedOut()); - EXPECT_EQ(1, ops.release_calls); + EXPECT_EQ(0, ops.release_calls) << "an unproven stop must not release"; EXPECT_EQ(1, ops.stop_complete_calls); EXPECT_FALSE(ops.last_stop_clean); - // A late ack after the fallback does not double-release. + // Everything the task can reach is still owned, and neither a restart nor a + // flash is permitted. A repeated tick does not change that. + EXPECT_TRUE(c.mayTouchOwnedState()); + EXPECT_FALSE(c.mayRestart()); + EXPECT_FALSE(c.mayBeginFlash()); + EXPECT_FALSE(c.acceptsNewWork()); + EXPECT_FALSE(c.isStopInProgress()) << "end()'s wait loop must terminate"; + + ops.now += kStopTimeoutMs * 10; + c.tick(); + EXPECT_EQ(L::State::StopUnproven, c.state()); + EXPECT_EQ(0, ops.release_calls); + + // A start attempt is refused outright: a new start is not proof that the + // previous clients stopped. + EXPECT_FALSE(c.requestStart()); + EXPECT_EQ(L::State::StopUnproven, c.state()); + EXPECT_EQ(1, ops.start_task_calls) << "no second task while unproven"; + EXPECT_TRUE(c.stopTimedOut()) << "a refused start cannot clear the dirty latch"; +} + +// The deadline is an availability policy, not a statement about what an ack +// means: the task publishes it only after tearing its clients down, so a late +// ack is exactly as trustworthy as a timely one. Honouring it releases the +// resources and makes the bridge restartable without a reboot. +TEST(MQTTLifecycle, LateAckRecoversFromAnUnprovenStop) { + FakeOps ops; + L::Coordinator c(ops, kStopTimeoutMs); + bringUpToRunning(c); + + ops.now = 1000; + ASSERT_TRUE(c.requestStop()); + ops.now = 1000 + kStopTimeoutMs; + c.tick(); + ASSERT_EQ(L::State::StopUnproven, c.state()); + ASSERT_EQ(0, ops.release_calls); + + ops.now = 1000 + kStopTimeoutMs * 4; + ASSERT_TRUE(c.onTaskStopped()); + + EXPECT_EQ(L::State::Stopped, c.state()); + EXPECT_EQ(1, ops.release_calls) << "the proof arrived; release now"; + EXPECT_TRUE(c.mayRestart()); + // The OTA barrier already reported dirty and its caller acted on that, so the + // flash gate stays shut until a clean start/stop cycle. + EXPECT_EQ(1, ops.stop_complete_calls); + EXPECT_FALSE(c.mayBeginFlash()); + EXPECT_TRUE(c.stopTimedOut()); + + // Restarting from the proven stop clears the latch, and a second ack is inert. EXPECT_FALSE(c.onTaskStopped()); EXPECT_EQ(1, ops.release_calls); + ASSERT_TRUE(c.requestStart()); + EXPECT_FALSE(c.stopTimedOut()); } TEST(MQTTLifecycle, TickWithoutPendingStopIsNoop) { @@ -371,7 +426,9 @@ TEST(MQTTLifecycle, OtaFlashBlockedUntilCleanStopAcknowledged) { } // Handoff OTA barrier: "MQTT stop times out: OTA aborts safely rather than -// writing under uncertain ownership." +// writing under uncertain ownership." The barrier is still released so the +// caller aborts, but flashing stays shut and — unlike the pre-F01 contract — +// the bridge cannot be restarted to paper over it. TEST(MQTTLifecycle, OtaAbortsWhenStopTimesOut) { FakeOps ops; L::Coordinator c(ops, kStopTimeoutMs); @@ -382,11 +439,16 @@ TEST(MQTTLifecycle, OtaAbortsWhenStopTimesOut) { ops.now = 500 + kStopTimeoutMs; c.tick(); - EXPECT_EQ(L::State::Stopped, c.state()); - EXPECT_FALSE(c.mayBeginFlash()); // dirty stop => flashing withheld - EXPECT_FALSE(ops.last_stop_clean); + EXPECT_EQ(L::State::StopUnproven, c.state()); + EXPECT_EQ(1, ops.stop_complete_calls); // barrier released... + EXPECT_FALSE(ops.last_stop_clean); // ...with "do not flash" + EXPECT_FALSE(c.mayBeginFlash()); + EXPECT_FALSE(c.requestStart()); // and no restart to hide it - // A fresh clean start/stop cycle clears the latch and re-enables flashing. + // Only the task's late acknowledgement makes the bridge usable again, and + // only a clean cycle after that re-enables flashing. + ASSERT_TRUE(c.onTaskStopped()); + EXPECT_FALSE(c.mayBeginFlash()); ASSERT_TRUE(c.requestStart()); EXPECT_FALSE(c.stopTimedOut()); ASSERT_TRUE(c.onTaskStarted()); @@ -415,8 +477,13 @@ TEST(MQTTLifecycle, NoRestartWhileStopInProgress) { } // Handoff OTA barrier: "Repeated failed OTA attempts do not ... leave the -// bridge permanently stopped." -TEST(MQTTLifecycle, RepeatedFailedStopsLeaveBridgeRestartable) { +// bridge permanently stopped." Re-encoded for the F01 contract: what keeps the +// bridge usable across repeated slow stops is the task's acknowledgement, not +// the deadline. A stop that is merely slow recovers every time; a stop that is +// never acknowledged deliberately does NOT, because restarting on top of +// clients that may still be running is the failure this state exists to +// prevent. That is an availability trade, made knowingly. +TEST(MQTTLifecycle, RepeatedSlowStopsRecoverOnTheirLateAck) { FakeOps ops; L::Coordinator c(ops, kStopTimeoutMs); @@ -427,19 +494,51 @@ TEST(MQTTLifecycle, RepeatedFailedStopsLeaveBridgeRestartable) { ops.now += 1000; ASSERT_TRUE(c.requestStop()); ops.now += kStopTimeoutMs; - c.tick(); // times out (dirty) + c.tick(); // deadline passes with no ack + EXPECT_EQ(L::State::StopUnproven, c.state()); + EXPECT_FALSE(c.mayRestart()); // not until the task proves it finished + EXPECT_EQ(attempt, ops.release_calls); + + ops.now += 1000; + ASSERT_TRUE(c.onTaskStopped()); // the slow teardown completes EXPECT_EQ(L::State::Stopped, c.state()); - EXPECT_TRUE(c.mayRestart()); // never permanently stuck + EXPECT_TRUE(c.mayRestart()); + EXPECT_EQ(attempt + 1, ops.release_calls); } EXPECT_EQ(3, ops.start_task_calls); } +// A never-acknowledged stop stays unusable for the rest of the boot: no +// release, no restart, no flash, however long the owner waits or however many +// times it asks. +TEST(MQTTLifecycle, NeverAcknowledgedStopStaysUnproven) { + FakeOps ops; + L::Coordinator c(ops, kStopTimeoutMs); + bringUpToRunning(c); + + ASSERT_TRUE(c.requestStop()); + ops.now += kStopTimeoutMs; + c.tick(); + + for (int i = 0; i < 5; ++i) { + ops.now += 60'000; + c.tick(); + EXPECT_FALSE(c.requestStart()); + EXPECT_FALSE(c.requestStop()); + EXPECT_EQ(L::State::StopUnproven, c.state()); + } + EXPECT_EQ(0, ops.release_calls); + EXPECT_FALSE(c.mayBeginFlash()); + EXPECT_TRUE(c.mayTouchOwnedState()); +} + // --- Diagnostics ----------------------------------------------------------- TEST(MQTTLifecycle, StateAndEventNamesAreStable) { EXPECT_STREQ("Stopped", L::stateName(L::State::Stopped)); EXPECT_STREQ("Running", L::stateName(L::State::Running)); EXPECT_STREQ("StopRequested", L::stateName(L::State::StopRequested)); + EXPECT_STREQ("StopUnproven", L::stateName(L::State::StopUnproven)); EXPECT_STREQ("StartRequested", L::eventName(L::Event::StartRequested)); EXPECT_STREQ("StopTimedOut", L::eventName(L::Event::StopTimedOut)); } From 0a78712ef2b2c57351bc321bf62112d715ceb615 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 17:35:49 -0700 Subject: [PATCH 67/93] feat(mqtt): effective-config value type and the recreate decision (F02/F03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decision half of the client-configuration fix, as a pure header so every transition is a host test rather than a hardware run. `MqttEffectiveConfig` is the complete owned description of what a slot's client should be configured with — URI (its own copy, because `slot.broker_uri` is rewritten in place), transport, trust policy, auth mode, credential pointers, buffer size, keepalive. Two things are deliberate: - the transport is derived from the URI scheme, and the trust policy is normalised against it: an unencrypted transport verifies nothing, whatever certificate material was requested, so a plaintext endpoint can never look like it retained a verified policy; - absent credentials are represented as nullptr and written as "", never omitted. IDF's `esp_mqtt_set_if_config()` treats NULL as "leave unchanged", so a cleared wrapper pointer cannot erase an SDK-held credential, while an empty string overwrites it and leaves the CONNECT's username flag clear (verified on an ESP32-S3 against a local broker on IDF 4.4). `mqttConfigRecreateDecision()` says whether the existing client can be reconfigured in place. Reuse covers every credential and auth-mode change and any endpoint move within one scheme — the cases that matter for reconnect and token renewal, where a client create/destroy cycle is the fork's documented internal-heap fragmentation driver. Recreation is reserved for the three fields that cannot be overwritten safely: a transport (scheme) change, a trust-policy or CA-certificate change, and growth beyond the allocated buffer capacity, which IDF 4.4 fixes at client init. Client certificates are excluded on purpose: `setClientCertificate()` has no caller in the firmware, so modelling mutual TLS here would be untested configuration surface. Noted in the header. Nothing consumes this yet; the bridge is migrated onto it after the result propagation and generation-guard steps, which it depends on. --- src/helpers/MQTTEffectiveConfig.h | 213 ++++++++++++++++++ .../test_mqtt_effective_config.cpp | 203 +++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 src/helpers/MQTTEffectiveConfig.h create mode 100644 test/test_mqtt_effective_config/test_mqtt_effective_config.cpp diff --git a/src/helpers/MQTTEffectiveConfig.h b/src/helpers/MQTTEffectiveConfig.h new file mode 100644 index 00000000..9408d524 --- /dev/null +++ b/src/helpers/MQTTEffectiveConfig.h @@ -0,0 +1,213 @@ +#pragma once + +#include +#include + +// The complete, owned description of what a slot's MQTT client should be +// configured with, plus the decision of whether the existing SDK client can be +// reconfigured in place or has to be recreated. +// +// Pure logic: no Arduino, esp-mqtt or PsychicMqttClient dependency, so every +// transition in the table below is exercised on the host +// (test/test_mqtt_effective_config). +// +// Why this exists (F02/F03). The bridge used to apply configuration field by +// field, skipping the cleanup of stale fields whenever +// `slot.initial_connect_done` was false — which `teardownSlot()` had just +// cleared, so normal live reconfiguration always skipped it. On hardware that +// sent the previous broker's JWT username in the CONNECT to a newly configured +// anonymous endpoint. Two rules fix it: +// +// 1. Every field is written on every apply. "Absent" is a value that gets +// written (the empty string), never an omission — IDF's +// `esp_mqtt_set_if_config()` treats NULL as "leave unchanged", so a +// cleared wrapper pointer cannot erase an SDK-held credential, while an +// empty string both overwrites it and leaves the CONNECT's username flag +// clear (verified on an ESP32-S3 against a local broker, IDF 4.4). +// 2. Where a field cannot be overwritten safely, the client is recreated. +// That is the exception, not the rule: esp-mqtt's stop/start cycle is the +// fork's documented internal-heap fragmentation driver, so recreation must +// never happen on the reconnect or token-renewal path. + +// Matches MQTTSlot::broker_uri so an effective config can hold its own copy. +#define MQTT_EFFECTIVE_URI_MAX 128 + +enum class MqttTransport : uint8_t { + Unknown = 0, + Tcp, // mqtt:// + Tls, // mqtts:// + Ws, // ws:// + Wss, // wss:// +}; + +enum class MqttTrust : uint8_t { + Plaintext = 0, // unencrypted transport: no server verification exists + Bundle, // shared CA bundle attach callback + PemCert, // one specific CA certificate +}; + +enum class MqttAuth : uint8_t { + None = 0, + UserPass, + Jwt, +}; + +struct MqttEffectiveConfig { + char uri[MQTT_EFFECTIVE_URI_MAX] = {0}; + MqttTransport transport = MqttTransport::Unknown; + MqttTrust trust = MqttTrust::Plaintext; + MqttAuth auth = MqttAuth::None; + // Borrowed pointers with a lifetime at least as long as the client's: preset + // certificates live in flash, credentials in the slot/bridge storage. + const char* pem = nullptr; + const char* username = nullptr; // nullptr == absent, written as "" + const char* password = nullptr; + uint16_t buffer_size = 0; + uint16_t keepalive = 0; + bool valid = false; // false until successfully built +}; + +// Client certificates are deliberately absent: `setClientCertificate()` has no +// caller in the firmware (no broker preset uses mutual TLS), so modelling it +// here would be untested configuration surface. Adding mutual TLS means adding +// it to this struct and to the recreate decision below. + +static inline MqttTransport mqttTransportFromUri(const char* uri) { + if (uri == nullptr) return MqttTransport::Unknown; + if (strncmp(uri, "mqtts://", 8) == 0) return MqttTransport::Tls; + if (strncmp(uri, "mqtt://", 7) == 0) return MqttTransport::Tcp; + if (strncmp(uri, "wss://", 6) == 0) return MqttTransport::Wss; + if (strncmp(uri, "ws://", 5) == 0) return MqttTransport::Ws; + return MqttTransport::Unknown; +} + +static inline bool mqttTransportIsEncrypted(MqttTransport t) { + return t == MqttTransport::Tls || t == MqttTransport::Wss; +} + +static inline const char* mqttTransportName(MqttTransport t) { + switch (t) { + case MqttTransport::Tcp: return "mqtt"; + case MqttTransport::Tls: return "mqtts"; + case MqttTransport::Ws: return "ws"; + case MqttTransport::Wss: return "wss"; + default: return "?"; + } +} + +// A credential the SDK must be told to forget is written as "" rather than +// left NULL. See rule 1 above. +static inline const char* mqttFieldOrEmpty(const char* v) { return v ? v : ""; } + +// Builds an effective config, deriving the transport from the URI and +// normalising the trust policy against it: an unencrypted transport verifies +// nothing, whatever certificate material was requested, so recording anything +// else would make a plaintext endpoint look like a trust change away from a +// verified one (and vice versa). Returns false for an empty or unrecognised +// URI, leaving *out invalid. +static inline bool mqttBuildEffectiveConfig(const char* uri, + MqttAuth auth, + const char* username, + const char* password, + MqttTrust requested_trust, + const char* pem, + uint16_t buffer_size, + uint16_t keepalive, + MqttEffectiveConfig* out) { + if (out == nullptr) return false; + *out = MqttEffectiveConfig(); + if (uri == nullptr || uri[0] == '\0') return false; + if (strlen(uri) >= MQTT_EFFECTIVE_URI_MAX) return false; + + const MqttTransport transport = mqttTransportFromUri(uri); + if (transport == MqttTransport::Unknown) return false; + + strncpy(out->uri, uri, MQTT_EFFECTIVE_URI_MAX - 1); + out->uri[MQTT_EFFECTIVE_URI_MAX - 1] = '\0'; + out->transport = transport; + out->auth = auth; + out->username = username; + out->password = password; + out->buffer_size = buffer_size; + out->keepalive = keepalive; + + if (!mqttTransportIsEncrypted(transport)) { + out->trust = MqttTrust::Plaintext; + out->pem = nullptr; + } else if (requested_trust == MqttTrust::PemCert && pem != nullptr) { + out->trust = MqttTrust::PemCert; + out->pem = pem; + } else if (requested_trust == MqttTrust::Bundle) { + out->trust = MqttTrust::Bundle; + } else { + // Encrypted transport with no usable trust material. Recorded as Plaintext + // trust so it is never confused with a verified policy; whether to connect + // at all is the caller's decision. + out->trust = MqttTrust::Plaintext; + } + + out->valid = true; + return true; +} + +struct MqttRecreateDecision { + bool recreate = false; + const char* reason = "reuse"; // static literal, for logs and tests +}; + +// Whether the SDK client that `applied` was written to can be reconfigured in +// place to reach `desired`. +// +// transport (scheme) change -> recreate. One `wss`->`mqtt` transition was +// observed working by reuse on hardware, but that proves one direction +// once, not that no websocket transport state survives the switch; the +// retained-`frame_state` bug class is real and unfixed upstream even in +// IDF 5.3, and this transition only happens when an operator changes the +// endpoint. +// trust change -> recreate. `cert_pem` cannot be cleared by +// writing "" (an empty PEM is a parse failure, not "no certificate"), and +// the bundle attach is a function pointer whose set_config semantics are +// not established on this build. Verification policy is the one field +// where a stale value silently weakens security. +// capacity growth -> recreate. IDF 4.4 allocates the MQTT buffers +// in `esp_mqtt_client_init()` and `esp_mqtt_set_config()` does not resize +// them; the wrapper's own reassembly buffer is likewise allocated once. +// everything else -> reuse, including every credential and +// auth-mode change, because those are rewritten unconditionally. +static inline MqttRecreateDecision mqttConfigRecreateDecision( + const MqttEffectiveConfig& applied, + const MqttEffectiveConfig& desired, + uint16_t allocated_buffer_size) { + MqttRecreateDecision d; + if (!desired.valid) { + d.recreate = false; + d.reason = "invalid-desired"; + return d; + } + if (!applied.valid) { + // Nothing applied yet: a fresh client needs configuring, not recreating. + d.reason = "first-apply"; + return d; + } + if (applied.transport != desired.transport) { + d.recreate = true; + d.reason = "transport-change"; + return d; + } + if (applied.trust != desired.trust) { + d.recreate = true; + d.reason = "trust-change"; + return d; + } + if (desired.trust == MqttTrust::PemCert && applied.pem != desired.pem) { + d.recreate = true; + d.reason = "ca-cert-change"; + return d; + } + if (desired.buffer_size > allocated_buffer_size) { + d.recreate = true; + d.reason = "buffer-growth"; + return d; + } + return d; +} diff --git a/test/test_mqtt_effective_config/test_mqtt_effective_config.cpp b/test/test_mqtt_effective_config/test_mqtt_effective_config.cpp new file mode 100644 index 00000000..70eadb00 --- /dev/null +++ b/test/test_mqtt_effective_config/test_mqtt_effective_config.cpp @@ -0,0 +1,203 @@ +#include "helpers/MQTTEffectiveConfig.h" + +#include + +#include + +namespace { + +const char kPemA[] = "-----BEGIN CERTIFICATE-----A"; +const char kPemB[] = "-----BEGIN CERTIFICATE-----B"; +const char kJwtUser[] = "v1_CC5D3CFD9C4C7B84"; +const char kToken[] = "eyJhbGciOi..."; + +MqttEffectiveConfig build(const char* uri, + MqttAuth auth = MqttAuth::None, + const char* user = nullptr, + const char* pass = nullptr, + MqttTrust trust = MqttTrust::Plaintext, + const char* pem = nullptr, + uint16_t buffer = 896) { + MqttEffectiveConfig c; + mqttBuildEffectiveConfig(uri, auth, user, pass, trust, pem, buffer, 75, &c); + return c; +} + +MqttRecreateDecision decide(const MqttEffectiveConfig& applied, + const MqttEffectiveConfig& desired, + uint16_t allocated = 896) { + return mqttConfigRecreateDecision(applied, desired, allocated); +} + +} // namespace + +TEST(MqttEffectiveConfig, TransportComesFromTheUriScheme) { + EXPECT_EQ(MqttTransport::Tcp, mqttTransportFromUri("mqtt://broker:1883")); + EXPECT_EQ(MqttTransport::Tls, mqttTransportFromUri("mqtts://broker:8883")); + EXPECT_EQ(MqttTransport::Ws, mqttTransportFromUri("ws://broker/mqtt")); + EXPECT_EQ(MqttTransport::Wss, mqttTransportFromUri("wss://broker:443/mqtt")); + EXPECT_EQ(MqttTransport::Unknown, mqttTransportFromUri("broker:1883")); + EXPECT_EQ(MqttTransport::Unknown, mqttTransportFromUri("")); + EXPECT_EQ(MqttTransport::Unknown, mqttTransportFromUri(nullptr)); + + EXPECT_TRUE(mqttTransportIsEncrypted(MqttTransport::Tls)); + EXPECT_TRUE(mqttTransportIsEncrypted(MqttTransport::Wss)); + EXPECT_FALSE(mqttTransportIsEncrypted(MqttTransport::Tcp)); + EXPECT_FALSE(mqttTransportIsEncrypted(MqttTransport::Ws)); +} + +TEST(MqttEffectiveConfig, BuildRejectsUnusableUris) { + MqttEffectiveConfig c; + EXPECT_FALSE(mqttBuildEffectiveConfig("", MqttAuth::None, nullptr, nullptr, + MqttTrust::Plaintext, nullptr, 896, 75, &c)); + EXPECT_FALSE(c.valid); + EXPECT_FALSE(mqttBuildEffectiveConfig(nullptr, MqttAuth::None, nullptr, nullptr, + MqttTrust::Plaintext, nullptr, 896, 75, &c)); + EXPECT_FALSE(mqttBuildEffectiveConfig("broker:1883", MqttAuth::None, nullptr, nullptr, + MqttTrust::Plaintext, nullptr, 896, 75, &c)); + + char too_long[MQTT_EFFECTIVE_URI_MAX + 16]; + memset(too_long, 'x', sizeof(too_long)); + memcpy(too_long, "mqtt://", 7); + too_long[sizeof(too_long) - 1] = '\0'; + EXPECT_FALSE(mqttBuildEffectiveConfig(too_long, MqttAuth::None, nullptr, nullptr, + MqttTrust::Plaintext, nullptr, 896, 75, &c)); +} + +TEST(MqttEffectiveConfig, BuildOwnsItsUriCopy) { + char uri[64]; + strcpy(uri, "mqtt://192.168.50.231:1883"); + MqttEffectiveConfig c = build(uri); + ASSERT_TRUE(c.valid); + + strcpy(uri, "mqtt://other:1883"); // slot.broker_uri is rewritten in place + EXPECT_STREQ("mqtt://192.168.50.231:1883", c.uri); +} + +// An unencrypted transport verifies nothing, whatever certificate material was +// requested. Recording anything else would make a plaintext endpoint look like +// it kept a verified policy. +TEST(MqttEffectiveConfig, TrustIsNormalisedAgainstTheTransport) { + MqttEffectiveConfig plain = build("mqtt://broker:1883", MqttAuth::None, nullptr, nullptr, + MqttTrust::Bundle, kPemA); + EXPECT_EQ(MqttTrust::Plaintext, plain.trust); + EXPECT_EQ(nullptr, plain.pem); + + MqttEffectiveConfig bundle = build("wss://broker:443/mqtt", MqttAuth::Jwt, kJwtUser, kToken, + MqttTrust::Bundle, nullptr); + EXPECT_EQ(MqttTrust::Bundle, bundle.trust); + + MqttEffectiveConfig pem = build("mqtts://broker:8883", MqttAuth::UserPass, "u", "p", + MqttTrust::PemCert, kPemA); + EXPECT_EQ(MqttTrust::PemCert, pem.trust); + EXPECT_EQ(kPemA, pem.pem); + + // Encrypted transport with no usable material is not a verified policy. + MqttEffectiveConfig none = build("mqtts://broker:8883", MqttAuth::None, nullptr, nullptr, + MqttTrust::PemCert, nullptr); + EXPECT_EQ(MqttTrust::Plaintext, none.trust); + EXPECT_EQ(nullptr, none.pem); +} + +// The six F02 transitions from the review, plus the one reproduced on hardware. +// None of them needs a new client: credentials are rewritten unconditionally. +TEST(MqttEffectiveConfig, CredentialAndAuthChangesReuseTheClient) { + const MqttEffectiveConfig jwt = build("wss://broker:443/mqtt", MqttAuth::Jwt, kJwtUser, kToken, + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig anon = build("wss://broker:443/mqtt", MqttAuth::None, nullptr, nullptr, + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig userpass = build("wss://broker:443/mqtt", MqttAuth::UserPass, "u", "p", + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig renewed = build("wss://broker:443/mqtt", MqttAuth::Jwt, kJwtUser, + "eyJhbGciOi...new", MqttTrust::Bundle, nullptr); + + EXPECT_FALSE(decide(jwt, anon).recreate); // JWT -> anonymous + EXPECT_FALSE(decide(userpass, anon).recreate); // user/pass -> anonymous + EXPECT_FALSE(decide(anon, jwt).recreate); // none -> configured + EXPECT_FALSE(decide(userpass, jwt).recreate); // user/pass -> JWT + EXPECT_FALSE(decide(jwt, renewed).recreate); // token renewal + EXPECT_FALSE(decide(jwt, jwt).recreate); // ordinary reconnect + EXPECT_STREQ("reuse", decide(jwt, jwt).reason); +} + +// Host/port/path within one scheme is an endpoint move, not a structural one. +TEST(MqttEffectiveConfig, EndpointMoveWithinOneSchemeReusesTheClient) { + const MqttEffectiveConfig a = build("mqtt://192.168.50.231:1883"); + const MqttEffectiveConfig b = build("mqtt://192.168.50.9:1884"); + EXPECT_FALSE(decide(a, b).recreate); + + const MqttEffectiveConfig wss_a = build("wss://a.example:443/mqtt", MqttAuth::Jwt, kJwtUser, + kToken, MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig wss_b = build("wss://b.example:443/other", MqttAuth::Jwt, kJwtUser, + kToken, MqttTrust::Bundle, nullptr); + EXPECT_FALSE(decide(wss_a, wss_b).recreate); +} + +TEST(MqttEffectiveConfig, TransportChangeForcesRecreate) { + const MqttEffectiveConfig wss = build("wss://broker:443/mqtt", MqttAuth::Jwt, kJwtUser, kToken, + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig tcp = build("mqtt://192.168.50.231:1883"); + const MqttEffectiveConfig tls = build("mqtts://broker:8883", MqttAuth::UserPass, "u", "p", + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig ws = build("ws://broker/mqtt"); + + EXPECT_TRUE(decide(wss, tcp).recreate); + EXPECT_STREQ("transport-change", decide(wss, tcp).reason); + EXPECT_TRUE(decide(tcp, wss).recreate); + EXPECT_TRUE(decide(wss, tls).recreate); // both encrypted, still a scheme change + EXPECT_TRUE(decide(tcp, ws).recreate); // both plaintext, still a scheme change +} + +TEST(MqttEffectiveConfig, TrustChangeForcesRecreate) { + const MqttEffectiveConfig bundle = build("mqtts://broker:8883", MqttAuth::None, nullptr, nullptr, + MqttTrust::Bundle, nullptr); + const MqttEffectiveConfig pem_a = build("mqtts://broker:8883", MqttAuth::None, nullptr, nullptr, + MqttTrust::PemCert, kPemA); + const MqttEffectiveConfig pem_b = build("mqtts://broker:8883", MqttAuth::None, nullptr, nullptr, + MqttTrust::PemCert, kPemB); + + EXPECT_TRUE(decide(bundle, pem_a).recreate); + EXPECT_STREQ("trust-change", decide(bundle, pem_a).reason); + EXPECT_TRUE(decide(pem_a, bundle).recreate); + EXPECT_TRUE(decide(pem_a, pem_b).recreate); + EXPECT_STREQ("ca-cert-change", decide(pem_a, pem_b).reason); + EXPECT_FALSE(decide(pem_a, pem_a).recreate); +} + +// IDF 4.4 allocates the MQTT buffers at client init and does not resize them, +// and the wrapper's reassembly buffer is allocated once for the client's life. +TEST(MqttEffectiveConfig, BufferGrowthForcesRecreateButShrinkDoesNot) { + const MqttEffectiveConfig small = build("mqtt://broker:1883", MqttAuth::None, nullptr, nullptr, + MqttTrust::Plaintext, nullptr, 512); + const MqttEffectiveConfig large = build("mqtt://broker:1883", MqttAuth::Jwt, kJwtUser, kToken, + MqttTrust::Plaintext, nullptr, 896); + + EXPECT_TRUE(decide(small, large, /*allocated=*/512).recreate); + EXPECT_STREQ("buffer-growth", decide(small, large, 512).reason); + EXPECT_FALSE(decide(small, large, /*allocated=*/896).recreate) + << "a client that already owns the capacity does not need recreating"; + EXPECT_FALSE(decide(large, small, /*allocated=*/896).recreate); +} + +TEST(MqttEffectiveConfig, FirstApplyAndInvalidDesiredNeverRecreate) { + const MqttEffectiveConfig unset; + const MqttEffectiveConfig wss = build("wss://broker:443/mqtt", MqttAuth::Jwt, kJwtUser, kToken, + MqttTrust::Bundle, nullptr); + EXPECT_FALSE(unset.valid); + EXPECT_FALSE(decide(unset, wss).recreate); + EXPECT_STREQ("first-apply", decide(unset, wss).reason); + + EXPECT_FALSE(decide(wss, unset).recreate); + EXPECT_STREQ("invalid-desired", decide(wss, unset).reason); +} + +// Rule 1: absent is a value that gets written, never an omission. +TEST(MqttEffectiveConfig, AbsentFieldsAreWrittenAsEmptyStrings) { + EXPECT_STREQ("", mqttFieldOrEmpty(nullptr)); + EXPECT_STREQ("alice", mqttFieldOrEmpty("alice")); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From ca986d40b53fa63f42ecd0504666aa7b20367e9c Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 17:36:46 -0700 Subject: [PATCH 68/93] fix(mqtt): size every client's buffers for a JWT CONNECT (F03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-PSRAM boards allocated 512-byte MQTT buffers for non-JWT slots, so the desired buffer size was a per-slot variable while the allocated size was fixed at `esp_mqtt_client_init()`: IDF 4.4 does not resize those buffers in `esp_mqtt_set_config()`, and the wrapper's own reassembly buffer is allocated once for the client's lifetime. A slot reconfigured from non-JWT to JWT therefore kept 512-byte buffers, and a valid JWT CONNECT (frame plus a 768-byte token) could not fit — with no error naming the cause. Stopping and starting the same handle could not change the capacity either. Give every client 896 bytes on every board. The cost is 384 bytes per client on a non-PSRAM board, which caps at 2 active slots, against a TLS handshake that needs 16 KB of contiguous internal DRAM. In exchange the capacity transition stops existing, so the reconfigure path never needs to recreate a client to grow its buffers. --- src/helpers/bridges/MQTTBridge.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1c5ce4ef..8c0ee058 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -4674,15 +4674,23 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ // preserving at-least-once delivery while capping duplicates at one. client->setMessageRetransmitTimeout(15000); - // Buffer sizing: 896 is the minimum safe size for JWT clients (CONNECT + 768-byte JWT). - // On PSRAM boards, use a uniform size to reduce fragmentation from mixed allocations. - // On non-PSRAM boards, use smaller buffers for non-JWT slots to reduce heap usage and - // leave smaller holes during teardown/recreate cycles. -#if defined(BOARD_HAS_PSRAM) + // Buffer sizing: 896 is the minimum safe size for a JWT CONNECT (frame plus a + // 768-byte token). Every client gets it, on every board. + // + // Non-PSRAM boards used to get 512 for non-JWT slots, which made the *desired* + // size a per-slot variable while the *allocated* size was fixed at client init + // — IDF 4.4 allocates the MQTT buffers in esp_mqtt_client_init() and + // esp_mqtt_set_config() does not resize them, and the wrapper's own reassembly + // buffer is likewise allocated once (F03). A slot reconfigured from non-JWT to + // JWT therefore kept 512-byte buffers and its valid CONNECT could not fit, + // with no error that named the cause. Sizing every client for the largest + // CONNECT it might ever have to send removes the transition entirely. + // + // The cost is 384 bytes per client on a non-PSRAM board (at most 2 active + // slots there), against a handshake that needs 16 KB of *contiguous* internal + // DRAM — noise, and it buys the removal of a recreate case. static const int MQTT_CLIENT_BUFFER_SIZE = 896; -#else - const int MQTT_CLIENT_BUFFER_SIZE = needs_large_buffer ? 896 : 512; -#endif + (void)needs_large_buffer; // kept: callers still express the intent client->setBufferSize(MQTT_CLIENT_BUFFER_SIZE); From abe838bf58b2bba24e76f826234ea079195970ec Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 17:40:45 -0700 Subject: [PATCH 69/93] fix(mqtt): typed results for client operations, and stop advancing state on failure (F06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `connect()`, `reconnect()`, `disconnect()`, `softDisconnect()` and `forceStop()` all returned void, so every caller in the bridge treated "asked" as "done": - `setupSlot()` marked a slot activated after `connect()` whatever happened. A failed `esp_mqtt_client_start()` therefore consumed one of the scarce active-slot positions, handed the slot to a reconnect ladder that is gated on activation, and was never retried by the deferred-setup path. - `reconnect()` explicitly proceeded after `esp_mqtt_set_config()` failed, reconnecting on the previous configuration — the renewed token in the buffer, the old one on the wire. - the renewal path recorded the new expiry before the bounce succeeded, so a failed bounce left the live session on the old credential with the next renewal not due for a whole token lifetime. - `softDisconnect()` logged its timeout and told its caller nothing. - `disconnect()` waited for the DISCONNECTED event with no bound, on the very task whose stop acknowledgement the shutdown waits for. Now every one of them returns `esp_err_t`, a failed configuration transaction aborts rather than starting or reconnecting on a half-updated config, and `disconnect()`'s wait is bounded (it still stops the client, and reports ESP_ERR_TIMEOUT when the event never arrived). Bridge consequences: - a failed start leaves the slot unactivated, so the existing deferred-setup retry revisits it and it holds no active-slot position; - a failed renewal bounce re-arms the renewal instead of recording it, so the next maintenance pass retries; - a reconnect that fails *locally* rolls back the backoff advance made for it. The ladder and the breaker bound broker and network faults; an uninitialised client or an uncommitted config transaction is neither, and inflating the ladder for it was how a local fault could trip a breaker meant for a broker. --- .../src/PsychicMqttClient.cpp | 133 +++++++++++------- lib/PsychicMqttClient/src/PsychicMqttClient.h | 47 +++++-- src/helpers/bridges/MQTTBridge.cpp | 78 ++++++++-- src/helpers/bridges/MQTTBridge.h | 5 +- 4 files changed, 194 insertions(+), 69 deletions(-) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index ba1dc5f1..cec11177 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -371,20 +371,20 @@ bool PsychicMqttClient::connected() return _connected; } -void PsychicMqttClient::connect() +esp_err_t PsychicMqttClient::connect() { #if ESP_IDF_VERSION_MAJOR == 5 if (_mqtt_cfg.broker.address.uri == nullptr) { ESP_LOGE(TAG, "MQTT URI not set."); - return; + return ESP_ERR_INVALID_STATE; } int desired_buffer = _mqtt_cfg.buffer.size > 0 ? _mqtt_cfg.buffer.size : 1024; #else if (_mqtt_cfg.uri == nullptr) { ESP_LOGE(TAG, "MQTT URI not set."); - return; + return ESP_ERR_INVALID_STATE; } int desired_buffer = _mqtt_cfg.buffer_size > 0 ? _mqtt_cfg.buffer_size : 1024; #endif @@ -400,41 +400,44 @@ void PsychicMqttClient::connect() { ESP_LOGE(TAG, "Failed to allocate reassembly buffer (%u bytes)", (unsigned)_buffer_capacity); _buffer_capacity = 0; + return ESP_ERR_NO_MEM; } } if (_client == nullptr) { _client = esp_mqtt_client_init(&_mqtt_cfg); + if (_client == nullptr) + { + ESP_LOGE(TAG, "esp_mqtt_client_init failed"); + return ESP_ERR_NO_MEM; + } // Register event handler only once when client is first created // to avoid memory leak from repeated registrations esp_mqtt_client_register_event(_client, MQTT_EVENT_ANY, _onMqttEventStatic, this); _config_dirty = false; } + else if (_config_dirty) + { + // A failed config update must not be followed by a start: the client + // would come up on a partly-updated configuration, which is how a slot + // connects with the previous broker's credentials. Leave _config_dirty + // set so the next attempt retries the whole transaction. + esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); + if (cfg_result != ESP_OK) + { + ESP_LOGE(TAG, "connect(): failed to apply mqtt config: %s", esp_err_to_name(cfg_result)); + return cfg_result; + } + _config_dirty = false; + ESP_LOGD(TAG, "connect(): applied mqtt config update"); + } else { - if (_config_dirty) - { - esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); - ESP_ERROR_CHECK_WITHOUT_ABORT(cfg_result); - if (cfg_result == ESP_OK) - { - _config_dirty = false; - ESP_LOGD(TAG, "connect(): applied mqtt config update"); - } - else - { - ESP_LOGW(TAG, "connect(): failed to apply mqtt config, will retry"); - } - } - else - { - ESP_LOGD(TAG, "connect(): mqtt config unchanged, skipping set_config"); - } + ESP_LOGD(TAG, "connect(): mqtt config unchanged, skipping set_config"); } esp_err_t start_result = esp_mqtt_client_start(_client); - ESP_ERROR_CHECK_WITHOUT_ABORT(start_result); if (start_result == ESP_OK) { _started = true; @@ -445,76 +448,103 @@ void PsychicMqttClient::connect() // Reporting success here hides the one state reconnect() cannot recover from. ESP_LOGE(TAG, "MQTT client failed to start: %s", esp_err_to_name(start_result)); } + return start_result; } -void PsychicMqttClient::reconnect() +esp_err_t PsychicMqttClient::reconnect() { if (_client == nullptr) { ESP_LOGW(TAG, "MQTT client not initialized, cannot reconnect."); - return; + return ESP_ERR_INVALID_STATE; } if (_config_dirty) { - // Apply config only when mutating setters changed _mqtt_cfg. + // Apply config only when mutating setters changed _mqtt_cfg. A failure + // aborts the reconnect: reconnecting on the previous config was how a + // renewed token silently failed to reach the connection. esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); - ESP_ERROR_CHECK_WITHOUT_ABORT(cfg_result); - if (cfg_result == ESP_OK) + if (cfg_result != ESP_OK) { - _config_dirty = false; - ESP_LOGD(TAG, "reconnect(): applied mqtt config update"); - } - else - { - ESP_LOGW(TAG, "reconnect(): failed to apply mqtt config, reconnecting with previous config"); + ESP_LOGE(TAG, "reconnect(): failed to apply mqtt config: %s", esp_err_to_name(cfg_result)); + return cfg_result; } + _config_dirty = false; + ESP_LOGD(TAG, "reconnect(): applied mqtt config update"); } else { ESP_LOGD(TAG, "reconnect(): mqtt config unchanged, skipping set_config"); } - ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_reconnect(_client)); + esp_err_t r = esp_mqtt_client_reconnect(_client); + if (r != ESP_OK) + { + ESP_LOGE(TAG, "MQTT reconnect request failed: %s", esp_err_to_name(r)); + return r; + } ESP_LOGI(TAG, "MQTT client reconnect requested."); + return ESP_OK; } -void PsychicMqttClient::disconnect() +esp_err_t PsychicMqttClient::disconnect(unsigned long timeout_ms) { if (_client == nullptr) { ESP_LOGW(TAG, "MQTT client not started."); - return; + return ESP_ERR_INVALID_STATE; } + bool clean = true; if (_connected) { ESP_LOGI(TAG, "Disconnecting MQTT client."); _stopMqttClient = false; esp_mqtt_client_disconnect(_client); - // Wait for all disconnect events to be processed - while (!_stopMqttClient) + // Bounded, unlike the original: this runs on the MQTT task whose stop + // acknowledgement the bridge's shutdown waits for, so a lost + // DISCONNECTED event used to wedge the whole teardown. + unsigned long waited = 0; + while (!_stopMqttClient && waited < timeout_ms) { vTaskDelay(10 / portTICK_PERIOD_MS); + waited += 10; + } + if (!_stopMqttClient) + { + ESP_LOGW(TAG, "disconnect: no DISCONNECTED event in %lums; stopping anyway", timeout_ms); + clean = false; } } - esp_mqtt_client_stop(_client); - _started = false; - ESP_LOGI(TAG, "MQTT client stopped."); + esp_err_t stop_result = esp_mqtt_client_stop(_client); + if (stop_result == ESP_OK) + { + _started = false; + ESP_LOGI(TAG, "MQTT client stopped."); + } + else + { + // The SDK task was not joined. Saying otherwise is what let the caller + // destroy the client from under a live task. + ESP_LOGE(TAG, "esp_mqtt_client_stop failed: %s", esp_err_to_name(stop_result)); + return stop_result; + } + return clean ? ESP_OK : ESP_ERR_TIMEOUT; } -void PsychicMqttClient::softDisconnect(unsigned long timeout_ms) +esp_err_t PsychicMqttClient::softDisconnect(unsigned long timeout_ms) { if (_client == nullptr) { ESP_LOGW(TAG, "MQTT client not started."); - return; + return ESP_ERR_INVALID_STATE; } if (!_connected) { // Nothing to close; leaving the task alone is the whole point. - return; + return ESP_OK; } ESP_LOGI(TAG, "Disconnecting MQTT transport (client task retained)."); @@ -530,25 +560,34 @@ void PsychicMqttClient::softDisconnect(unsigned long timeout_ms) if (!_stopMqttClient) { ESP_LOGW(TAG, "softDisconnect: no DISCONNECTED event in %lums", timeout_ms); + return ESP_ERR_TIMEOUT; } + return ESP_OK; } -void PsychicMqttClient::forceStop() +esp_err_t PsychicMqttClient::forceStop() { if (_client == nullptr) { ESP_LOGW(TAG, "MQTT client not started."); - return; + return ESP_ERR_INVALID_STATE; } if (_connected) { ESP_LOGI(TAG, "Forced stop MQTT client."); } - ESP_ERROR_CHECK_WITHOUT_ABORT(esp_mqtt_client_stop(_client)); + esp_err_t r = esp_mqtt_client_stop(_client); + if (r != ESP_OK) + { + // The SDK task was not joined: the caller must not destroy this client. + ESP_LOGE(TAG, "forceStop: esp_mqtt_client_stop failed: %s", esp_err_to_name(r)); + return r; + } _connected = false; _started = false; ESP_LOGI(TAG, "MQTT client forcefully stopped."); + return ESP_OK; } int PsychicMqttClient::subscribe(const char *topic, int qos) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index 28e53fc3..5d214679 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -349,8 +349,17 @@ public: * @brief Connects the MQTT client to the server. * * @note All parameters must be set before calling this method. + * + * @return ESP_OK when esp_mqtt_client_start() accepted the client. + * ESP_ERR_INVALID_STATE if no URI is set, ESP_ERR_NO_MEM if the + * client or its reassembly buffer could not be allocated, or the + * error from esp_mqtt_set_config()/esp_mqtt_client_start(). + * + * A failed configuration update does NOT start the client: starting with a + * half-updated config is how a slot ends up connected with the previous + * broker's credentials. */ - void connect(); + esp_err_t connect(); /** * @brief Reconnects a previously started MQTT client. @@ -359,14 +368,28 @@ public: * re-initiating a connection on an already-started client, especially * when auto-reconnect is disabled. Updates config before reconnecting * so credential changes (e.g., refreshed JWT tokens) take effect. + * + * @return ESP_OK when the reconnect was requested, ESP_ERR_INVALID_STATE on + * an uninitialised client, or the error from esp_mqtt_set_config(). + * A failed config update aborts the reconnect rather than + * reconnecting with the previous configuration. */ - void reconnect(); + esp_err_t reconnect(); /** - * @brief Disconnects the MQTT client from the server. - * This call might be blocking until the client is stopped cleanly + * @brief Disconnects the MQTT client and stops its task. + * + * Blocks until the DISCONNECTED event arrives or timeout_ms elapses, then + * stops the client either way. The wait used to be unbounded, which turned + * a lost event into a wedged caller — and that caller is the MQTT task + * whose acknowledgement the bridge's shutdown waits on. + * + * @return ESP_OK when the client stopped after a clean disconnect, + * ESP_ERR_TIMEOUT when the DISCONNECTED event never arrived (the + * stop still ran), ESP_ERR_INVALID_STATE on an uninitialised + * client, or the error from esp_mqtt_client_stop(). */ - void disconnect(); + esp_err_t disconnect(unsigned long timeout_ms = 10000); /** * @brief Closes the transport but leaves the client task running. @@ -380,8 +403,12 @@ public: * @param timeout_ms how long to wait for the DISCONNECTED event before * giving up. Bounded on purpose: disconnect()'s wait is * unbounded and a lost event would wedge the caller. + * + * @return ESP_OK when the DISCONNECTED event arrived, ESP_ERR_TIMEOUT if it + * did not (the transport may still be open), ESP_ERR_INVALID_STATE + * on an uninitialised client, ESP_OK when already disconnected. */ - void softDisconnect(unsigned long timeout_ms = 5000); + esp_err_t softDisconnect(unsigned long timeout_ms = 5000); /** * @brief True once esp_mqtt_client_start() has succeeded and no stop has run. @@ -392,10 +419,14 @@ public: bool isStarted() const { return _started; } /** - * @brief Forcefully stops the MQTT client and disconnects from the server. + * @brief Stops the MQTT client without waiting for a DISCONNECTED event. * This does not trigger the onDisconnect callbacks. + * + * @return the result of esp_mqtt_client_stop(), or ESP_ERR_INVALID_STATE on + * an uninitialised client. A non-OK result means the SDK task was + * NOT joined: the caller must not destroy the client after it. */ - void forceStop(); + esp_err_t forceStop(); /** * @brief Subscribes to a topic. Server must be connected diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 8c0ee058..97ecd867 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2091,7 +2091,19 @@ bool MQTTBridge::setupSlot(int index) { } } - slot.client->connect(); + // Activation is now conditional on the client actually starting. A failed + // esp_mqtt_set_config()/esp_mqtt_client_start() used to be invisible: the slot + // was marked activated, so it consumed one of the scarce active-slot + // positions, the reconnect ladder (which is gated on activation) governed it, + // and nothing retried the setup. Leaving it unactivated hands it to the + // existing deferred-setup retry in maintainSlotConnections() instead. + const esp_err_t connect_result = slot.client->connect(); + if (connect_result != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d start failed (%s) - will retry", index + 1, + esp_err_to_name(connect_result)); + slot.last_reconnect_attempt = millis(); + return false; + } slot.initial_connect_done = true; return true; } @@ -2137,17 +2149,16 @@ void MQTTBridge::teardownSlot(int index) { // which only stops slots still marked connected: a publishing slot's socket fails first, so // the guard skips it — measured across a 62 s deauth, five slots, zero stops. An idle slot // with no traffic to fail on is the one case that could still reach here that way. -void MQTTBridge::reconnectSlotClient(int index) { - if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; +esp_err_t MQTTBridge::reconnectSlotClient(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return ESP_ERR_INVALID_ARG; MQTTSlot& slot = _slots[index]; - if (slot.client == nullptr) return; + if (slot.client == nullptr) return ESP_ERR_INVALID_STATE; if (!slot.client->isStarted()) { MQTT_DEBUG_PRINTLN("MQTT%d start (client was stopped)", index + 1); - slot.client->connect(); - return; + return slot.client->connect(); } - slot.client->reconnect(); + return slot.client->reconnect(); } @@ -2304,21 +2315,40 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // to avoid internal heap leak/fragmentation from destroy/create cycles MQTT_DEBUG_PRINTLN("MQTT%d token renewal: reconnecting with fresh credentials", index + 1); MQTT_TRACE_HEAP("renewal:before-bounce", index); + esp_err_t bounce_result; if (slot.client->isStarted()) { // Keep the esp-mqtt task alive across the handshake. disconnect() // would stop it, returning its 6 KiB stack into the hole the two // 16 KiB mbedTLS record buffers just vacated — which is what walks // the largest free block down 16 KiB at a time on non-PSRAM boards. - slot.client->softDisconnect(); + const esp_err_t soft = slot.client->softDisconnect(); + if (soft != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d renewal: soft disconnect did not complete (%s)", + index + 1, esp_err_to_name(soft)); + } MQTT_TRACE_HEAP("renewal:after-disconnect", index); slot.client->setCredentials(_jwt_username, slot.auth_token); MQTT_TRACE_HEAP("renewal:after-credentials", index); - slot.client->reconnect(); + bounce_result = slot.client->reconnect(); } else { // Client was stopped (teardown/reconfigure). reconnect() is a no-op // on a stopped client, so this path must start it. slot.client->setCredentials(_jwt_username, slot.auth_token); - slot.client->connect(); + bounce_result = slot.client->connect(); + } + if (bounce_result != ESP_OK) { + // The fresh token is in the buffer but did not reach the + // connection: the config transaction failed, or the client would + // not start. Recording this as a completed renewal would leave the + // live session on the old credential until the broker enforced exp, + // with the next renewal not due for a whole token lifetime. Re-arm + // the renewal instead so the next maintenance pass retries the + // bounce, and leave the reconnect ladder alone — this is a local + // failure, not a broker fault. + MQTT_DEBUG_PRINTLN("MQTT%d renewal bounce failed (%s) - retrying next pass", + index + 1, esp_err_to_name(bounce_result)); + slot.last_token_renewal = 0; + return; } MQTT_TRACE_HEAP("renewal:after-reconnect", index); reconnect_attempted = true; @@ -2439,6 +2469,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns static_cast(now_millis), static_cast(slot.last_reconnect_attempt), slot.reconnect_backoff, static_cast(index))) { slot.last_reconnect_attempt = now_millis; + const uint8_t backoff_before = slot.reconnect_backoff; + const uint8_t failures_before = slot.max_backoff_failures; + const bool breaker_before = slot.circuit_breaker_tripped; MQTTConnectionPolicy::BackoffAdvance advance = MQTTConnectionPolicy::advanceBackoff( slot.reconnect_backoff, slot.max_backoff_failures); slot.reconnect_backoff = advance.reconnect_backoff; @@ -2463,7 +2496,18 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns } // Via the helper: reconnect() is a no-op on a client whose start failed, // which would back off forever without ever starting it. - reconnectSlotClient(index); + const esp_err_t r = reconnectSlotClient(index); + if (r != ESP_OK) { + // A local failure — uninitialised client, or a config transaction that + // did not commit — is not a broker fault. The ladder and the breaker + // exist to bound broker/network faults, so roll back the advance made + // above; the retry interval still paces the next attempt. + MQTT_DEBUG_PRINTLN("MQTT%d reconnect not accepted locally (%s) - backoff unchanged", + index + 1, esp_err_to_name(r)); + slot.reconnect_backoff = backoff_before; + slot.max_backoff_failures = failures_before; + slot.circuit_breaker_tripped = breaker_before; + } } } } @@ -4367,8 +4411,16 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { reconnectSlotClient(i); } else if (action == MQTTConnectionPolicy::StaleTokenAction::Bounce) { MQTT_DEBUG_PRINTLN("MQTT%d bouncing for the corrected-clock token", i + 1); - _slots[i].client->softDisconnect(); - _slots[i].client->reconnect(); + const esp_err_t soft = _slots[i].client->softDisconnect(); + if (soft != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d soft disconnect did not complete (%s)", i + 1, + esp_err_to_name(soft)); + } + const esp_err_t rc = _slots[i].client->reconnect(); + if (rc != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d corrected-clock reconnect failed (%s)", i + 1, + esp_err_to_name(rc)); + } } else { MQTT_DEBUG_PRINTLN("MQTT%d token re-created, no bounce (broker does not enforce exp)", i + 1); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 0ec28ccc..4450a4f3 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -490,7 +490,10 @@ private: void teardownSlot(int index); // Disconnect the slot's client (keeps the object alive) // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a // no-op on a stopped client). See the definition. - void reconnectSlotClient(int index); + // ESP_OK when the reconnect/start was accepted by the SDK. A local failure + // (uninitialised client, failed config transaction) is not a broker fault and + // must not advance this slot's backoff ladder — see maintainSlotConnection(). + esp_err_t reconnectSlotClient(int index); void maintainSlotConnections(); // Maintain all slot connections (token renewal, reconnect) void maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted); bool createSlotAuthToken(int index); // Create/renew JWT token for a slot From a1a8c19459144e8ab05e5aca07a8fcc49df042fe Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 17:46:20 -0700 Subject: [PATCH 70/93] fix(mqtt): decide slot teardown by SDK lifecycle state, not connectivity (F04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `client->connected()` answers "is the network up", and it was being used for "is there anything to stop". A client resolving DNS, negotiating TLS or waiting after a failed CONNECT reports not-connected, so `teardownSlot()` skipped it: selecting `none` cleared the bridge flags while the SDK task kept running, and when the in-flight handshake completed the callback marked the disabled slot connected and scheduled its status. Each slot now carries the client's SDK lifecycle state (Absent, Configured, Starting, Connected, Disconnected, Stopped, Quarantined) and a generation counter for the logs. The bridge task owns that state, which makes it the authority the callbacks consult: - a CONNECTED event for a slot that is disabled, never started, or already stopped is logged and dropped instead of marking the slot connected; - teardown stops any *live* client, so a mid-handshake client can no longer outlive its configuration. Teardown also gains a reason, because "stop the task" and "close the transport" are different needs: - `Disable` (preset `none`, shutdown) stops the client — the stop is the point; - `Reconfigure` closes the transport with `softDisconnect()` and keeps the esp-mqtt task. Stopping it there would return its 6 KiB stack into the hole the two 16 KiB mbedTLS record buffers just vacated, which is the fork's documented internal-heap fragmentation driver — and per the soak campaign's own conclusion, feeding `softDisconnect()` into the reconfigure path is the fix for it, not serialisation. The campaign closed 2026-08-20, so the reconfigure churn is no longer anyone's measurement lever. A stop that does not complete now quarantines that client: its SDK task was never joined, so it is never reused, never destroyed, and the token buffer its config still points at is never freed. `get mqttN.diag` reports `quarantined`. Two knock-ons this required: - `setupSlot()` starts or reconnects according to the SDK state. `esp_mqtt_client_start()` fails on an already-started client, and now that its result is honoured, a reconfigure that kept the task would otherwise leave the slot permanently unactivated. - the active-slot cap counts resource holders as well as configured ones. It keyed off `initial_connect_done`, which teardown clears, so a started or quarantined client stopped counting against the cap and a board could oversubscribe past the concurrent-TLS limit the cap exists to enforce. --- src/helpers/bridges/MQTTBridge.cpp | 172 ++++++++++++++++++++++++++--- src/helpers/bridges/MQTTBridge.h | 41 ++++++- 2 files changed, 194 insertions(+), 19 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 97ecd867..3e3cdf76 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -463,6 +463,19 @@ void MQTTBridge::applyWifiPowerSave() { #endif } +const char* MQTTBridge::clientStateName(ClientState st) { + switch (st) { + case ClientState::Absent: return "absent"; + case ClientState::Configured: return "configured"; + case ClientState::Starting: return "starting"; + case ClientState::Connected: return "connected"; + case ClientState::Disconnected: return "disconnected"; + case ClientState::Stopped: return "stopped"; + case ClientState::Quarantined: return "quarantined"; + } + return "?"; +} + bool MQTTBridge::stopUnprovenLatched() { return s_stop_unproven; } uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } @@ -561,6 +574,11 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) // is configured but missing a token/IATA/credential, so it was never set up and // has no client yet. Previously reported "disc", which read as a network fault. state = "wait"; + } else if (slot.client && slot.client_state == ClientState::Quarantined) { + // Its stop did not complete, so the SDK task was never joined: the slot is + // out of service for the rest of the boot and its resources are retained + // on purpose. + state = "quarantined"; } else if (!slot.client) { // Ready to connect but the client object could not be allocated. state = "no client"; @@ -1723,10 +1741,24 @@ bool MQTTBridge::ensureSlotClient(int index) { MQTT_DEBUG_PRINTLN("MQTT%d: out of memory allocating client", index + 1); return false; } + slot.client_state = ClientState::Configured; slot.client->setAutoReconnect(false); // we handle reconnect with our own backoff slot.client->onConnect([this, index](bool sessionPresent) { + // A CONNECT started before this slot was disabled or reconfigured can still + // complete afterwards. Accepting it marked a slot connected that the + // operator had switched off, scheduled its status publish, and published + // through the old session (F04). The bridge task owns client_state, so it + // is the authority on whether this event was asked for. + const ClientState st = _slots[index].client_state; + if (!_slots[index].enabled || !(st == ClientState::Starting || st == ClientState::Disconnected)) { + MQTT_DEBUG_PRINTLN("MQTT%d ignoring late CONNECTED (state=%s, gen=%lu, enabled=%d)", + index + 1, clientStateName(st), + (unsigned long)_slots[index].generation, (int)_slots[index].enabled); + return; + } MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); + _slots[index].client_state = ClientState::Connected; _slots[index].connected = true; _slot_force_jwt_mint[index] = false; // NOTE: reconnect_backoff / max_backoff_failures are NOT reset here. @@ -1759,6 +1791,11 @@ bool MQTTBridge::ensureSlotClient(int index) { }); slot.client->onDisconnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d disconnected", index + 1); + // Only a live client's disconnect is news. One arriving for a client we + // already stopped (or quarantined) must not resurrect its state. + if (clientStateIsLive(_slots[index].client_state)) { + _slots[index].client_state = ClientState::Disconnected; + } _slots[index].disconnect_count++; if (_slots[index].first_disconnect_time == 0) { _slots[index].first_disconnect_time = millis(); @@ -1841,20 +1878,42 @@ void MQTTBridge::releaseSlotAuthToken(int index) { void MQTTBridge::destroySlotClients() { for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { MQTTSlot& slot = _slots[i]; - if (slot.client != nullptr) { - if (slot.client->connected()) { - slot.client->disconnect(); + if (slot.client == nullptr) { + releaseSlotAuthToken(i); + continue; + } + + if (slot.client_state == ClientState::Quarantined) { + // A previous stop failed, so this client's SDK task was never joined. + // Deleting it now is the use-after-free F01 is about, and freeing its + // token would pull the buffer out from under a config the task may still + // read. Leak both, deliberately, until the node reboots. + MQTT_DEBUG_PRINTLN("MQTT%d client quarantined - not destroyed, token retained", i + 1); + continue; + } + + if (clientStateIsLive(slot.client_state)) { + const esp_err_t r = slot.client->disconnect(); + if (r != ESP_OK && r != ESP_ERR_TIMEOUT) { + MQTT_DEBUG_PRINTLN("MQTT%d stop FAILED during shutdown (%s) - not destroying", i + 1, + esp_err_to_name(r)); + slot.client_state = ClientState::Quarantined; + continue; } + slot.client_state = ClientState::Stopped; #ifdef ESP_PLATFORM vTaskDelay(pdMS_TO_TICKS(50)); #else delay(50); #endif - delete slot.client; - slot.client = nullptr; } - // Unconditional: only now is the token unreachable from the client's stored - // config, and a token without a client would otherwise leak. + + delete slot.client; + slot.client = nullptr; + slot.client_state = ClientState::Absent; + slot.generation++; + // Only now is the token unreachable from the client's stored config, and a + // token without a client would otherwise leak. releaseSlotAuthToken(i); } } @@ -1862,7 +1921,18 @@ void MQTTBridge::destroySlotClients() { int MQTTBridge::activatedSlotCount() const { int n = 0; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { - if (_slots[i].enabled && _slots[i].initial_connect_done) n++; + const MQTTSlot& s = _slots[i]; + // Two ways to hold a position, and the cap must respect both. The original + // one is the configuration view: enabled and through a successful setup. + // The second is the resource view: a client that has been started still + // owns a task, a socket and an mbedTLS context — which is what the cap + // actually protects — and a quarantined one owns them for the rest of the + // boot. `initial_connect_done` alone missed those, because teardown clears + // it, so a board could oversubscribe past _max_active_slots. + const bool holds_config_position = s.enabled && s.initial_connect_done; + const bool holds_resources = s.client != nullptr && + (clientStateIsLive(s.client_state) || s.client_state == ClientState::Quarantined); + if (holds_config_position || holds_resources) n++; } return n; } @@ -1909,8 +1979,17 @@ bool MQTTBridge::setupSlot(int index) { // them. setCredentials / setServer below overwrite the config fields in place // before connect() restarts the ESP-IDF client. if (slot.initial_connect_done) { - if (slot.client->connected()) { - slot.client->disconnect(); + // Close the transport (keeping the task) if this client is live, for the + // same reason applySlotPreset() does: a handshake in flight against the + // previous endpoint must not complete after the new config is applied. + if (clientStateIsLive(slot.client_state)) { + const esp_err_t r = slot.client->softDisconnect(); + if (r != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d re-apply: transport did not close cleanly (%s)", + index + 1, esp_err_to_name(r)); + } + slot.client_state = ClientState::Disconnected; + slot.generation++; } // Clear TLS verification fields so a stale CA-bundle attach or cert // pointer from a prior preset doesn't override the new one. @@ -2097,13 +2176,20 @@ bool MQTTBridge::setupSlot(int index) { // positions, the reconnect ladder (which is gated on activation) governed it, // and nothing retried the setup. Leaving it unactivated hands it to the // existing deferred-setup retry in maintainSlotConnections() instead. - const esp_err_t connect_result = slot.client->connect(); + // Start or reconnect according to what the SDK client actually is, not what + // the network is doing. esp_mqtt_client_start() fails on an already-started + // client, so a reconfigure that kept the task (TeardownReason::Reconfigure) + // has to reconnect instead — and with connect()'s result now honoured, using + // the wrong one would leave the slot permanently unactivated. + const esp_err_t connect_result = reconnectSlotClient(index); if (connect_result != ESP_OK) { MQTT_DEBUG_PRINTLN("MQTT%d start failed (%s) - will retry", index + 1, esp_err_to_name(connect_result)); slot.last_reconnect_attempt = millis(); return false; } + slot.client_state = ClientState::Starting; + slot.generation++; slot.initial_connect_done = true; return true; } @@ -2112,12 +2198,43 @@ bool MQTTBridge::setupSlot(int index) { // the client object alive so a subsequent setupSlot() can reuse its mbedTLS // context. This is called both on reconfigure (preset change) and at shutdown; // destruction of the underlying client happens once in destroySlotClients(). -void MQTTBridge::teardownSlot(int index) { +void MQTTBridge::teardownSlot(int index, TeardownReason reason) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; - if (slot.client && slot.client->connected()) { - slot.client->disconnect(); + // Gated on the SDK lifecycle state, not on connectivity: a client resolving + // DNS, negotiating TLS or waiting after a failed CONNECT reports + // not-connected, and the old `connected()` gate left exactly those running — + // free to complete their handshake against an endpoint the operator had + // already replaced or switched off (F04). + if (slot.client && clientStateIsLive(slot.client_state)) { + if (reason == TeardownReason::Reconfigure) { + // Close the transport, keep the task. The in-flight handshake cannot + // complete against the old endpoint any more, and the esp-mqtt task's + // 6 KiB stack does not get returned into the hole the mbedTLS record + // buffers just vacated (the documented fragmentation driver). + const esp_err_t r = slot.client->softDisconnect(); + if (r != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d reconfigure: transport did not close cleanly (%s)", + index + 1, esp_err_to_name(r)); + } + slot.client_state = ClientState::Disconnected; + } else { + const esp_err_t r = slot.client->disconnect(); + if (r == ESP_OK || r == ESP_ERR_TIMEOUT) { + // ESP_ERR_TIMEOUT: no DISCONNECTED event, but the stop itself returned + // OK, so the SDK task is joined and the object is safe to reuse. + slot.client_state = ClientState::Stopped; + } else { + // The stop did not complete: the SDK task was not joined. Never touch + // this client again — not to reuse it, not to destroy it, and do not + // free the token buffer its config still points at. + MQTT_DEBUG_PRINTLN("MQTT%d stop FAILED (%s) - client quarantined for this boot", + index + 1, esp_err_to_name(r)); + slot.client_state = ClientState::Quarantined; + } + } + slot.generation++; #ifdef ESP_PLATFORM vTaskDelay(pdMS_TO_TICKS(50)); #else @@ -2154,11 +2271,24 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { MQTTSlot& slot = _slots[index]; if (slot.client == nullptr) return ESP_ERR_INVALID_STATE; + if (slot.client_state == ClientState::Quarantined) { + // Its SDK task was never joined; touching it again is exactly what F01 + // forbids. + return ESP_ERR_INVALID_STATE; + } + + esp_err_t r; if (!slot.client->isStarted()) { MQTT_DEBUG_PRINTLN("MQTT%d start (client was stopped)", index + 1); - return slot.client->connect(); + r = slot.client->connect(); + } else { + r = slot.client->reconnect(); } - return slot.client->reconnect(); + if (r == ESP_OK) { + slot.client_state = ClientState::Starting; + slot.generation++; + } + return r; } @@ -2841,9 +2971,15 @@ void MQTTBridge::applySlotPreset(int slot_index, const char* preset_name) { if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[slot_index]; - teardownSlot(slot_index); + const bool disabling = (strcmp(preset_name, MQTT_PRESET_NONE) == 0 || preset_name[0] == '\0'); + // Selecting `none` must actually stop the client: clearing the bridge flags + // used to leave its task and transport running until full shutdown, and an + // in-flight handshake could still complete and mark the disabled slot + // connected (F04). Every other case keeps the task and only closes the + // transport. + teardownSlot(slot_index, disabling ? TeardownReason::Disable : TeardownReason::Reconfigure); - if (strcmp(preset_name, MQTT_PRESET_NONE) == 0 || preset_name[0] == '\0') { + if (disabling) { slot.enabled = false; slot.preset = nullptr; return; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 4450a4f3..0e9eeb06 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -98,9 +98,38 @@ private: static const uint32_t kNtpMinValidEpoch = 1767225600UL; // 2026-01-01 UTC static const uint32_t kNtpMaxValidEpoch = 4102444800UL; // 2100-01-01 UTC + // What the SDK client is doing, as opposed to whether the network is up. + // `client->connected()` answers the second question and was being used for + // the first: a client resolving DNS, negotiating TLS or waiting after a + // failed CONNECT reports not-connected, so teardown skipped it and left its + // task running (F04). Absent is 0 so a memset-initialised slot is correct. + enum class ClientState : uint8_t { + Absent = 0, // no client object + Configured, // client allocated, never started + Starting, // start/reconnect requested, awaiting CONNECTED + Connected, // CONNECTED received + Disconnected, // started, no session (our reconnect ladder governs it) + Stopped, // stop completed; the SDK task is joined and gone + Quarantined, // stop failed: the SDK task was NOT joined. Never destroy, + // never reuse, never free anything it still points at. + }; + + static const char* clientStateName(ClientState s); + // True while the SDK client has been started and not proven stopped, i.e. + // while it may still own a task, a socket and a TLS context. + static bool clientStateIsLive(ClientState s) { + return s == ClientState::Starting || s == ClientState::Connected || + s == ClientState::Disconnected; + } + // Connection slot - each slot holds one MQTT connection struct MQTTSlot { PsychicMqttClient* client; + ClientState client_state; + // Bumped on every start/stop. Only used for diagnostics and log lines: the + // accept/reject decision for a late callback is made on client_state, which + // the bridge task owns. + uint32_t generation; const MQTTPresetDef* preset; // Points to MQTT_PRESETS[] entry, nullptr for custom/none bool enabled; // true when preset is not "none" bool connected; // Updated in callbacks @@ -487,7 +516,17 @@ private: int activatedSlotCount() const; bool canActivateSlot(int index) const; // force as in destroySlotClients(): skip the unbounded wait, dirty-stop path only. - void teardownSlot(int index); // Disconnect the slot's client (keeps the object alive) + // Why a slot is being torn down. The distinction is not cosmetic: + // - Reconfigure: the slot is about to connect somewhere else, so the + // transport must close (or an in-flight handshake could complete against + // the OLD endpoint) but the esp-mqtt task should stay. Stopping it returns + // its 6 KiB stack into the hole the two 16 KiB mbedTLS record buffers just + // vacated, which is the fork's documented internal-heap fragmentation + // driver; softDisconnect() avoids exactly that. + // - Disable: the slot is going away, so the task and its transport must go + // with it. Here the stop IS the point. + enum class TeardownReason : uint8_t { Reconfigure, Disable }; + void teardownSlot(int index, TeardownReason reason = TeardownReason::Disable); // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a // no-op on a stopped client). See the definition. // ESP_OK when the reconnect/start was accepted by the SDK. A local failure From 7c2e78b94a1af77e34c504a3f123e8d2482eba3f Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 18:03:59 -0700 Subject: [PATCH 71/93] fix(mqtt): write every owned field on every apply; recreate only where needed (F02) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stale-configuration fix, and the reason the four preceding commits went first. `setupSlot()` applied configuration field by field, interleaved with the decisions that produced it, and cleared stale fields only when `slot.initial_connect_done` was set — which `teardownSlot()` clears at the top of every reconfigure. So live reconfiguration always skipped the cleanup. On a 5-slot board this was reproducible in one command: moving a slot from a JWT preset to an anonymous custom endpoint sent the previous configuration's `v1_` username in the CONNECT to the new broker (95-byte CONNECT captured at a throwaway broker on the LAN). The same config after a reboot sent 26 bytes with no username, which is what pinned it to the reconfigure path. That cleanup could not have worked anyway: it nulled the wrapper's pointers, and IDF's `esp_mqtt_set_if_config()` treats NULL as "leave unchanged", so an SDK-held credential survives it. An empty string does overwrite it, and leaves the CONNECT's username flag clear. So configuration is now decided in full first, then applied in one place: - every owned field is written on every apply — server, trust material, username, password — with absent credentials written as "" rather than omitted; - `mqttConfigRecreateDecision()` (previous commit) picks reuse or recreation. Credentials, auth-mode changes and endpoint moves within one scheme reuse the client, because a create/destroy cycle on the reconnect and renewal paths is the fork's documented internal-heap fragmentation driver. A transport (scheme) change, a trust-policy or CA change, and buffer-capacity growth recreate it, because those are the fields a write cannot safely replace; - `recreateSlotClient()` will not destroy a client whose stop was not proven — it quarantines it and fails the setup instead; - the applied configuration is recorded only after the client actually starts. Verified on hardware (Heltec V4, 5 live slots, both paths): - reuse path — a custom JWT slot (639-byte CONNECT carrying `v1_` plus the token) with its audience cleared now sends 26 bytes with no credentials, with no client recreation; - recreate path — `mqtt://` to `wss://` logged `recreating client (transport-change)` and reconnected. Also fixes a cap regression the hardware run caught: `canActivateSlot()` now counts the positions held by the *other* slots. A slot being reconfigured keeps its esp-mqtt task alive, so once the count included live clients it counted itself out of its own position and a full board refused to reconfigure a slot — disabling it instead. Asking "is there room for this slot" rather than "is there room for one more" is also what makes a resource-based count safe. --- src/helpers/bridges/MQTTBridge.cpp | 180 +++++++++++++++++++++++------ src/helpers/bridges/MQTTBridge.h | 20 ++++ 2 files changed, 165 insertions(+), 35 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 3e3cdf76..da0fa302 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -783,6 +783,10 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg _slots[i].last_reconnect_attempt = 0; _slots[i].last_log_time = 0; _slots[i].port = 1883; + _slots[i].client_state = ClientState::Absent; + _slots[i].generation = 0; + _slots[i].applied_config = MqttEffectiveConfig(); + _slots[i].allocated_buffer_size = 0; _slot_reconfigure_pending[i] = false; _slot_force_jwt_mint[i] = false; _status_publish_pending[i] = false; @@ -1742,6 +1746,11 @@ bool MQTTBridge::ensureSlotClient(int index) { return false; } slot.client_state = ClientState::Configured; + // The SDK allocates its buffers in esp_mqtt_client_init() (inside the first + // connect()) from the size set before it, and never resizes them. Record what + // this client will therefore own, so a later config that needs more capacity + // is recognised as needing a new client rather than silently truncating. + slot.allocated_buffer_size = kMqttClientBufferSize; slot.client->setAutoReconnect(false); // we handle reconnect with our own backoff slot.client->onConnect([this, index](bool sessionPresent) { @@ -1918,9 +1927,12 @@ void MQTTBridge::destroySlotClients() { } } -int MQTTBridge::activatedSlotCount() const { +int MQTTBridge::activatedSlotCount() const { return activatedSlotCountExcluding(-1); } + +int MQTTBridge::activatedSlotCountExcluding(int skip) const { int n = 0; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (i == skip) continue; const MQTTSlot& s = _slots[i]; // Two ways to hold a position, and the cap must respect both. The original // one is the configuration view: enabled and through a successful setup. @@ -1939,9 +1951,14 @@ int MQTTBridge::activatedSlotCount() const { bool MQTTBridge::canActivateSlot(int index) const { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; - // Already holding a position (a reconfigure of a live slot) — no new position needed. - if (_slots[index].enabled && _slots[index].initial_connect_done) return true; - return activatedSlotCount() < _max_active_slots; + // Count what the OTHER slots hold. A slot being set up or reconfigured must + // not count itself out of its own position — and it can now hold resources + // while being reconfigured, because a reconfigure keeps the esp-mqtt task + // alive (TeardownReason::Reconfigure). Asking "is there room for this slot" + // rather than "is there room for one more" is also what makes the count safe + // to base on live resources instead of on initial_connect_done, which + // teardown clears. + return activatedSlotCountExcluding(index) < _max_active_slots; } // Returns true only when the slot reached connect(). A false result leaves the slot @@ -1991,22 +2008,13 @@ bool MQTTBridge::setupSlot(int index) { slot.client_state = ClientState::Disconnected; slot.generation++; } - // Clear TLS verification fields so a stale CA-bundle attach or cert - // pointer from a prior preset doesn't override the new one. - esp_mqtt_client_config_t* cfg = slot.client->getMqttConfig(); - #if ESP_IDF_VERSION_MAJOR == 5 - cfg->broker.verification.certificate = nullptr; - cfg->broker.verification.certificate_len = 0; - cfg->broker.verification.crt_bundle_attach = nullptr; - cfg->credentials.username = nullptr; - cfg->credentials.authentication.password = nullptr; - #else - cfg->cert_pem = nullptr; - cfg->cert_len = 0; - cfg->crt_bundle_attach = nullptr; - cfg->username = nullptr; - cfg->password = nullptr; - #endif + // The config fields are NOT cleared here any more. Two reasons: this block + // only ran when initial_connect_done was set, which teardownSlot() clears + // first, so live reconfiguration always skipped it (F02); and nulling a + // wrapper pointer cannot clear an SDK-held string anyway, because IDF's + // esp_mqtt_set_if_config() treats NULL as "leave unchanged". Every owned + // field is written unconditionally below instead, with absent credentials + // written as "" — which does clear them. if (slot.auth_token) slot.auth_token[0] = '\0'; slot.connected = false; slot.token_expires_at = 0; @@ -2032,11 +2040,25 @@ bool MQTTBridge::setupSlot(int index) { #endif #endif + // What this slot should be configured with. Decided first, in full, and + // applied in one place below: field-by-field application interleaved with + // decisions is what let a stale credential survive a reconfigure (F02). + // Every pointer here must outlive the client — esp-mqtt stores the pointer + // and re-reads it whenever a later connect() re-applies the config — so these + // are preset literals in flash, or slot/bridge members, never locals. + const char* cfg_uri = nullptr; + MqttAuth cfg_auth = MqttAuth::None; + const char* cfg_user = nullptr; + const char* cfg_pass = nullptr; + MqttTrust cfg_trust = MqttTrust::Plaintext; + const char* cfg_pem = nullptr; + if (slot.preset) { // Preset-based slot - slot.client->setServer(slot.preset->server_url); + cfg_uri = slot.preset->server_url; if (slot.preset->ca_cert) { - slot.client->setCACert(slot.preset->ca_cert); + cfg_trust = MqttTrust::PemCert; + cfg_pem = slot.preset->ca_cert; } // A JWT slot with no usable token would connect unauthenticated and be rejected. @@ -2048,7 +2070,9 @@ bool MQTTBridge::setupSlot(int index) { slot.last_reconnect_attempt = millis(); return false; } - slot.client->setCredentials(_jwt_username, slot.auth_token); + cfg_auth = MqttAuth::Jwt; + cfg_user = _jwt_username; + cfg_pass = slot.auth_token; } else if (slot.preset->auth_type == MQTT_AUTH_USERPASS) { const char* user = nullptr; const char* pass = slot.preset->userpass_password @@ -2062,7 +2086,9 @@ bool MQTTBridge::setupSlot(int index) { user = slot.username; } if (user && user[0] != '\0' && pass && pass[0] != '\0') { - slot.client->setCredentials(user, pass); + cfg_auth = MqttAuth::UserPass; + cfg_user = user; + cfg_pass = pass; } } } else { @@ -2115,7 +2141,7 @@ bool MQTTBridge::setupSlot(int index) { } snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); } - slot.client->setServer(slot.broker_uri); + cfg_uri = slot.broker_uri; MQTT_DEBUG_PRINTLN("MQTT%d custom broker URI: %s (host='%s', port=%u)", index + 1, slot.broker_uri, slot.host, (unsigned)slot.port); @@ -2124,8 +2150,7 @@ bool MQTTBridge::setupSlot(int index) { // a use-after-free race: connect() launches an async FreeRTOS task, and // calling setCACertBundle() again from a later slot would free the global // crts array while a prior slot's TLS handshake may still be reading it. - bool needs_tls = (strncmp(slot.broker_uri, "mqtts://", 8) == 0 || - strncmp(slot.broker_uri, "wss://", 6) == 0); + const bool needs_tls = mqttTransportIsEncrypted(mqttTransportFromUri(slot.broker_uri)); if (needs_tls) { if (!s_ca_bundle_loaded) { size_t bundle_len = 0; @@ -2138,8 +2163,10 @@ bool MQTTBridge::setupSlot(int index) { if (bundle_len > 0) { MQTT_DEBUG_PRINTLN("MQTT global CA bundle init: embedded bundle (%u bytes)", (unsigned)bundle_len); - // Load the bundle into the global s_crt_bundle via the first client. - // This is a one-time operation; subsequent clients reuse via attachArduinoCACertBundle. + // Load the bundle into the global s_crt_bundle via this client. The + // load is global and one-shot (calling it again would free the crts + // array while another slot's handshake may still be reading it); the + // per-client attach pointer is set uniformly in the apply step below. slot.client->setCACertBundle(rootca_crt_bundle_start, bundle_len); s_ca_bundle_loaded = true; } else { @@ -2151,6 +2178,7 @@ bool MQTTBridge::setupSlot(int index) { } MQTT_DEBUG_PRINTLN("MQTT%d TLS verify: CA bundle %s", index + 1, s_ca_bundle_loaded ? "active" : "unavailable"); + if (s_ca_bundle_loaded) cfg_trust = MqttTrust::Bundle; } else { MQTT_DEBUG_PRINTLN("MQTT%d custom broker uses non-TLS transport", index + 1); } @@ -2163,11 +2191,19 @@ bool MQTTBridge::setupSlot(int index) { slot.last_reconnect_attempt = millis(); return false; } - slot.client->setCredentials(_jwt_username, slot.auth_token); + cfg_auth = MqttAuth::Jwt; + cfg_user = _jwt_username; + cfg_pass = slot.auth_token; MQTT_DEBUG_PRINTLN("MQTT%d custom broker using JWT auth (audience: %s)", index + 1, slot.audience); - } else if (strlen(slot.username) > 0) { - slot.client->setCredentials(slot.username, slot.password); + } else if (slot.username[0] != '\0') { + cfg_auth = MqttAuth::UserPass; + cfg_user = slot.username; + cfg_pass = slot.password; } + // No else: an anonymous endpoint gets empty credentials written to it, so + // whatever the previous configuration left in the SDK is overwritten. This + // exact transition (JWT preset -> anonymous custom) was observed sending + // the old v1_ username to the new broker. } // Activation is now conditional on the client actually starting. A failed @@ -2176,6 +2212,49 @@ bool MQTTBridge::setupSlot(int index) { // positions, the reconnect ladder (which is gated on activation) governed it, // and nothing retried the setup. Leaving it unactivated hands it to the // existing deferred-setup retry in maintainSlotConnections() instead. + // --- one apply, every field ------------------------------------------------ + MqttEffectiveConfig desired; + // Keepalive is recorded as 0: optimizeMqttClientConfig() owns it, it is + // rewritten on every apply, and it plays no part in the recreate decision. + if (!mqttBuildEffectiveConfig(cfg_uri, cfg_auth, cfg_user, cfg_pass, cfg_trust, cfg_pem, + kMqttClientBufferSize, 0, &desired)) { + MQTT_DEBUG_PRINTLN("MQTT%d: unusable broker URI '%s' - not connecting", index + 1, + cfg_uri ? cfg_uri : "(none)"); + slot.last_reconnect_attempt = millis(); + return false; + } + + // Recreate only where a field cannot be overwritten: a transport (scheme) + // change, a trust-policy or CA change, or growth past the allocated buffer + // capacity. Everything else — credentials, auth mode, an endpoint move within + // one scheme — is reconfigured in place, because a client create/destroy cycle + // on the reconnect/renewal path is the documented fragmentation driver. + const MqttRecreateDecision decision = + mqttConfigRecreateDecision(slot.applied_config, desired, slot.allocated_buffer_size); + if (decision.recreate) { + MQTT_DEBUG_PRINTLN("MQTT%d recreating client (%s)", index + 1, decision.reason); + if (!recreateSlotClient(index)) { + slot.last_reconnect_attempt = millis(); + return false; + } + } + + // Write every owned field, in one place, whether or not it changed. Absent + // credentials are written as "" rather than left alone: NULL means "leave + // unchanged" to esp_mqtt_set_config(), while an empty string clears the + // stored value and leaves the CONNECT's username flag clear. + slot.client->setServer(cfg_uri); + if (desired.trust == MqttTrust::PemCert) { + slot.client->setCACert(desired.pem); + slot.client->attachArduinoCACertBundle(false); + } else if (desired.trust == MqttTrust::Bundle) { + slot.client->attachArduinoCACertBundle(true); + } else { + slot.client->attachArduinoCACertBundle(false); + } + slot.client->setCredentials(mqttFieldOrEmpty(desired.username), + mqttFieldOrEmpty(desired.password)); + // Start or reconnect according to what the SDK client actually is, not what // the network is doing. esp_mqtt_client_start() fails on an already-started // client, so a reconfigure that kept the task (TeardownReason::Reconfigure) @@ -2188,12 +2267,43 @@ bool MQTTBridge::setupSlot(int index) { slot.last_reconnect_attempt = millis(); return false; } - slot.client_state = ClientState::Starting; - slot.generation++; + // reconnectSlotClient() moved the state to Starting and bumped the generation. + // Record what the client is now configured with, so the next apply can tell + // an in-place change from a structural one. + slot.applied_config = desired; slot.initial_connect_done = true; return true; } +// Destroy and re-create this slot's client, for the configuration changes that +// cannot be applied to a live SDK client (see mqttConfigRecreateDecision). +// +// The stop must be PROVEN before the object is freed: a client whose +// esp_mqtt_client_stop() did not return OK still has a task, and deleting it +// then is the use-after-free F01 is about. Such a client is quarantined and this +// returns false — the slot stays unactivated and its resources are retained +// until the node reboots, rather than being freed under a live task. +bool MQTTBridge::recreateSlotClient(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + if (slot.client == nullptr) return ensureSlotClient(index); + + teardownSlot(index, TeardownReason::Disable); // stop, do not just close the transport + if (slot.client_state == ClientState::Quarantined) { + MQTT_DEBUG_PRINTLN("MQTT%d cannot recreate: previous stop unproven", index + 1); + return false; + } + + delete slot.client; + slot.client = nullptr; + slot.client_state = ClientState::Absent; + slot.applied_config = MqttEffectiveConfig(); // nothing is applied to a client that does not exist + slot.allocated_buffer_size = 0; + // The token buffer survives: the new client is configured from it below, and + // its lifetime is the slot's, not the client's (see MQTTSlot::auth_token). + return ensureSlotClient(index); +} + // Disconnect the slot's MQTT client and clear per-connection state, but leave // the client object alive so a subsequent setupSlot() can reuse its mbedTLS // context. This is called both on reconfigure (preset change) and at shutdown; @@ -4877,7 +4987,7 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ // The cost is 384 bytes per client on a non-PSRAM board (at most 2 active // slots there), against a handshake that needs 16 KB of *contiguous* internal // DRAM — noise, and it buys the removal of a recreate case. - static const int MQTT_CLIENT_BUFFER_SIZE = 896; + static const int MQTT_CLIENT_BUFFER_SIZE = kMqttClientBufferSize; (void)needs_large_buffer; // kept: callers still express the intent client->setBufferSize(MQTT_CLIENT_BUFFER_SIZE); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 0e9eeb06..6c81cfc2 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -11,6 +11,7 @@ #include "helpers/MQTTPresets.h" #include "helpers/MQTTLifecycle.h" #include "helpers/AlertFaultPolicy.h" +#include "helpers/MQTTEffectiveConfig.h" #include #ifdef WITH_SNMP @@ -91,6 +92,12 @@ public: private: static const size_t AUTH_TOKEN_SIZE = 768; + // Every client gets buffers sized for a JWT CONNECT (frame + 768-byte token), + // on every board: the SDK fixes the allocation at client init and does not + // resize it, so a per-slot size made the desired and allocated capacities + // disagree (F03). + static const uint16_t kMqttClientBufferSize = 896; + // NTP acceptance bounds. A reply outside them is rejected outright rather // than allowed to set the clock, the RTC and every JWT minted afterwards. static const uint16_t kNtpPort = 123; @@ -126,6 +133,12 @@ private: struct MQTTSlot { PsychicMqttClient* client; ClientState client_state; + // What this client was last successfully configured with, and the buffer + // capacity it actually owns (the SDK fixes that at client init). Together + // they answer "can the next configuration be applied in place?" — the + // question `initial_connect_done` was standing in for, wrongly. + MqttEffectiveConfig applied_config; + uint16_t allocated_buffer_size; // Bumped on every start/stop. Only used for diagnostics and log lines: the // accept/reject decision for a late callback is made on client_state, which // the bridge task owns. @@ -501,6 +514,10 @@ private: // This avoids delete/new cycles that shed ~40 KB of mbedTLS buffers per // reconfigure and fragment the internal heap on non-PSRAM boards. bool ensureSlotClient(int index); // Allocate this slot's persistent client + callbacks on first use + // Stop, destroy and re-allocate this slot's client. Only for configuration + // changes that cannot be applied to a live client, and only after the stop is + // proven — false means the client is quarantined and must not be reused. + bool recreateSlotClient(int index); bool ensureSlotAuthToken(int index); // Allocate this slot's JWT token buffer on first token creation void releaseSlotAuthToken(int index);// Free the token buffer (only with the client — see MQTTSlot) // No force variant: the only caller that ever passed one was the dirty-stop @@ -514,6 +531,9 @@ private: // setup-retry path, and live reconfigure all gate on these so the cap cannot be // exceeded by one route while another enforces it. int activatedSlotCount() const; + // Positions held by every slot except `skip` (-1 for none). canActivateSlot() + // excludes the candidate so a live reconfigure cannot fail its own cap check. + int activatedSlotCountExcluding(int skip) const; bool canActivateSlot(int index) const; // force as in destroySlotClients(): skip the unbounded wait, dirty-stop path only. // Why a slot is being torn down. The distinction is not cosmetic: From b4daf8ca43194dad8021a9f94ebd1619e6a23fe3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 20:03:13 -0700 Subject: [PATCH 72/93] fix(mqtt): close the review's P1 and P2 findings on the F01/F04/F06 work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the branch found one merge-blocking lifecycle hole and three correctness gaps where the implementation stopped short of contracts the design had already written down. All four are real; each was confirmed against the source (two of them against hardware) before anything changed. **P1 — a quarantined client could still produce a clean bridge stop.** The cooperative teardown set `_teardown_complete` unconditionally, so a slot whose `esp_mqtt_client_stop()` had not completed — deliberately skipped by `destroySlotClients()` and marked Quarantined — still let the trampoline publish the acknowledgement. The owner then freed the queue and buffers and allowed a restart while that SDK task might still be running: exactly the ownership ambiguity StopUnproven exists to remove. The ack is now withheld unless EVERY client is proven stopped, so one unproven client leaves the whole bridge unproven. The rule lives in MQTTClientState.h (`mqttStopMayBeAcknowledged`) with host tests, alongside the state predicates moved out of the bridge. **P2 — the F04 protection did not cover a client that was still connecting.** `softDisconnect()` returns ESP_OK immediately when the client is not connected, so for a slot mid-DNS/TLS/CONNECT it cancelled nothing: the attempt ran on and its CONNECTED event arrived after the new configuration was applied, and with callbacks registered once per client and esp-mqtt events carrying no generation, nothing could tell it from the new attempt's. A reconfigure that lands on a `Starting` client now stops it, joining its SDK task, before applying the new configuration. A Connected client still takes the cheap softDisconnect path, which is where the fragmentation argument applies. One helper (`closeLiveClientForReconfigure`) so the two call sites cannot drift. **P2 — a failed renewal bounce still advanced the effective expiry.** Minting updates `token_expires_at` immediately and the renewal decision read it, so a bounce that failed looked complete: the next pass saw a fresh future expiry and never retried, and clearing `last_token_renewal` re-armed nothing. Slots now carry `applied_token_expires_at` — the expiry of the credential the CONNECTION is using — which only advances when a connect or reconnect has carried it. A failed bounce leaves it on the old credential, so the renewal stays due. **P2 — config-committed and start-accepted were conflated.** `connect()` returned one result for both, so a start that failed after the configuration had committed left `applied_config` describing the previous configuration, and the next recreate-or-reuse decision could reuse a client whose trust policy was not the one it believed. `applyConfig()` is now its own wrapper operation; `applied_config` records the commit, activation records the start. The reconnect ladder resets there too rather than in `teardownSlot()` — the old endpoint's history still applies until a replacement configuration actually commits. Two of my own bugs surfaced on hardware while testing this, both fixed here: - `recreateSlotClient()` called the full `teardownSlot()`, which cleared `broker_uri`, the just-minted token and both expiries out from under a configuration that had already been decided, so a recreate handed the SDK an empty URI and an empty token. It now stops the client and swaps the object, touching nothing else, and the apply step refuses to configure a URI that changed under it rather than passing it on. - `stopSlotClient()` quarantined on any non-OK result, but `ESP_FAIL` from `esp_mqtt_client_stop()` means "client is in invalid state", i.e. not started: there was no task to join, the safest state there is. It was observed quarantining healthy clients on hardware. The case that genuinely cannot be proven is a stop that never RETURNS, which cannot surface here at all — it hangs the task, which is what the bridge-level timeout contains. Hardware (Heltec V4, 5 live slots): a reconfigure landing on a connecting client logs `reconfigure during connect - stopping to cancel the attempt`; a broker holding the CONNACK sees the client close the socket and the disabled slot never connects; `wss`→`mqtt`→`wss` recreate cycles reconnect each way; a blackholed endpoint recovers. 499/499 native tests, four envs clean. --- .../src/PsychicMqttClient.cpp | 110 ++++---- lib/PsychicMqttClient/src/PsychicMqttClient.h | 16 ++ src/helpers/MQTTClientState.h | 82 ++++++ src/helpers/bridges/MQTTBridge.cpp | 263 +++++++++++++----- src/helpers/bridges/MQTTBridge.h | 42 ++- .../test_mqtt_client_state.cpp | 89 ++++++ 6 files changed, 458 insertions(+), 144 deletions(-) create mode 100644 src/helpers/MQTTClientState.h create mode 100644 test/test_mqtt_client_state/test_mqtt_client_state.cpp diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index cec11177..ccd11341 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -371,21 +371,59 @@ bool PsychicMqttClient::connected() return _connected; } -esp_err_t PsychicMqttClient::connect() +esp_err_t PsychicMqttClient::applyConfig() { #if ESP_IDF_VERSION_MAJOR == 5 if (_mqtt_cfg.broker.address.uri == nullptr) - { - ESP_LOGE(TAG, "MQTT URI not set."); - return ESP_ERR_INVALID_STATE; - } - int desired_buffer = _mqtt_cfg.buffer.size > 0 ? _mqtt_cfg.buffer.size : 1024; #else if (_mqtt_cfg.uri == nullptr) +#endif { ESP_LOGE(TAG, "MQTT URI not set."); return ESP_ERR_INVALID_STATE; } + + if (_client == nullptr) + { + // esp_mqtt_client_init() takes the whole configuration, including the + // buffer sizes it allocates once and never resizes. + _client = esp_mqtt_client_init(&_mqtt_cfg); + if (_client == nullptr) + { + ESP_LOGE(TAG, "esp_mqtt_client_init failed"); + return ESP_ERR_NO_MEM; + } + // Register event handler only once when client is first created + // to avoid memory leak from repeated registrations + esp_mqtt_client_register_event(_client, MQTT_EVENT_ANY, _onMqttEventStatic, this); + _config_dirty = false; + return ESP_OK; + } + + if (!_config_dirty) + { + ESP_LOGD(TAG, "applyConfig(): mqtt config unchanged, skipping set_config"); + return ESP_OK; + } + + esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); + if (cfg_result != ESP_OK) + { + // Leave _config_dirty set so the next attempt retries the whole + // transaction rather than starting on a partly-updated configuration. + ESP_LOGE(TAG, "applyConfig(): failed to apply mqtt config: %s", esp_err_to_name(cfg_result)); + return cfg_result; + } + _config_dirty = false; + ESP_LOGD(TAG, "applyConfig(): applied mqtt config update"); + return ESP_OK; +} + +esp_err_t PsychicMqttClient::connect() +{ +#if ESP_IDF_VERSION_MAJOR == 5 + int desired_buffer = _mqtt_cfg.buffer.size > 0 ? _mqtt_cfg.buffer.size : 1024; +#else int desired_buffer = _mqtt_cfg.buffer_size > 0 ? _mqtt_cfg.buffer_size : 1024; #endif @@ -404,38 +442,11 @@ esp_err_t PsychicMqttClient::connect() } } - if (_client == nullptr) - { - _client = esp_mqtt_client_init(&_mqtt_cfg); - if (_client == nullptr) - { - ESP_LOGE(TAG, "esp_mqtt_client_init failed"); - return ESP_ERR_NO_MEM; - } - // Register event handler only once when client is first created - // to avoid memory leak from repeated registrations - esp_mqtt_client_register_event(_client, MQTT_EVENT_ANY, _onMqttEventStatic, this); - _config_dirty = false; - } - else if (_config_dirty) - { - // A failed config update must not be followed by a start: the client - // would come up on a partly-updated configuration, which is how a slot - // connects with the previous broker's credentials. Leave _config_dirty - // set so the next attempt retries the whole transaction. - esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); - if (cfg_result != ESP_OK) - { - ESP_LOGE(TAG, "connect(): failed to apply mqtt config: %s", esp_err_to_name(cfg_result)); - return cfg_result; - } - _config_dirty = false; - ESP_LOGD(TAG, "connect(): applied mqtt config update"); - } - else - { - ESP_LOGD(TAG, "connect(): mqtt config unchanged, skipping set_config"); - } + // A failed config transaction must not be followed by a start: the client + // would come up on a partly-updated configuration, which is how a slot + // connects with the previous broker's credentials. + esp_err_t cfg_result = applyConfig(); + if (cfg_result != ESP_OK) return cfg_result; esp_err_t start_result = esp_mqtt_client_start(_client); if (start_result == ESP_OK) @@ -458,24 +469,11 @@ esp_err_t PsychicMqttClient::reconnect() ESP_LOGW(TAG, "MQTT client not initialized, cannot reconnect."); return ESP_ERR_INVALID_STATE; } - if (_config_dirty) - { - // Apply config only when mutating setters changed _mqtt_cfg. A failure - // aborts the reconnect: reconnecting on the previous config was how a - // renewed token silently failed to reach the connection. - esp_err_t cfg_result = esp_mqtt_set_config(_client, &_mqtt_cfg); - if (cfg_result != ESP_OK) - { - ESP_LOGE(TAG, "reconnect(): failed to apply mqtt config: %s", esp_err_to_name(cfg_result)); - return cfg_result; - } - _config_dirty = false; - ESP_LOGD(TAG, "reconnect(): applied mqtt config update"); - } - else - { - ESP_LOGD(TAG, "reconnect(): mqtt config unchanged, skipping set_config"); - } + // A failed config update aborts the reconnect: reconnecting on the previous + // config was how a renewed token silently failed to reach the connection. + esp_err_t cfg_result = applyConfig(); + if (cfg_result != ESP_OK) return cfg_result; + esp_err_t r = esp_mqtt_client_reconnect(_client); if (r != ESP_OK) { diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index 5d214679..93b85a43 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -345,6 +345,22 @@ public: */ bool connected(); + /** + * @brief Commits the pending configuration to the SDK client, creating it + * if it does not exist yet, WITHOUT starting it. + * + * Exists so a caller can tell "the configuration is now what I asked for" + * from "the client started": esp_mqtt_client_start() can fail after the + * configuration has been committed, and a caller that cannot distinguish + * those records a stale view of what the SDK actually holds. + * + * @return ESP_OK when the SDK client holds this configuration. + * ESP_ERR_INVALID_STATE if no URI is set, ESP_ERR_NO_MEM if the + * client could not be allocated, or the error from + * esp_mqtt_set_config() (which leaves the update pending). + */ + esp_err_t applyConfig(); + /** * @brief Connects the MQTT client to the server. * diff --git a/src/helpers/MQTTClientState.h b/src/helpers/MQTTClientState.h new file mode 100644 index 00000000..dd1fee44 --- /dev/null +++ b/src/helpers/MQTTClientState.h @@ -0,0 +1,82 @@ +#pragma once + +#include + +// What a slot's MQTT client is doing, as opposed to whether its network is up, +// and the two decisions that hang off it. +// +// Pure logic: no Arduino, esp-mqtt or bridge dependency, so the shutdown +// contract below is a host test rather than a hardware run. +// +// `client->connected()` answers "is the network up" and was being used for "is +// there anything to stop": a client resolving DNS, negotiating TLS or waiting +// after a failed CONNECT reports not-connected, so teardown skipped it and left +// its task running (F04). + +enum class MqttClientState : uint8_t { + Absent = 0, // no client object + Configured, // client allocated, never started + Starting, // start/reconnect requested, awaiting CONNECTED + Connected, // CONNECTED received + Disconnected, // started, no session (our reconnect ladder governs it) + Stopped, // stop completed; the SDK task is joined and gone + Quarantined, // stop failed: the SDK task was NOT joined. Never destroy, + // never reuse, never free anything it still points at. +}; + +static inline const char* mqttClientStateName(MqttClientState s) { + switch (s) { + case MqttClientState::Absent: return "absent"; + case MqttClientState::Configured: return "configured"; + case MqttClientState::Starting: return "starting"; + case MqttClientState::Connected: return "connected"; + case MqttClientState::Disconnected: return "disconnected"; + case MqttClientState::Stopped: return "stopped"; + case MqttClientState::Quarantined: return "quarantined"; + } + return "?"; +} + +// The client has been started and not proven stopped: it may still own a task, +// a socket and a TLS context. Quarantined is deliberately NOT live — it is +// worse: it may own them and we can never find out. +static inline bool mqttClientStateIsLive(MqttClientState s) { + return s == MqttClientState::Starting || s == MqttClientState::Connected || + s == MqttClientState::Disconnected; +} + +// A connection attempt is in flight, so an event for it can still arrive. +// Closing the transport is not enough here: PsychicMqttClient::softDisconnect() +// returns immediately when the client is not yet connected, which means an +// in-progress DNS/TLS/CONNECT is left to complete on its own. Cancelling one +// requires a real stop. +static inline bool mqttClientStateHasAttemptInFlight(MqttClientState s) { + return s == MqttClientState::Starting; +} + +// The SDK task is known to be gone (or never existed), so the object may be +// destroyed and anything it pointed at may be freed. +static inline bool mqttClientStateIsProvenStopped(MqttClientState s) { + return s == MqttClientState::Absent || s == MqttClientState::Configured || + s == MqttClientState::Stopped; +} + +// The bridge's cooperative stop may only be acknowledged when EVERY client is +// proven stopped. +// +// This is the invariant the whole shutdown contract rests on: the owner treats +// the acknowledgement as proof that the task destroyed its clients, and only +// then frees the queue and buffers and allows a restart. One client whose +// esp_mqtt_client_stop() never completed makes that false — its SDK task may +// still be running — so the acknowledgement must be withheld and the bridge +// left in its unproven state, however many other slots stopped cleanly. +static inline bool mqttStopMayBeAcknowledged(const MqttClientState* states, int count) { + // An empty set is trivially proven (no slots, nothing to stop); a null array + // with a nonzero count is a caller bug and must not read as proof. + if (count <= 0) return true; + if (states == nullptr) return false; + for (int i = 0; i < count; i++) { + if (!mqttClientStateIsProvenStopped(states[i])) return false; + } + return true; +} diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index da0fa302..9e176c4b 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -463,19 +463,6 @@ void MQTTBridge::applyWifiPowerSave() { #endif } -const char* MQTTBridge::clientStateName(ClientState st) { - switch (st) { - case ClientState::Absent: return "absent"; - case ClientState::Configured: return "configured"; - case ClientState::Starting: return "starting"; - case ClientState::Connected: return "connected"; - case ClientState::Disconnected: return "disconnected"; - case ClientState::Stopped: return "stopped"; - case ClientState::Quarantined: return "quarantined"; - } - return "?"; -} - bool MQTTBridge::stopUnprovenLatched() { return s_stop_unproven; } uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } @@ -1444,6 +1431,29 @@ void MQTTBridge::mqttTaskLoop() { teardownSlot(i); } destroySlotClients(); + + // The acknowledgement means "every client is destroyed, nothing I own is + // still running". One client whose esp_mqtt_client_stop() never completed + // makes that false — destroySlotClients() deliberately skipped it and its + // SDK task may still be executing — so the ack must be WITHHELD, whatever + // the other slots did. Publishing it anyway would tell the owner to free + // the queue and buffers and allow a restart while that task lives, which + // is the ownership ambiguity this whole contract exists to remove. + MqttClientState states[RUNTIME_MQTT_SLOTS]; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) states[i] = _slots[i].client_state; + if (!mqttStopMayBeAcknowledged(states, RUNTIME_MQTT_SLOTS)) { + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (!mqttClientStateIsProvenStopped(states[i])) { + MQTT_DEBUG_PRINTLN("MQTT%d stop unproven (%s) - withholding the bridge stop ack", + i + 1, mqttClientStateName(states[i])); + } + } + // Leave _teardown_complete false: the trampoline publishes no ack, the + // owner's stop times out into StopUnproven, and nothing is released or + // restarted for the rest of this boot. + return; + } + // Record that the ordered teardown finished, but do NOT publish the ack // here: the owner treats the ack as permission to free everything this // task can reach, and between this point and vTaskDelete(nullptr) the task @@ -1881,6 +1891,7 @@ void MQTTBridge::releaseSlotAuthToken(int index) { slot.auth_token = static_cast( MQTTRuntimeBufferLifecycle::release(slot.auth_token, psram_free)); slot.token_expires_at = 0; + slot.applied_token_expires_at = 0; slot.last_token_renewal = 0; } @@ -1902,14 +1913,11 @@ void MQTTBridge::destroySlotClients() { } if (clientStateIsLive(slot.client_state)) { - const esp_err_t r = slot.client->disconnect(); - if (r != ESP_OK && r != ESP_ERR_TIMEOUT) { - MQTT_DEBUG_PRINTLN("MQTT%d stop FAILED during shutdown (%s) - not destroying", i + 1, - esp_err_to_name(r)); - slot.client_state = ClientState::Quarantined; + stopSlotClient(i); + if (slot.client_state == ClientState::Quarantined) { + MQTT_DEBUG_PRINTLN("MQTT%d not destroyed: stop unproven during shutdown", i + 1); continue; } - slot.client_state = ClientState::Stopped; #ifdef ESP_PLATFORM vTaskDelay(pdMS_TO_TICKS(50)); #else @@ -1996,17 +2004,17 @@ bool MQTTBridge::setupSlot(int index) { // them. setCredentials / setServer below overwrite the config fields in place // before connect() restarts the ESP-IDF client. if (slot.initial_connect_done) { - // Close the transport (keeping the task) if this client is live, for the - // same reason applySlotPreset() does: a handshake in flight against the - // previous endpoint must not complete after the new config is applied. + // Same rule as applySlotPreset(): a handshake in flight against the previous + // endpoint must not survive the new configuration, and only a stop can + // cancel one. One helper so the two paths cannot drift apart. if (clientStateIsLive(slot.client_state)) { - const esp_err_t r = slot.client->softDisconnect(); - if (r != ESP_OK) { - MQTT_DEBUG_PRINTLN("MQTT%d re-apply: transport did not close cleanly (%s)", - index + 1, esp_err_to_name(r)); - } - slot.client_state = ClientState::Disconnected; + closeLiveClientForReconfigure(index); slot.generation++; + if (slot.client_state == ClientState::Quarantined) { + MQTT_DEBUG_PRINTLN("MQTT%d: cannot re-apply, stop unproven", index + 1); + slot.last_reconnect_attempt = millis(); + return false; + } } // The config fields are NOT cleared here any more. Two reasons: this block // only ran when initial_connect_done was set, which teardownSlot() clears @@ -2018,11 +2026,10 @@ bool MQTTBridge::setupSlot(int index) { if (slot.auth_token) slot.auth_token[0] = '\0'; slot.connected = false; slot.token_expires_at = 0; + slot.applied_token_expires_at = 0; slot.last_token_renewal = 0; - slot.reconnect_backoff = 0; - slot.max_backoff_failures = 0; - slot.circuit_breaker_tripped = false; slot.last_reconnect_attempt = 0; + // Ladder reset happens after the new configuration commits, below. // The refusal that set this belonged to the credentials being cleared here. _slot_force_jwt_mint[index] = false; } @@ -2243,6 +2250,18 @@ bool MQTTBridge::setupSlot(int index) { // credentials are written as "" rather than left alone: NULL means "leave // unchanged" to esp_mqtt_set_config(), while an empty string clears the // stored value and leaves the CONNECT's username flag clear. + // cfg_uri points at storage the slot owns (slot.broker_uri) or at flash + // (preset->server_url), and esp-mqtt keeps the pointer, so it must still hold + // what `desired` was built from. This caught a real bug: an earlier version of + // recreateSlotClient() ran the full slot teardown, which cleared broker_uri + // out from under the configuration that had already been decided, and the SDK + // was then handed an empty URI. + if (cfg_uri == nullptr || strcmp(cfg_uri, desired.uri) != 0) { + MQTT_DEBUG_PRINTLN("MQTT%d: broker URI changed under the config apply - not connecting", + index + 1); + slot.last_reconnect_attempt = millis(); + return false; + } slot.client->setServer(cfg_uri); if (desired.trust == MqttTrust::PemCert) { slot.client->setCACert(desired.pem); @@ -2255,6 +2274,29 @@ bool MQTTBridge::setupSlot(int index) { slot.client->setCredentials(mqttFieldOrEmpty(desired.username), mqttFieldOrEmpty(desired.password)); + // Commit the configuration as its own transaction, and record it as applied + // the moment it commits — not after the client starts. The SDK really does + // hold this configuration once set_config/init returns OK, so a start that + // fails afterwards must not leave the bridge believing the PREVIOUS config is + // applied: the next recreate-or-reuse decision would compare against it and + // could reuse a client whose trust policy is not the one it thinks. + const esp_err_t config_result = slot.client->applyConfig(); + if (config_result != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d config transaction failed (%s) - will retry", index + 1, + esp_err_to_name(config_result)); + slot.last_reconnect_attempt = millis(); + return false; + } + slot.applied_config = desired; + + // The new endpoint's history starts here, now that the configuration it + // belongs to is committed. Resetting the ladder in teardownSlot() cleared it + // before there was any replacement config — so a reconfigure that then failed + // to commit lost the backoff state that still applied to the old endpoint. + slot.reconnect_backoff = 0; + slot.max_backoff_failures = 0; + slot.circuit_breaker_tripped = false; + // Start or reconnect according to what the SDK client actually is, not what // the network is doing. esp_mqtt_client_start() fails on an already-started // client, so a reconfigure that kept the task (TeardownReason::Reconfigure) @@ -2268,9 +2310,8 @@ bool MQTTBridge::setupSlot(int index) { return false; } // reconnectSlotClient() moved the state to Starting and bumped the generation. - // Record what the client is now configured with, so the next apply can tell - // an in-place change from a structural one. - slot.applied_config = desired; + // applied_config was recorded at the commit above; activation records that the + // client also started. slot.initial_connect_done = true; return true; } @@ -2288,11 +2329,25 @@ bool MQTTBridge::recreateSlotClient(int index) { MQTTSlot& slot = _slots[index]; if (slot.client == nullptr) return ensureSlotClient(index); - teardownSlot(index, TeardownReason::Disable); // stop, do not just close the transport + // Stop the client only. NOT teardownSlot(): that is the "this slot is going + // away" path, and it clears the very state the in-flight setup is holding — + // broker_uri (which cfg_uri points at), the token just minted for this + // configuration, and both expiries. Recreation swaps the client object under + // a configuration that has already been decided; it must not disturb it. + if (clientStateIsLive(slot.client_state)) { + stopSlotClient(index); + slot.generation++; + #ifdef ESP_PLATFORM + vTaskDelay(pdMS_TO_TICKS(50)); + #else + delay(50); + #endif + } if (slot.client_state == ClientState::Quarantined) { MQTT_DEBUG_PRINTLN("MQTT%d cannot recreate: previous stop unproven", index + 1); return false; } + slot.connected = false; delete slot.client; slot.client = nullptr; @@ -2308,6 +2363,80 @@ bool MQTTBridge::recreateSlotClient(int index) { // the client object alive so a subsequent setupSlot() can reuse its mbedTLS // context. This is called both on reconfigure (preset change) and at shutdown; // destruction of the underlying client happens once in destroySlotClients(). +// Close a live client for a reconfigure: cheap where that is safe, a real stop +// where it is not. +// +// softDisconnect() cannot cancel a connection that has not completed. +// esp_mqtt_client_disconnect() acts on a live session, and the wrapper returns +// ESP_OK immediately when the client is not connected — so for a client +// mid-DNS/TLS/CONNECT nothing is closed and nothing is cancelled. That attempt +// would run to completion against the OLD endpoint and deliver its CONNECTED +// event after the new configuration was applied, indistinguishable from the new +// attempt's (F04): esp-mqtt gives events no generation of their own, and the +// callbacks are registered once per client, so there is nothing in the event to +// tell them apart. The only way to separate them is to make sure the old +// attempt is dead first, which means stopping the client and joining its task. +// +// That costs a stop/start cycle, but only when an operator reconfigures a slot +// *while it is connecting*. A Connected client still takes the cheap route, +// which is where the fragmentation argument actually applies. +void MQTTBridge::closeLiveClientForReconfigure(int index) { + MQTTSlot& slot = _slots[index]; + if (mqttClientStateHasAttemptInFlight(slot.client_state)) { + MQTT_DEBUG_PRINTLN("MQTT%d reconfigure during connect - stopping to cancel the attempt", + index + 1); + stopSlotClient(index); + return; + } + const esp_err_t r = slot.client->softDisconnect(); + if (r != ESP_OK) { + MQTT_DEBUG_PRINTLN("MQTT%d reconfigure: transport did not close cleanly (%s)", + index + 1, esp_err_to_name(r)); + } + slot.client_state = ClientState::Disconnected; +} + +// Stop a live client and record whether the stop was proven. ESP_ERR_TIMEOUT +// means no DISCONNECTED event arrived but esp_mqtt_client_stop() itself +// returned OK, so the SDK task is joined and the object is safe to reuse; any +// other error means it was NOT joined, and that client is out of service for +// the rest of the boot. +void MQTTBridge::stopSlotClient(int index) { + MQTTSlot& slot = _slots[index]; + const esp_err_t r = slot.client->disconnect(); + + // What the SDK's results actually mean here (mqtt_client.h documents + // esp_mqtt_client_stop as "ESP_OK on success, ESP_ERR_INVALID_ARG on wrong + // initialization, ESP_FAIL if client is in invalid state"): + // + // ESP_OK the SDK task was joined. + // ESP_ERR_TIMEOUT our own wrapper's code for "no DISCONNECTED event, but + // the stop itself returned OK" — the task is still joined. + // ESP_FAIL the client was not started. There was no task to join, + // which is the safest state of all. Treating this as a + // failure quarantined perfectly healthy clients — observed + // on hardware when a reconfigure landed on a slot whose + // connect attempt had already failed. + // + // The case that genuinely cannot be proven is a stop that never RETURNS: the + // SDK waits on its API mutex and the task's stopped event without a bound. + // That does not surface here at all — it hangs this task, which is exactly + // what the bridge-level stop timeout (StopUnproven) exists to contain. So + // with IDF 4.4's contract the branch below should be unreachable; it stays + // because acting on an unexpected result by not touching the client again is + // the only safe response if that ever changes. + if (r == ESP_OK || r == ESP_ERR_TIMEOUT || r == ESP_FAIL) { + if (r == ESP_FAIL) { + MQTT_DEBUG_PRINTLN("MQTT%d stop: client was not started (nothing to join)", index + 1); + } + slot.client_state = ClientState::Stopped; + return; + } + MQTT_DEBUG_PRINTLN("MQTT%d stop returned %s - client quarantined for this boot", + index + 1, esp_err_to_name(r)); + slot.client_state = ClientState::Quarantined; +} + void MQTTBridge::teardownSlot(int index, TeardownReason reason) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; @@ -2319,30 +2448,9 @@ void MQTTBridge::teardownSlot(int index, TeardownReason reason) { // already replaced or switched off (F04). if (slot.client && clientStateIsLive(slot.client_state)) { if (reason == TeardownReason::Reconfigure) { - // Close the transport, keep the task. The in-flight handshake cannot - // complete against the old endpoint any more, and the esp-mqtt task's - // 6 KiB stack does not get returned into the hole the mbedTLS record - // buffers just vacated (the documented fragmentation driver). - const esp_err_t r = slot.client->softDisconnect(); - if (r != ESP_OK) { - MQTT_DEBUG_PRINTLN("MQTT%d reconfigure: transport did not close cleanly (%s)", - index + 1, esp_err_to_name(r)); - } - slot.client_state = ClientState::Disconnected; + closeLiveClientForReconfigure(index); } else { - const esp_err_t r = slot.client->disconnect(); - if (r == ESP_OK || r == ESP_ERR_TIMEOUT) { - // ESP_ERR_TIMEOUT: no DISCONNECTED event, but the stop itself returned - // OK, so the SDK task is joined and the object is safe to reuse. - slot.client_state = ClientState::Stopped; - } else { - // The stop did not complete: the SDK task was not joined. Never touch - // this client again — not to reuse it, not to destroy it, and do not - // free the token buffer its config still points at. - MQTT_DEBUG_PRINTLN("MQTT%d stop FAILED (%s) - client quarantined for this boot", - index + 1, esp_err_to_name(r)); - slot.client_state = ClientState::Quarantined; - } + stopSlotClient(index); } slot.generation++; #ifdef ESP_PLATFORM @@ -2359,10 +2467,13 @@ void MQTTBridge::teardownSlot(int index, TeardownReason reason) { slot.initial_connect_done = false; slot.broker_uri[0] = '\0'; slot.token_expires_at = 0; + slot.applied_token_expires_at = 0; slot.last_token_renewal = 0; - slot.reconnect_backoff = 0; - slot.max_backoff_failures = 0; - slot.circuit_breaker_tripped = false; + // The reconnect ladder is deliberately NOT cleared here. It describes the + // endpoint this slot has been failing against, and that history still applies + // until a replacement configuration actually commits — setupSlot() clears it + // there. Clearing it on teardown meant a reconfigure that never committed + // (bad URI, failed config transaction) also forgave a tripped breaker. slot.last_reconnect_attempt = 0; slot.last_log_time = 0; slot.last_deferred_log_ms = 0; @@ -2397,6 +2508,9 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { if (r == ESP_OK) { slot.client_state = ClientState::Starting; slot.generation++; + // The attempt carries whatever credential is configured right now, so this + // is the point at which a freshly minted token becomes the one in use. + slot.applied_token_expires_at = slot.token_expires_at; } return r; } @@ -2513,9 +2627,15 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // exp on live sessions (waev's 55-minute tokens). const unsigned long renewal_buffer = MQTTConnectionPolicy::renewalBufferSecs( static_cast(slotTokenLifetime(index))); + // Against the APPLIED expiry — the credential the live connection is using — + // not the one sitting in the token buffer. Minting updates the buffer's + // expiry immediately, so reading that here meant a renewal whose bounce + // failed looked complete: the next pass saw a fresh future expiry, decided + // no renewal was due, and never retried the bounce. The connection stayed on + // the old credential until the broker enforced exp (F06). bool token_needs_renewal = MQTTConnectionPolicy::tokenNeedsRenewal( time_synced, static_cast(current_time), - static_cast(slot.token_expires_at), + static_cast(slot.applied_token_expires_at), static_cast(renewal_buffer)); // Throttle renewal attempts to once per minute @@ -2525,7 +2645,7 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns if (token_needs_renewal && can_attempt_renewal) { slot.last_token_renewal = now_millis; - unsigned long old_token_expires_at = slot.token_expires_at; + unsigned long old_token_expires_at = slot.applied_token_expires_at; if (createSlotAuthToken(index)) { MQTT_DEBUG_PRINTLN("MQTT%d token renewed", index + 1); @@ -2549,6 +2669,11 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns if (!exp_forces_bounce && old_token_expired_or_imminent && slot.client->connected()) { MQTT_DEBUG_PRINTLN("MQTT%d token renewed, no bounce (broker does not enforce exp)", index + 1); + // This broker does not act on exp, so the live session's older + // credential is not a problem and the renewal is complete. Recording + // it stops the renewal from staying due and re-attempting every + // minute for the rest of the token's life. + slot.applied_token_expires_at = slot.token_expires_at; } if (exp_forces_bounce || !slot.client->connected()) { // Disconnect + reconnect with fresh credentials, reusing existing client @@ -2585,9 +2710,13 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // the renewal instead so the next maintenance pass retries the // bounce, and leave the reconnect ladder alone — this is a local // failure, not a broker fault. - MQTT_DEBUG_PRINTLN("MQTT%d renewal bounce failed (%s) - retrying next pass", + // Leave applied_token_expires_at pointing at the OLD credential: + // that is still what the connection is using, so the renewal stays + // due and the next pass retries the bounce (paced by the one-a- + // minute renewal throttle). The freshly minted token stays in the + // buffer and will be used by that retry, or by the next reconnect. + MQTT_DEBUG_PRINTLN("MQTT%d renewal bounce failed (%s) - still due, retrying", index + 1, esp_err_to_name(bounce_result)); - slot.last_token_renewal = 0; return; } MQTT_TRACE_HEAP("renewal:after-reconnect", index); @@ -2601,10 +2730,14 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns } else { // Token renewed but old one still valid — just update credentials for next reconnect slot.client->setCredentials(_jwt_username, slot.auth_token); + // Staged for the next reconnect, and the credential in use is still + // valid, so the renewal decision is settled for this token. + slot.applied_token_expires_at = slot.token_expires_at; } } else { MQTT_DEBUG_PRINTLN("MQTT%d token renewal failed", index + 1); slot.token_expires_at = 0; + slot.applied_token_expires_at = 0; } return; // Token renewal handled connect; skip backoff logic below } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 6c81cfc2..d919211d 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -12,6 +12,7 @@ #include "helpers/MQTTLifecycle.h" #include "helpers/AlertFaultPolicy.h" #include "helpers/MQTTEffectiveConfig.h" +#include "helpers/MQTTClientState.h" #include #ifdef WITH_SNMP @@ -105,29 +106,12 @@ private: static const uint32_t kNtpMinValidEpoch = 1767225600UL; // 2026-01-01 UTC static const uint32_t kNtpMaxValidEpoch = 4102444800UL; // 2100-01-01 UTC - // What the SDK client is doing, as opposed to whether the network is up. - // `client->connected()` answers the second question and was being used for - // the first: a client resolving DNS, negotiating TLS or waiting after a - // failed CONNECT reports not-connected, so teardown skipped it and left its - // task running (F04). Absent is 0 so a memset-initialised slot is correct. - enum class ClientState : uint8_t { - Absent = 0, // no client object - Configured, // client allocated, never started - Starting, // start/reconnect requested, awaiting CONNECTED - Connected, // CONNECTED received - Disconnected, // started, no session (our reconnect ladder governs it) - Stopped, // stop completed; the SDK task is joined and gone - Quarantined, // stop failed: the SDK task was NOT joined. Never destroy, - // never reuse, never free anything it still points at. - }; - - static const char* clientStateName(ClientState s); - // True while the SDK client has been started and not proven stopped, i.e. - // while it may still own a task, a socket and a TLS context. - static bool clientStateIsLive(ClientState s) { - return s == ClientState::Starting || s == ClientState::Connected || - s == ClientState::Disconnected; - } + // Per-slot SDK client state and the shutdown contract live in + // MQTTClientState.h so both are host-testable; ClientState is an alias so the + // bridge code reads naturally. + using ClientState = MqttClientState; + static const char* clientStateName(ClientState s) { return mqttClientStateName(s); } + static bool clientStateIsLive(ClientState s) { return mqttClientStateIsLive(s); } // Connection slot - each slot holds one MQTT connection struct MQTTSlot { @@ -158,7 +142,14 @@ private: // esp-mqtt re-reads it whenever a later connect() re-applies a dirtied config. // Freed only alongside the client in destroySlotClients(). char* auth_token; // nullptr or empty string = no valid token + // Two expiries, deliberately. token_expires_at describes the token in the + // buffer (minting updates it immediately); applied_token_expires_at + // describes the credential the CONNECTION is actually using, and only + // advances when a connect/reconnect has carried it. The renewal decision + // reads the applied one, so a renewal whose bounce failed stays due and is + // retried instead of looking complete (F06). unsigned long token_expires_at; + unsigned long applied_token_expires_at; unsigned long last_token_renewal; // Custom broker settings (only used when preset_name is "custom") @@ -547,6 +538,11 @@ private: // with it. Here the stop IS the point. enum class TeardownReason : uint8_t { Reconfigure, Disable }; void teardownSlot(int index, TeardownReason reason = TeardownReason::Disable); + // Close a live client for a reconfigure: softDisconnect where that is enough, + // a real stop where an in-flight connection attempt has to be cancelled. + void closeLiveClientForReconfigure(int index); + // Stop a live client, recording Stopped (proven) or Quarantined (not joined). + void stopSlotClient(int index); // Reconnect a slot, starting it instead when the client is stopped (reconnect() is a // no-op on a stopped client). See the definition. // ESP_OK when the reconnect/start was accepted by the SDK. A local failure diff --git a/test/test_mqtt_client_state/test_mqtt_client_state.cpp b/test/test_mqtt_client_state/test_mqtt_client_state.cpp new file mode 100644 index 00000000..9b0f7c36 --- /dev/null +++ b/test/test_mqtt_client_state/test_mqtt_client_state.cpp @@ -0,0 +1,89 @@ +#include "helpers/MQTTClientState.h" + +#include + +#include +#include + +namespace { + +const MqttClientState kAllStates[] = { + MqttClientState::Absent, MqttClientState::Configured, + MqttClientState::Starting, MqttClientState::Connected, + MqttClientState::Disconnected, MqttClientState::Stopped, + MqttClientState::Quarantined, +}; + +bool ack(std::vector states) { + return mqttStopMayBeAcknowledged(states.data(), (int)states.size()); +} + +} // namespace + +TEST(MqttClientState, LiveMeansStartedAndNotProvenStopped) { + EXPECT_TRUE(mqttClientStateIsLive(MqttClientState::Starting)); + EXPECT_TRUE(mqttClientStateIsLive(MqttClientState::Connected)); + EXPECT_TRUE(mqttClientStateIsLive(MqttClientState::Disconnected)); + + EXPECT_FALSE(mqttClientStateIsLive(MqttClientState::Absent)); + EXPECT_FALSE(mqttClientStateIsLive(MqttClientState::Configured)); + EXPECT_FALSE(mqttClientStateIsLive(MqttClientState::Stopped)); + // Quarantined is not "live": it is worse. It may own a task and we can never + // find out, which is why it is not simply lumped in with the live states. + EXPECT_FALSE(mqttClientStateIsLive(MqttClientState::Quarantined)); +} + +TEST(MqttClientState, OnlyStartingHasAnAttemptInFlight) { + for (MqttClientState s : kAllStates) { + EXPECT_EQ(s == MqttClientState::Starting, mqttClientStateHasAttemptInFlight(s)) + << mqttClientStateName(s); + } +} + +TEST(MqttClientState, ProvenStoppedIsTheComplementOfLiveAndQuarantined) { + for (MqttClientState s : kAllStates) { + const bool proven = mqttClientStateIsProvenStopped(s); + EXPECT_EQ(!mqttClientStateIsLive(s) && s != MqttClientState::Quarantined, proven) + << mqttClientStateName(s); + } +} + +// The invariant the shutdown contract rests on: the acknowledgement means the +// task destroyed its clients, so it may not be published while any client's +// stop is unproven. +TEST(MqttClientState, StopIsAcknowledgedOnlyWhenEveryClientIsProvenStopped) { + EXPECT_TRUE(ack({})); // no slots configured at all + EXPECT_TRUE(ack({MqttClientState::Absent, MqttClientState::Stopped, + MqttClientState::Configured})); + + // One quarantined client withholds the acknowledgement, whatever the others did. + EXPECT_FALSE(ack({MqttClientState::Quarantined})); + EXPECT_FALSE(ack({MqttClientState::Stopped, MqttClientState::Stopped, + MqttClientState::Quarantined, MqttClientState::Absent})); + EXPECT_FALSE(ack({MqttClientState::Quarantined, MqttClientState::Absent, + MqttClientState::Absent, MqttClientState::Absent, + MqttClientState::Absent, MqttClientState::Absent})); + + // So does a client that is merely still live: teardown did not finish it. + EXPECT_FALSE(ack({MqttClientState::Stopped, MqttClientState::Connected})); + EXPECT_FALSE(ack({MqttClientState::Starting})); + EXPECT_FALSE(ack({MqttClientState::Disconnected})); + + // An empty set is trivially proven; a null array with a nonzero count is a + // caller bug and must never read as proof. + EXPECT_TRUE(mqttStopMayBeAcknowledged(nullptr, 0)); + EXPECT_FALSE(mqttStopMayBeAcknowledged(nullptr, 3)); +} + +TEST(MqttClientState, EveryStateHasAName) { + for (MqttClientState s : kAllStates) { + ASSERT_NE(nullptr, mqttClientStateName(s)); + EXPECT_GT(strlen(mqttClientStateName(s)), 0u); + } + EXPECT_STREQ("quarantined", mqttClientStateName(MqttClientState::Quarantined)); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 4cf0038acb668d5d8915d76bc89848eb0188b40f Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Sep 2026 20:03:13 -0700 Subject: [PATCH 73/93] fix(mqtt): stop accepting time from lwIP SNTP, which validates nothing here (F07) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review asked for the pinned SNTP implementation's response checking to be established rather than assumed. It is `SNTP_CHECK_RESPONSE = 0`: the lwIP default in `lwip/src/include/lwip/apps/sntp_opts.h`, not overridden in this build's ESP32 `lwipopts.h` or any sdkconfig — and lwIP ships precompiled in the SDK, so no `-D` of ours can change it. At 0 it checks neither that the reply came from the server it queried nor that the originate timestamp matches the request it sent. Those are two of the checks that make the new validated probe trustworthy, so SNTP was strictly weaker than the path it backed up. It was reachable three ways, all of which set the system clock: - the fallback inside `syncTimeWithNTP()`, which ran precisely when the validated probe had failed — i.e. when interference is most likely; - `refreshNTP()`, hourly, for the life of the node; - `configTime()` after a *successful* validated sync, whose timezone side effect was all that was wanted but which also left a background SNTP poller running that would go on accepting unvalidated replies. All three are gone. The periodic refresh runs the same validated probe as every other sync, and the timezone is set directly with `setenv("TZ", "UTC0")` + `tzset()`. Nothing is lost operationally: SNTP queried the same servers over the same UDP/123 with a weaker parser. The RTC/system-clock fallback is a separate decision and stays. --- src/helpers/bridges/MQTTBridge.cpp | 90 ++++++++++++------------------ src/helpers/bridges/MQTTBridge.h | 4 +- 2 files changed, 37 insertions(+), 57 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 9e176c4b..d1c866e2 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -4490,13 +4489,14 @@ void MQTTBridge::storeRawRadioData(const uint8_t* raw_data, int len, float snr, // NTP time sync // --------------------------------------------------------------------------- +// Periodic refresh. This used to call configTime(), i.e. start lwIP's SNTP and +// let it set the system clock asynchronously — an acceptance path with NO +// validation of its own (see the syncTimeWithNTP() note below), running every +// hour for the life of the node. It now goes through the same validated probe +// as every other sync, which costs the MQTT task about a second per attempt and +// touches nothing until a reply passes every check. void MQTTBridge::refreshNTP() { - // Lightweight periodic refresh: just restart SNTP which runs async in the background. - // No blocking DNS, no UDP sockets, no retry loops on the MQTT task loop. - // The heavy syncTimeWithNTP() is only used for initial sync and WiFi reconnect recovery. - configTime(0, 0, effectiveNtpPrimary(_obs)); - _last_ntp_sync = millis(); - MQTT_DEBUG_PRINTLN("NTP refresh triggered (async SNTP)"); + syncTimeWithNTP(/*force=*/true, /*primary_only=*/false); } // One validated NTP exchange with one named server, on a fresh ephemeral socket. @@ -4656,48 +4656,27 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { } } - // Fallback: use ESP32 built-in SNTP (configTime) when no server passed validation - #ifdef ESP_PLATFORM - if (!ntp_ok) { - MQTT_DEBUG_PRINTLN("NTP client failed, trying SNTP fallback..."); - for (int s = 0; s < server_count && !ntp_ok; s++) { - const char* server = servers[s]; - MQTT_DEBUG_PRINTLN("SNTP fallback trying %s...", server); - // A plausible clock is not evidence this server answered. The device usually - // already holds valid time here — from an earlier sync, or the RTC — so polling - // time(nullptr) declared the very first server successful without a packet ever - // arriving, stopped the fallback walk there, and refreshed _last_ntp_sync. Worse - // on the `set mqtt.ntp` validation path, where a typo is supposed to fail fast. - // Wait for SNTP itself to report completion. The status is one-shot — reading - // COMPLETED clears it — so drop any result an earlier sync left behind, and do - // that *before* starting this one: configTime() returns after sntp_init(), so a - // fast reply can complete inside it, and clearing afterwards would erase the - // very result being waited for. - if (sntp_enabled()) { - sntp_stop(); - } - sntp_set_sync_status(SNTP_SYNC_STATUS_RESET); - configTime(0, 0, server); - for (int i = 0; i < 20; i++) { - delay(500); - if (sntp_get_sync_status() != SNTP_SYNC_STATUS_COMPLETED) continue; - epochTime = (unsigned long)time(nullptr); - if (epochTime >= kMinValidEpoch) { - ntp_ok = true; - ntp_server_used = server; - MQTT_DEBUG_PRINTLN("SNTP fallback succeeded on %s: %lu", server, epochTime); - } else { - MQTT_DEBUG_PRINTLN("SNTP fallback: %s synced an implausible epoch %lu", server, epochTime); - } - break; - } - } - } - #endif + // There is deliberately no SNTP fallback here any more. + // + // It used to call configTime() and accept SNTP_SYNC_STATUS_COMPLETED plus a + // plausible epoch. lwIP's SNTP checks its own response only as far as + // SNTP_CHECK_RESPONSE allows, and in this build that is **0**: the default in + // lwip/src/include/lwip/apps/sntp_opts.h, not overridden in the ESP32 + // lwipopts.h or any sdkconfig here — and lwIP is shipped precompiled in the + // SDK, so a -D from our build cannot change it. At 0 it verifies neither that + // the reply came from the server it queried nor that the originate timestamp + // matches the request it sent. Those are exactly the two checks that make the + // probe above trustworthy, so the fallback was strictly weaker than the path + // it backed up — and it ran precisely when the validated path had failed, + // which is when interference is most likely. + // + // Nothing is lost operationally: it queried the same servers over the same + // UDP/123 with a weaker parser. The RTC/system-clock fallback below is a + // separate decision and stays. - // No server answered, but the clock itself may still be usable. Requiring a real - // SNTP completion above removed something the plausible-clock test was doing by - // accident: an RTC-backed device on a network that blocks NTP (UDP/123) while + // No server answered, but the clock itself may still be usable. Requiring a + // validated reply above removed something the old plausible-clock test was + // doing by accident: an RTC-backed device on a network that blocks NTP (UDP/123) while // allowing the broker (443) stayed synced and kept minting JWTs. _ntp_synced gates // slot setup outright, so losing that strands those deployments with no slots at // all. Keep the behaviour, but as its own decision rather than as a claim about a @@ -4734,13 +4713,14 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { accepted.tv_usec = 0; settimeofday(&accepted, nullptr); - // Only when a server supplied the accepted epoch. The fallback above necessarily - // points configTime() at each server before knowing whether it replies; this is - // the post-acceptance call, and there is nothing to re-point it at when the epoch - // came from a local clock. - if (ntp_server_used) { - configTime(0, 0, ntp_server_used); - } + // Keep the process timezone at UTC without starting lwIP's SNTP. This used + // to be configTime(0, 0, ntp_server_used), whose timezone side effect is all + // that was wanted here — the rest of it starts a background SNTP poller that + // would go on setting the clock from replies nothing validates, for the life + // of the node (see the note where the SNTP fallback used to be). System time + // is UTC; the prefs Timezone is applied separately by the message builders. + setenv("TZ", "UTC0", 1); + tzset(); if (_rtc) { _rtc->setCurrentTime(epochTime); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index d919211d..8d180a48 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -285,7 +285,7 @@ private: volatile bool _status_publish_pending[RUNTIME_MQTT_SLOTS]; // CLI-requested forced NTP sync, marshalled onto the MQTT task (Core 0). - // All NTP I/O (_ntp_client, configTime) must run on Core 0; the CLI thread + // All NTP I/O must run on Core 0; the CLI thread // (Core 1) sets _ntp_force_requested and blocks in requestForcedNtpSync() // until the task publishes the outcome via _ntp_force_result/_ntp_force_done. // Single-requester assumption: CLI commands are serialized, so at most one @@ -296,7 +296,7 @@ private: // CLI-requested NTP connectivity diagnostic, marshalled onto the MQTT task (Core 0) // with the same handshake as the forced sync. Probe-only: it queries each server and - // records the reported time but never calls configTime()/setCurrentTime(), so the + // records the reported time but never sets the system clock or the RTC, so the // system clock is left untouched. Results are written by the task and read by the CLI // thread once _ntp_diag_done is set. volatile bool _ntp_diag_requested; From f83a789a69d1b6bd1b1414a1165f28d103c1e079 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 09:10:32 -0700 Subject: [PATCH 74/93] Restore WiFi flags dropped from ThinkNode M7 companion wifi env in upstream merge The upstream/dev merge kept our side of variants/thinknode_m7/platformio.ini, losing -D ENABLE_WIFI_INTERFACE and +. Companion WiFi is now gated on that flag, so the env built silently without WiFi. --- variants/thinknode_m7/platformio.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index 7da14112..482848de 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -136,6 +136,7 @@ extends = ThinkNode_M7 build_flags = ${ThinkNode_M7.build_flags} -I examples/companion_radio/ui-orig + -D ENABLE_WIFI_INTERFACE -D MAX_CONTACTS=350 -D MAX_GROUP_CHANNELS=40 -D DISPLAY_CLASS=NullDisplayDriver @@ -146,6 +147,7 @@ build_flags = ; -D MESH_PACKET_LOGGING=1 build_src_filter = ${ThinkNode_M7.build_src_filter} + + + + + +<../examples/companion_radio/*.cpp> From 4c683b13b30cb2acb5f29849fd4f758fd6c33c72 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:14:16 -0700 Subject: [PATCH 75/93] fix(network): pre-merge review fixes - Guard the boot-time link bootstrap to observer ESP32 builds. It ran unconditionally in MyMesh::begin(), breaking every non-observer repeater/room server build (ESP32 and nRF52). - Rename NetworkInterface -> NetworkLink (class, accessor, files). Arduino-ESP32 3.x ships its own NetworkInterface class and header, which broke the ESP32-C6 builds. Drop WiFi.setAutoConnect(), a no-op on 2.x and removed in 3.x. - Refresh stored Wi-Fi credentials every bridge tick so the STA reconnect loop picks up `set wifi.ssid` / `set wifi.pwd` without a reboot, as the bridge did before the link moved out of it. Skip reconnects while the SSID is empty. - Restore the "WiFi connected: " / "WiFi disconnected: reason N" debug lines the bridge used to print. - Alert on Ethernet only once it has held a lease this boot or when no Wi-Fi is configured; Wi-Fi-only installs of an Ethernet-preferred image keep Wi-Fi alerts instead of reporting "Ethernet down". - Record wifi.setup_complete only for Ethernet LAN onboarding, so Wi-Fi builds keep the SSID-based first-boot portal rule. - Use seq_cst for the route-switch lock/mutation flag handshake. - Docs: SNMP RSSI sentinel is -127; describe link-return vs medium-switch reconnect behavior accurately; note runtime credential pickup. - Test: unknown keys inside a known /mqtt.json group are ignored, which keeps wifi.setup_complete downgrade-safe. --- MQTT_IMPLEMENTATION.md | 7 +- examples/simple_repeater/MyMesh.cpp | 14 +-- examples/simple_room_server/MyMesh.cpp | 14 +-- src/helpers/AlertReporter.cpp | 2 +- src/helpers/CommonCLI_Observer.cpp | 26 ++--- src/helpers/ESP32Board.cpp | 16 +-- .../{NetworkInterface.cpp => NetworkLink.cpp} | 97 ++++++++++++------- .../{NetworkInterface.h => NetworkLink.h} | 11 ++- src/helpers/SNMPAgent.cpp | 4 +- src/helpers/SNMPAgent.h | 2 +- src/helpers/bridges/MQTTBridge.cpp | 11 ++- src/helpers/bridges/MQTTBridge.h | 4 +- src/helpers/esp32/WebConfigServer.cpp | 22 ++--- .../test_mqtt_prefs_serializer.cpp | 14 +++ 14 files changed, 149 insertions(+), 95 deletions(-) rename src/helpers/{NetworkInterface.cpp => NetworkLink.cpp} (90%) rename src/helpers/{NetworkInterface.h => NetworkLink.h} (89%) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 60e96fc9..550513ec 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -872,8 +872,11 @@ the radio actually performs in that case. - CH390 startup initializes Arduino's shared network event runtime without associating WiFi; this keeps the framework's DNS and TLS hostname paths safe when Ethernet wins directly. - Ethernet/WiFi transitions are logged. A lost or changed route stops every started MQTT - client, including one whose disconnect callback arrived first; once a usable route returns, - route-caused backoff is cleared and one immediate reconnect is allowed + client, including one whose disconnect callback arrived first. When the same link returns, + each slot gets one immediate attempt at its current backoff rung (a tripped circuit breaker + gets one immediate probe); a switch to the other medium also clears backoff and breakers +- WiFi credentials changed at runtime (`set wifi.ssid` / `set wifi.pwd`) are used on the next + reconnect attempt without a reboot - Packets are queued while a slot is disconnected and flushed when it recovers ### Raw Radio Data Capture diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 9ab4bd99..29518694 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -3,7 +3,7 @@ #include #include // for qsort() #include -#include +#include #include #if defined(ESP_PLATFORM) #include @@ -1148,7 +1148,8 @@ void MyMesh::begin(FILESYSTEM *fs) { } #endif - NetworkInterface& boot_network = activeNetworkInterface(); +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) + NetworkLink& boot_network = activeNetworkLink(); if (boot_network.isAutomatic()) { char network_hostname[NetworkHostname::kBufferSize]; NetworkHostname::build(network_hostname, sizeof(network_hostname), @@ -1165,6 +1166,7 @@ void MyMesh::begin(FILESYSTEM *fs) { boot_network.mediumName(), boot_network.statusName(), (unsigned long)(millis() - ethernet_probe_started_at)); } +#endif acl.load(_fs, self_id); // TODO: key_store.begin(); @@ -1504,13 +1506,13 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { if (force_ap) { // The setup AP owns WiFi outright; refuse while the bridge holds the STA. if (bridge && bridge->isRunning() && - activeNetworkInterface().medium() != NetworkMedium::Ethernet) { + activeNetworkLink().medium() != NetworkMedium::Ethernet) { strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); return true; } _webconfig->startSetupMode(reply); - } else if (activeNetworkInterface().isConnected()) { - _webconfig->startLanMode(activeNetworkInterface().localIP(), + } else if (activeNetworkLink().isConnected()) { + _webconfig->startLanMode(activeNetworkLink().localIP(), !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { _webconfig->startSetupMode(reply); @@ -1551,7 +1553,7 @@ void MyMesh::onConfigBatchEnd() { void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; char wifi_rssi[12] = "null"; - NetworkInterface& network = activeNetworkInterface(); + NetworkLink& network = activeNetworkLink(); const char* network_medium = network.mediumName(); if (network.isConnected()) { strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 64c61a2e..f5b41860 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #if defined(ESP_PLATFORM) #include @@ -949,7 +949,8 @@ void MyMesh::begin(FILESYSTEM *fs) { // load persisted prefs _cli.loadPrefs(_fs); - NetworkInterface& boot_network = activeNetworkInterface(); +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) + NetworkLink& boot_network = activeNetworkLink(); if (boot_network.isAutomatic()) { char network_hostname[NetworkHostname::kBufferSize]; NetworkHostname::build(network_hostname, sizeof(network_hostname), @@ -966,6 +967,7 @@ void MyMesh::begin(FILESYSTEM *fs) { boot_network.mediumName(), boot_network.statusName(), (unsigned long)(millis() - ethernet_probe_started_at)); } +#endif acl.load(_fs, self_id); region_map.load(_fs); @@ -1306,13 +1308,13 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { if (force_ap) { // The setup AP owns WiFi outright; refuse while the bridge holds the STA. if (bridge && bridge->isRunning() && - activeNetworkInterface().medium() != NetworkMedium::Ethernet) { + activeNetworkLink().medium() != NetworkMedium::Ethernet) { strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); return true; } _webconfig->startSetupMode(reply); - } else if (activeNetworkInterface().isConnected()) { - _webconfig->startLanMode(activeNetworkInterface().localIP(), + } else if (activeNetworkLink().isConnected()) { + _webconfig->startLanMode(activeNetworkLink().localIP(), !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { _webconfig->startSetupMode(reply); @@ -1353,7 +1355,7 @@ void MyMesh::onConfigBatchEnd() { void MyMesh::buildStatsJson(char* buf, size_t buf_size) { char ip[20] = ""; char wifi_rssi[12] = "null"; - NetworkInterface& network = activeNetworkInterface(); + NetworkLink& network = activeNetworkLink(); const char* network_medium = network.mediumName(); if (network.isConnected()) { strncpy(ip, network.localIP().toString().c_str(), sizeof(ip) - 1); diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index c473b274..4a58ad61 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -6,7 +6,7 @@ #include #ifdef WITH_MQTT_BRIDGE #include "AlertFaultPolicy.h" -#include "NetworkInterface.h" +#include "NetworkLink.h" #endif // Header layout for PAYLOAD_TYPE_GRP_TXT before encryption: diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 273666be..f1d757f8 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -15,7 +15,7 @@ #include "TxtDataHelpers.h" #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) -#include "NetworkInterface.h" +#include "NetworkLink.h" #include #include #include @@ -416,9 +416,9 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf // runs on the Arduino loop task, shared with mesh/radio processing and the // web config batch, so a synchronous wait of up to 30 s would stall the // node. The sync runs in the background; verify with `get mqtt.ntp.diag`. - if (!activeNetworkInterface().isConnected()) { + if (!activeNetworkLink().isConnected()) { snprintf(reply, 160, "OK - saved (%s not connected; NTP sync pending)", - activeNetworkInterface().mediumName()); + activeNetworkLink().mediumName()); } else if (!_callbacks->isMqttBridgeRunning()) { strcpy(reply, "OK - saved (MQTT bridge not running)"); } else if (_callbacks->syncMqttNtp()) { @@ -460,8 +460,8 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf _mqtt_prefs.wifi_power_save = ps_value; if (!persistObserverPrefs(reply)) return true; #ifdef ESP_PLATFORM - if (strcmp(activeNetworkInterface().mediumName(), "wifi") == 0 && - activeNetworkInterface().isConnected()) { + if (strcmp(activeNetworkLink().mediumName(), "wifi") == 0 && + activeNetworkLink().isConnected()) { wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : (ps_value == 2) ? WIFI_PS_MAX_MODEM : WIFI_PS_MIN_MODEM; esp_err_t ps_result = esp_wifi_set_ps(ps_mode); @@ -962,9 +962,9 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf #ifdef ESP_PLATFORM // Connectivity probe across all configured NTP servers; never updates the clock. // Serial console (sender_timestamp == 0) gets a detailed table; LoRa gets a compact list. - if (!activeNetworkInterface().isConnected()) { + if (!activeNetworkLink().isConnected()) { snprintf(reply, 160, "Error: %s not connected", - activeNetworkInterface().mediumName()); + activeNetworkLink().mediumName()); } else if (!_callbacks->isMqttBridgeRunning()) { strcpy(reply, "Error: MQTT bridge not running"); } else if (!_callbacks->runMqttNtpDiag(reply, 160, sender_timestamp == 0)) { @@ -1039,10 +1039,10 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf strcpy(reply, _mqtt_prefs.wifi_password[0] ? "> ******** (serial only)" : "> (not set)"); } } else if (strcmp(config, "link.diag") == 0) { - activeNetworkInterface().formatDiagnostics(reply, 160); + activeNetworkLink().formatDiagnostics(reply, 160); } else if (memcmp(config, "link.status", 11) == 0 || memcmp(config, "wifi.status", 11) == 0) { - NetworkInterface& network = activeNetworkInterface(); + NetworkLink& network = activeNetworkLink(); const bool wifi_alias = config[0] == 'w'; if (wifi_alias && strcmp(network.mediumName(), "wifi") != 0) { snprintf(reply, 160, "> n/a (%s selected; use get link.status)", @@ -1181,9 +1181,9 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, #ifdef WITH_MQTT_BRIDGE if (memcmp(command, "tls.bundletest ", 15) == 0) { #ifdef ESP_PLATFORM - if (!activeNetworkInterface().isConnected()) { + if (!activeNetworkLink().isConnected()) { snprintf(reply, 160, "ERR: %s not connected", - activeNetworkInterface().mediumName()); + activeNetworkLink().mediumName()); } else { size_t bundle_len = 0; if (rootca_crt_bundle_start != nullptr && @@ -1228,9 +1228,9 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, // ota check -> report available build, do not flash // ota update -> download and flash, then reboot #if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) - if (!activeNetworkInterface().isConnected()) { + if (!activeNetworkLink().isConnected()) { snprintf(reply, 160, "ERR: %s not connected", - activeNetworkInterface().mediumName()); + activeNetworkLink().mediumName()); } else if (memcmp(command, "ota check", 9) == 0) { // Check is synchronous so its result lands in this reply, and runs with the // MQTT bridge UP: the slim per-variant manifest is tiny, so the fetch only diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 20e079b0..19c3e5cc 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -1,7 +1,7 @@ #ifdef ESP_PLATFORM #include "ESP32Board.h" -#include "NetworkInterface.h" +#include "NetworkLink.h" #include #if defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA) // Repeater or Room Server only @@ -46,7 +46,7 @@ void otaReleaseTransport() { ota_server_running = false; ota_started_at = 0; if (ota_network_locked) { - activeNetworkInterface().unlockSwitching(); + activeNetworkLink().unlockSwitching(); ota_network_locked = false; } HttpPort80Lease::release(HttpPort80Lease::Owner::Ota); @@ -67,7 +67,7 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { } inhibit_sleep = true; // prevent sleep during OTA - activeNetworkInterface().lockSwitching(); + activeNetworkLink().lockSwitching(); ota_network_locked = true; // If the device is already on its selected network, serve ElegantOTA on that @@ -78,8 +78,8 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { // station IP can't be reached. IPAddress ip; if (NetworkPolicy::startOtaUsesSelectedNetwork( - force_ap, activeNetworkInterface().isConnected())) { - ip = activeNetworkInterface().localIP(); + force_ap, activeNetworkLink().isConnected())) { + ip = activeNetworkLink().localIP(); } else { ota_raised_ap = WiFi.softAP("MeshCore-OTA", NULL); if (!ota_raised_ap) { @@ -290,7 +290,7 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep // mesh-receive call chain (it overflows the loopTask canary). Run the work in a // dedicated 24 KB-stack task and block here until it finishes. The big stack is // freed when the task exits; on a successful update the chip reboots inside it. - NetworkInterface& network = activeNetworkInterface(); + NetworkLink& network = activeNetworkLink(); network.lockSwitching(); OtaTaskArgs args = { this, current_ver, dry_run, reply, false, false }; TaskHandle_t handle = nullptr; @@ -312,9 +312,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char strcpy(reply, "ERR: OTA not configured (build via build.sh)"); return false; #else - if (!activeNetworkInterface().isConnected()) { + if (!activeNetworkLink().isConnected()) { snprintf(reply, 160, "ERR: %s not connected", - activeNetworkInterface().mediumName()); + activeNetworkLink().mediumName()); return false; } diff --git a/src/helpers/NetworkInterface.cpp b/src/helpers/NetworkLink.cpp similarity index 90% rename from src/helpers/NetworkInterface.cpp rename to src/helpers/NetworkLink.cpp index 73f2411d..10dfb23a 100644 --- a/src/helpers/NetworkInterface.cpp +++ b/src/helpers/NetworkLink.cpp @@ -1,4 +1,4 @@ -#include "NetworkInterface.h" +#include "NetworkLink.h" #if defined(ESP_PLATFORM) @@ -21,9 +21,16 @@ extern void tcpipInit(); #endif +// Same "MQTT: " prefix as MQTT_DEBUG_PRINTLN so existing log greps keep matching. +#if defined(MQTT_DEBUG) + #define NETWORK_DEBUG_PRINTLN(F, ...) do { if (Serial.availableForWrite() > 0) { Serial.printf("MQTT: " F "\n", ##__VA_ARGS__); } } while (0) +#else + #define NETWORK_DEBUG_PRINTLN(...) do {} while (0) +#endif + namespace { -class NetworkInterfaceBase : public NetworkInterface { +class NetworkLinkBase : public NetworkLink { protected: std::atomic _outage_bits{AlertFaultPolicy::packOutageSnapshot({false, 0, 0})}; std::atomic _connected_at{0}; @@ -75,7 +82,7 @@ class NetworkInterfaceBase : public NetworkInterface { } }; -class WiFiNetworkInterface final : public NetworkInterfaceBase { +class WiFiNetworkLink final : public NetworkLinkBase { bool _event_registered = false; char _hostname[32] = {}; char _ssid[33] = {}; @@ -118,29 +125,36 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { _hostname[sizeof(_hostname) - 1] = '\0'; } - bool begin(const char* wifi_ssid, const char* wifi_password) override { - if (!configValid(wifi_ssid)) return false; - strncpy(_ssid, wifi_ssid, sizeof(_ssid) - 1); + void updateWifiCredentials(const char* wifi_ssid, const char* wifi_password) override { + strncpy(_ssid, wifi_ssid ? wifi_ssid : "", sizeof(_ssid) - 1); _ssid[sizeof(_ssid) - 1] = '\0'; strncpy(_password, wifi_password ? wifi_password : "", sizeof(_password) - 1); _password[sizeof(_password) - 1] = '\0'; + } + + bool begin(const char* wifi_ssid, const char* wifi_password) override { + if (!configValid(wifi_ssid)) return false; + updateWifiCredentials(wifi_ssid, wifi_password); // Arduino-ESP32 applies this stored value when it creates the STA netif. // It must be set before WiFi.mode()/begin() for the first DHCP exchange. if (_hostname[0] != '\0') WiFi.setHostname(_hostname); WiFi.mode(WIFI_STA); WiFi.setAutoReconnect(true); - WiFi.setAutoConnect(true); if (!_event_registered) { WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { switch (event) { case ARDUINO_EVENT_WIFI_STA_GOT_IP: + NETWORK_DEBUG_PRINTLN("WiFi connected: %s", + IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); noteConnected(millis()); _reconnect_backoff_attempt = 0; break; case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: noteDisconnected(millis(), info.wifi_sta_disconnected.reason); + NETWORK_DEBUG_PRINTLN("WiFi disconnected: reason %d", + info.wifi_sta_disconnected.reason); break; default: break; @@ -195,7 +209,7 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { const bool transitioned = _last_connected; if (transitioned) { _connected_at.store(0, std::memory_order_relaxed); - } else if (snapshot.down && MQTTConnectionPolicy::wifiReconnectDue( + } else if (snapshot.down && _ssid[0] != '\0' && MQTTConnectionPolicy::wifiReconnectDue( now_ms, snapshot.started_ms, (uint32_t)_last_reconnect_attempt, _reconnect_backoff_attempt)) { _last_reconnect_attempt = now_ms; @@ -224,7 +238,7 @@ class WiFiNetworkInterface final : public NetworkInterfaceBase { }; #if defined(NETWORK_PREFER_ETHERNET) -class EthernetNetworkInterface final : public NetworkInterfaceBase { +class EthernetNetworkLink final : public NetworkLinkBase { public: enum class EventState : uint8_t { None, @@ -402,9 +416,9 @@ class EthernetNetworkInterface final : public NetworkInterfaceBase { } }; -class AutomaticNetworkInterface final : public NetworkInterface { - EthernetNetworkInterface _ethernet; - WiFiNetworkInterface _wifi; +class AutomaticNetworkLink final : public NetworkLink { + EthernetNetworkLink _ethernet; + WiFiNetworkLink _wifi; std::atomic _selected{NetworkMedium::None}; std::atomic _ethernet_started{false}; std::atomic _wifi_started{false}; @@ -417,16 +431,23 @@ class AutomaticNetworkInterface final : public NetworkInterface { std::atomic _ethernet_no_ip_since{0}; std::atomic _switch_locks{0}; std::atomic _switch_in_progress{false}; + std::atomic _ethernet_seen{false}; // held a lease at least once this boot - NetworkInterface& selectedInterface(NetworkMedium selected) { - return selected == NetworkMedium::Ethernet - ? static_cast(_ethernet) - : static_cast(_wifi); + // Ethernet is the primary for alerts once it has carried traffic, or when it + // is the only configured medium; a Wi-Fi-only install keeps Wi-Fi alerts. + bool ethernetIsAlertPrimary() const { + return _ethernet_seen.load(std::memory_order_acquire) || !wifiConfigured(); } - const NetworkInterface& selectedInterface(NetworkMedium selected) const { + + NetworkLink& selectedInterface(NetworkMedium selected) { return selected == NetworkMedium::Ethernet - ? static_cast(_ethernet) - : static_cast(_wifi); + ? static_cast(_ethernet) + : static_cast(_wifi); + } + const NetworkLink& selectedInterface(NetworkMedium selected) const { + return selected == NetworkMedium::Ethernet + ? static_cast(_ethernet) + : static_cast(_wifi); } bool wifiConfigured() const { return _wifi_ssid[0] != '\0'; } @@ -444,14 +465,14 @@ class AutomaticNetworkInterface final : public NetworkInterface { std::memory_order_release); } + // Lock and mutation each publish one flag then read the other; seq_cst keeps + // both sides from missing each other. bool beginUnlockedMutation() { bool expected = false; - if (!_switch_in_progress.compare_exchange_strong( - expected, true, std::memory_order_acq_rel, - std::memory_order_relaxed)) { + if (!_switch_in_progress.compare_exchange_strong(expected, true)) { return false; } - if (_switch_locks.load(std::memory_order_acquire) != 0) { + if (_switch_locks.load() != 0) { _switch_in_progress.store(false, std::memory_order_release); return false; } @@ -532,6 +553,11 @@ class AutomaticNetworkInterface final : public NetworkInterface { } bool isAutomatic() const override { return true; } + void updateWifiCredentials(const char* wifi_ssid, const char* wifi_password) override { + rememberWifi(wifi_ssid, wifi_password); + _wifi.updateWifiCredentials(_wifi_ssid, _wifi_password); + } + void setHostname(const char* hostname) override { // Whichever medium wins now or during a later failover presents the same // stable DHCP identity to the LAN. @@ -589,6 +615,7 @@ class AutomaticNetworkInterface final : public NetworkInterface { delay(25); } + if (_ethernet.isConnected()) _ethernet_seen.store(true, std::memory_order_release); const NetworkMedium initial = NetworkPolicy::bootSelection( _ethernet.isConnected(), wifiConfigured()); if (initial == NetworkMedium::Ethernet) { @@ -606,7 +633,7 @@ class AutomaticNetworkInterface final : public NetworkInterface { _switch_locks.load(std::memory_order_acquire) != 0; const bool ethernet_stopped = ethernet_started && - _ethernet.eventState() == EthernetNetworkInterface::EventState::Stopped; + _ethernet.eventState() == EthernetNetworkLink::EventState::Stopped; if (ethernet_stopped) { _ethernet_started.store(false, std::memory_order_release); ethernet_started = false; @@ -615,11 +642,11 @@ class AutomaticNetworkInterface final : public NetworkInterface { _last_ethernet_init_attempt.store(now_ms, std::memory_order_relaxed); } } - const EthernetNetworkInterface::EventState ethernet_event = + const EthernetNetworkLink::EventState ethernet_event = _ethernet.eventState(); const bool ethernet_link_up = - ethernet_event == EthernetNetworkInterface::EventState::LinkUp || - ethernet_event == EthernetNetworkInterface::EventState::GotIp; + ethernet_event == EthernetNetworkLink::EventState::LinkUp || + ethernet_event == EthernetNetworkLink::EventState::GotIp; if (ethernet_started && ethernet_link_up && !_ethernet.isConnected()) { if (_ethernet_no_ip_since.load(std::memory_order_relaxed) == 0) { uint32_t started_at = now_ms; @@ -647,6 +674,7 @@ class AutomaticNetworkInterface final : public NetworkInterface { const NetworkTransition ethernet_transition = _ethernet.maintain(now_ms, wifi_power_save); + if (_ethernet.isConnected()) _ethernet_seen.store(true, std::memory_order_release); const bool wifi_started = _wifi_started.load(std::memory_order_acquire); const NetworkTransition wifi_transition = wifi_started ? _wifi.maintain(now_ms, wifi_power_save) @@ -721,9 +749,8 @@ class AutomaticNetworkInterface final : public NetworkInterface { void lockSwitching() override { uint8_t value = _switch_locks.load(std::memory_order_relaxed); while (value != UINT8_MAX && !_switch_locks.compare_exchange_weak( - value, static_cast(value + 1), - std::memory_order_acq_rel, std::memory_order_relaxed)) {} - while (_switch_in_progress.load(std::memory_order_acquire)) delay(1); + value, static_cast(value + 1))) {} + while (_switch_in_progress.load()) delay(1); } void unlockSwitching() override { uint8_t value = _switch_locks.load(std::memory_order_relaxed); @@ -819,21 +846,21 @@ class AutomaticNetworkInterface final : public NetworkInterface { : selectedInterface(selected).outageSnapshot(); } NetworkMedium alertMedium() const override { - return NetworkMedium::Ethernet; + return ethernetIsAlertPrimary() ? NetworkMedium::Ethernet : NetworkMedium::WiFi; } AlertFaultPolicy::OutageSnapshot alertOutageSnapshot() const override { - return _ethernet.outageSnapshot(); + return ethernetIsAlertPrimary() ? _ethernet.outageSnapshot() : _wifi.outageSnapshot(); } }; #endif } // namespace -NetworkInterface& activeNetworkInterface() { +NetworkLink& activeNetworkLink() { #if defined(NETWORK_PREFER_ETHERNET) - static AutomaticNetworkInterface network; + static AutomaticNetworkLink network; #else - static WiFiNetworkInterface network; + static WiFiNetworkLink network; #endif return network; } diff --git a/src/helpers/NetworkInterface.h b/src/helpers/NetworkLink.h similarity index 89% rename from src/helpers/NetworkInterface.h rename to src/helpers/NetworkLink.h index 339c1470..afc125a9 100644 --- a/src/helpers/NetworkInterface.h +++ b/src/helpers/NetworkLink.h @@ -20,9 +20,9 @@ * MQTT shutdown deliberately does not stop this interface because an OTA * download runs after the broker clients have been released. */ -class NetworkInterface { +class NetworkLink { public: - virtual ~NetworkInterface() = default; + virtual ~NetworkLink() = default; virtual const char* mediumName() const = 0; virtual NetworkMedium medium() const = 0; @@ -33,6 +33,11 @@ class NetworkInterface { // retain it for any later fallback interface start. virtual void setHostname(const char* hostname) = 0; virtual bool begin(const char* wifi_ssid, const char* wifi_password) = 0; + // Latest stored credentials; the next reconnect attempt uses them without a reboot. + virtual void updateWifiCredentials(const char* wifi_ssid, const char* wifi_password) { + (void)wifi_ssid; + (void)wifi_password; + } virtual NetworkTransition maintain(uint32_t now_ms, uint8_t wifi_power_save) = 0; // Automatic selectors are boot-owned so first-run services can use the @@ -71,6 +76,6 @@ class NetworkInterface { }; /** Build-selected singleton. Wi-Fi is the compatibility default. */ -NetworkInterface& activeNetworkInterface(); +NetworkLink& activeNetworkLink(); #endif diff --git a/src/helpers/SNMPAgent.cpp b/src/helpers/SNMPAgent.cpp index 8394a588..eb29bd68 100644 --- a/src/helpers/SNMPAgent.cpp +++ b/src/helpers/SNMPAgent.cpp @@ -1,7 +1,7 @@ #ifdef WITH_SNMP #include "SNMPAgent.h" -#include "NetworkInterface.h" +#include "NetworkLink.h" #include #include @@ -79,7 +79,7 @@ void MeshSNMPAgent::loop() { _psram_free = 0; #endif - const int signal = activeNetworkInterface().rssi(); + const int signal = activeNetworkLink().rssi(); _wifi_rssi = signal == INT_MIN ? -127 : signal; _snmp.loop(); diff --git a/src/helpers/SNMPAgent.h b/src/helpers/SNMPAgent.h index 2f792634..fe94b10f 100644 --- a/src/helpers/SNMPAgent.h +++ b/src/helpers/SNMPAgent.h @@ -14,7 +14,7 @@ // .2.x.0 = radio (packets, RSSI, SNR, noise floor, air time) // .3.x.0 = mqtt (connected slots, queue depth, skipped publishes) // .4.x.0 = memory (free heap, max alloc, internal free, PSRAM free) -// .5.x.0 = network (RSSI, or 0 when the selected medium has no RSSI) +// .5.x.0 = network (RSSI, or -127 when disconnected or the medium has no RSSI) class MeshSNMPAgent { public: diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 78273d3a..a9d72775 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -101,7 +101,7 @@ void MQTTBridge::getEffectiveMqttOrigin(const NodePrefs* np, const MQTTPrefs* ob } static bool isNetworkConfigValid(const MQTTPrefs* obs) { - return obs && activeNetworkInterface().configValid(obs->wifi_ssid); + return obs && activeNetworkLink().configValid(obs->wifi_ssid); } #ifdef WITH_MQTT_BRIDGE @@ -221,7 +221,7 @@ static void agentLogHeap(const char* location, const char* message, const char* static MQTTBridge* s_mqtt_bridge_instance = nullptr; unsigned long MQTTBridge::getWifiConnectedAtMillis() { - return activeNetworkInterface().connectedAtMillis(); + return activeNetworkLink().connectedAtMillis(); } #if defined(WITH_MQTT_NEIGHBORS) @@ -413,10 +413,10 @@ int MQTTBridge::getMaxActiveSlots() { } uint8_t MQTTBridge::getLastWifiDisconnectReason() { - return activeNetworkInterface().lastDisconnectReason(); + return activeNetworkLink().lastDisconnectReason(); } unsigned long MQTTBridge::getLastWifiDisconnectTime() { - return activeNetworkInterface().lastDisconnectTime(); + return activeNetworkLink().lastDisconnectTime(); } unsigned long MQTTBridge::getSlotCurrentOutageStartMs(int slot_index) const { @@ -624,7 +624,7 @@ static inline uint32_t mqttStopTimeoutForSlots(int slots) { MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mgr, mesh::RTCClock *rtc, mesh::LocalIdentity *identity) : BridgeBase(prefs, mgr, rtc), _obs(obs), - _network(&activeNetworkInterface()), + _network(&activeNetworkLink()), _queue_count(0), _last_status_publish(0), _last_status_retry(0), _status_interval(300000), _ntp_client(_ntp_udp, effectiveNtpPrimary(obs), 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), @@ -2740,6 +2740,7 @@ void MQTTBridge::checkConfigurationMismatch() { bool MQTTBridge::handleNetworkConnection(unsigned long now) { const NetworkMedium previous_medium = _network->medium(); + _network->updateWifiCredentials(_obs->wifi_ssid, _obs->wifi_password); const NetworkTransition transition = _network->maintain((uint32_t)now, _obs->wifi_power_save); const NetworkMedium selected_medium = _network->medium(); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 04e27649..46ef18d4 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -2,7 +2,7 @@ #include "MeshCore.h" #include "helpers/bridges/BridgeBase.h" -#include "helpers/NetworkInterface.h" +#include "helpers/NetworkLink.h" #include #include #include @@ -525,7 +525,7 @@ private: // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt.json. // _prefs (held by BridgeBase) still provides upstream fields (freq/sf/node_name…). MQTTPrefs* _obs = nullptr; - NetworkInterface* _network = nullptr; + NetworkLink* _network = nullptr; public: MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mgr, mesh::RTCClock *rtc, mesh::LocalIdentity *identity); diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 5c75a1f1..510d9b75 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include @@ -232,7 +232,7 @@ bool WebConfigServer::getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t } if (ip && ip_len > 0) { const IPAddress address = w->_mode == MODE_LAN - ? activeNetworkInterface().localIP() : WiFi.softAPIP(); + ? activeNetworkLink().localIP() : WiFi.softAPIP(); snprintf(ip, ip_len, "%s", address.toString().c_str()); } return true; @@ -291,7 +291,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { _mode = MODE_SETUP; _initial_setup = !mqttNetworkSetupComplete(_obs); createServer(); - activeNetworkInterface().lockSwitching(); + activeNetworkLink().lockSwitching(); _network_locked = true; _was_setup_ap = true; _last_activity = millis(); @@ -311,10 +311,10 @@ bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[ HttpPort80Lease::ownerName()); return false; } - activeNetworkInterface().lockSwitching(); + activeNetworkLink().lockSwitching(); _network_locked = true; - if (!activeNetworkInterface().isConnected() || ip == IPAddress()) { - activeNetworkInterface().unlockSwitching(); + if (!activeNetworkLink().isConnected() || ip == IPAddress()) { + activeNetworkLink().unlockSwitching(); _network_locked = false; HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); strcpy(reply, "Err: selected network not connected"); @@ -324,7 +324,7 @@ bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[ uint8_t session_entropy[sizeof(_session_secret) + 6]; const size_t entropy_size = sizeof(_session_secret) + (_initial_setup ? 6 : 0); if (!fillRandomBytes(session_entropy, entropy_size)) { - activeNetworkInterface().unlockSwitching(); + activeNetworkLink().unlockSwitching(); _network_locked = false; HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); strcpy(reply, "Err: secure random source unavailable"); @@ -445,7 +445,7 @@ void WebConfigServer::finalizeTeardown() { _setup_code[0] = 0; _setup_reminder_at = 0; if (_network_locked) { - activeNetworkInterface().unlockSwitching(); + activeNetworkLink().unlockSwitching(); _network_locked = false; } HttpPort80Lease::release(HttpPort80Lease::Owner::WebConfig); @@ -474,7 +474,7 @@ void WebConfigServer::tick(uint32_t now) { if (_mode == MODE_LAN && _initial_setup && _setup_reminder_at != 0 && (int32_t)(now - _setup_reminder_at) >= 0) { Serial.printf("WC: Ethernet setup http://%s/ code %s\n", - activeNetworkInterface().localIP().toString().c_str(), + activeNetworkLink().localIP().toString().c_str(), _setup_code); _setup_reminder_at = now + 60000; if (_setup_reminder_at == 0) _setup_reminder_at = 1; @@ -581,8 +581,8 @@ void WebConfigServer::drainBatch(uint32_t now) { return; // more commands next tick } } - if (_initial_setup && _admin_pwd_set && _batch_all_ok && - (_mode == MODE_LAN || _obs->wifi_ssid[0] != '\0')) { + // Wi-Fi onboarding is recorded by the stored SSID; only Ethernet LAN setup needs the marker. + if (_initial_setup && _admin_pwd_set && _batch_all_ok && _mode == MODE_LAN) { if (_cb->onInitialSetupComplete()) { _initial_setup = false; _setup_code[0] = 0; diff --git a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp index 3427bb60..d94c658c 100644 --- a/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp +++ b/test/test_mqtt_prefs_serializer/test_mqtt_prefs_serializer.cpp @@ -485,6 +485,20 @@ TEST(MQTTPrefsSerializer, UnknownGroupsAreIgnoredSoAppendedKeysAreDowngradeSafe) EXPECT_EQ(45, prefs.display_timeout_secs); } +TEST(MQTTPrefsSerializer, UnknownKeysInsideAKnownGroupAreDowngradeSafe) { + // Firmware that predates wifi.setup_complete sees it as an unknown key in a + // known group; the rest of the group must still load. + MQTTPrefs prefs = defaults(); + InputStream input( + "{version:1,wifi:{ssid:\"home\",future_key:1,power_save:2}}"); + MQTTPrefsSerializer serializer(&prefs); + ASSERT_TRUE(serializer.loadSerial(input)); + bool repaired = false; + ASSERT_TRUE(serializer.apply(&repaired)); + EXPECT_STREQ("home", prefs.wifi_ssid); + EXPECT_EQ(2, prefs.wifi_power_save); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 62ad163f284a4e5105030c911d871c17dff15048 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:14:23 -0700 Subject: [PATCH 76/93] feat(thinknode-m7): make the MQTT observer envs Ethernet-preferred Fold the CH390 overlay and NETWORK_PREFER_ETHERNET into ThinkNode_M7_{repeater,room_server}_observer_mqtt and drop the separate *_observer_mqtt_ethernet twins. The env names (and so the OTA manifests) are unchanged, so existing M7 observers OTA into the Ethernet-preferred image; without a cable they select stored Wi-Fi after a brief boot probe. --- MQTT_IMPLEMENTATION.md | 13 +++++++---- variants/thinknode_m7/platformio.ini | 35 +++++++--------------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 550513ec..aeb4ebc3 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -225,11 +225,14 @@ pio run -e ThinkNode_M7_repeater_observer_mqtt pio run -e ThinkNode_M7_room_server_observer_mqtt ``` -**ThinkNode M7 — WiFi only:** the M7 has an onboard CH390 Ethernet controller, and -`ThinkNode_M7_companion_radio_ethernet` uses it, but the MQTT bridge's link -management is bound to the WiFi station API, so the observer envs uplink over WiFi. -See `UPSTREAM_BUGS.md` for the Ethernet gap. The board has PSRAM, so these builds -get neighbors publication (`WITH_MQTT_NEIGHBORS`) automatically. +**ThinkNode M7 — Ethernet preferred, WiFi fallback:** the observer envs use the onboard +CH390 Ethernet when it has a DHCP lease at boot, otherwise the stored WiFi network, and +switch between the two at runtime (`get link.status`, `get link.diag`). Nodes without a +cable behave like WiFi observers after a brief (about 1 s) Ethernet probe at boot. The DHCP hostname is +`meshcore-` on either medium. A first boot on Ethernet with no WiFi configured +opens WebConfig on the LAN with a one-time login code printed on serial (and shown on +the display), and requires replacing the admin password. The board has PSRAM, so these +builds get neighbors publication (`WITH_MQTT_NEIGHBORS`) automatically. **TLora naming:** The env prefix `LilyGo_TLora_V2_1_1_6` is LilyGo’s **T-LoRa V2.1–1.6** board (SX1276); PlatformIO selects **`ttgo-lora32-v1`** (TTGO LoRa32 V1.0). **MQTT observer** envs extend a slim base **without** `sensor_base` so the image fits `min_spiffs`; **all other** `LilyGo_TLora_V2_1_1_6_*` targets still use optional I2C environmental sensors as before. The **`lilygo_tlora_c6`** variant is separate hardware (ESP32-C6). diff --git a/variants/thinknode_m7/platformio.ini b/variants/thinknode_m7/platformio.ini index 620fe390..60945e04 100644 --- a/variants/thinknode_m7/platformio.ini +++ b/variants/thinknode_m7/platformio.ini @@ -191,10 +191,9 @@ extends = ThinkNode_M7 build_src_filter = ${ThinkNode_M7.build_src_filter} +<../examples/kiss_modem/> -; Wi-Fi remains the default observer transport. Ethernet twins below select the -; onboard CH390 through the Ethernet-preferred network manager without enabling the legacy CLI -; transport. The board has PSRAM, so MAX_NEIGHBOURS enables -; WITH_MQTT_NEIGHBORS (see MQTTBridge.h). +; MQTT observers prefer the onboard CH390 Ethernet and fall back to Wi-Fi, without +; enabling the legacy Ethernet CLI transport (no ETHERNET_ENABLED). The board has +; PSRAM, so MAX_NEIGHBOURS enables WITH_MQTT_NEIGHBORS (see MQTTBridge.h). [env:ThinkNode_M7_repeater_observer_mqtt] extends = ThinkNode_M7 extra_scripts = @@ -204,6 +203,8 @@ board_ssl_cert_source = adafruit-full board_build.embed_files = src/certs/x509_crt_bundle.bin build_flags = ${ThinkNode_M7.build_flags} + ${ThinkNode_M7_ch390.build_flags} + -D NETWORK_PREFER_ETHERNET=1 -D ADVERT_NAME='"MQTT Observer"' -D ADVERT_LAT=0.0 -D ADVERT_LON=0.0 @@ -231,6 +232,7 @@ build_src_filter = ${ThinkNode_M7.build_src_filter} +<../examples/simple_repeater> lib_deps = ${ThinkNode_M7.lib_deps} + ${ThinkNode_M7_ch390.lib_deps} ${esp32_ota.lib_deps} elims/PsychicMqttClient@^0.2.4 bblanchon/ArduinoJson @ 7.4.3 @@ -239,17 +241,6 @@ lib_deps = paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent -[env:ThinkNode_M7_repeater_observer_mqtt_ethernet] -extends = env:ThinkNode_M7_repeater_observer_mqtt -build_flags = - ${env:ThinkNode_M7_repeater_observer_mqtt.build_flags} - ${ThinkNode_M7_ch390.build_flags} - -D NETWORK_PREFER_ETHERNET=1 -build_src_filter = ${env:ThinkNode_M7_repeater_observer_mqtt.build_src_filter} -lib_deps = - ${env:ThinkNode_M7_repeater_observer_mqtt.lib_deps} - ${ThinkNode_M7_ch390.lib_deps} - [env:ThinkNode_M7_room_server_observer_mqtt] extends = ThinkNode_M7 extra_scripts = @@ -259,6 +250,8 @@ board_ssl_cert_source = adafruit-full board_build.embed_files = src/certs/x509_crt_bundle.bin build_flags = ${ThinkNode_M7.build_flags} + ${ThinkNode_M7_ch390.build_flags} + -D NETWORK_PREFER_ETHERNET=1 -D ADVERT_NAME='"ThinkNode M7 Room Observer"' -D ADVERT_LAT=0.0 -D ADVERT_LON=0.0 @@ -287,6 +280,7 @@ build_src_filter = ${ThinkNode_M7.build_src_filter} +<../examples/simple_room_server> lib_deps = ${ThinkNode_M7.lib_deps} + ${ThinkNode_M7_ch390.lib_deps} ${esp32_ota.lib_deps} elims/PsychicMqttClient@^0.2.4 bblanchon/ArduinoJson @ 7.4.3 @@ -294,14 +288,3 @@ lib_deps = JChristensen/Timezone paulstoffregen/Time@1.6.1 0neblock/SNMP_Agent - -[env:ThinkNode_M7_room_server_observer_mqtt_ethernet] -extends = env:ThinkNode_M7_room_server_observer_mqtt -build_flags = - ${env:ThinkNode_M7_room_server_observer_mqtt.build_flags} - ${ThinkNode_M7_ch390.build_flags} - -D NETWORK_PREFER_ETHERNET=1 -build_src_filter = ${env:ThinkNode_M7_room_server_observer_mqtt.build_src_filter} -lib_deps = - ${env:ThinkNode_M7_room_server_observer_mqtt.lib_deps} - ${ThinkNode_M7_ch390.lib_deps} From 7aed59075b07af9906efb82c6dc8275393778b82 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:23:12 -0700 Subject: [PATCH 77/93] fix(network): bound link-transition teardown and gate Ethernet restarts - Link down/switch edges now softDisconnect() every started slot instead of disconnect(). The full stop could wait forever for a DISCONNECTED event on the Core 0 MQTT task, and it destroyed and recreated every slot's esp-mqtt task on each Wi-Fi drop, where the bridge previously stopped none (measured: 62 s deauth, five slots, zero stops). softDisconnect() is bounded and keeps the task; reconnectSlotClient() then calls reconnect() on the new route. Rename the transition action to disconnect_started_slots to match. - Ethernet no-IP recovery and init retries rebuild the CH390 netif, so they now run inside the route-switch mutation gate. An OTA/WebConfig lock taken after maintain() samples the lock count can no longer have the interface torn down underneath it. --- MQTT_IMPLEMENTATION.md | 4 ++-- src/helpers/NetworkLink.cpp | 14 ++++++++++--- src/helpers/NetworkPolicy.h | 2 +- src/helpers/bridges/MQTTBridge.cpp | 20 ++++++++----------- .../test_network_policy.cpp | 8 ++++---- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index aeb4ebc3..67ecebea 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -874,8 +874,8 @@ the radio actually performs in that case. selection log includes the measured probe duration. - CH390 startup initializes Arduino's shared network event runtime without associating WiFi; this keeps the framework's DNS and TLS hostname paths safe when Ethernet wins directly. -- Ethernet/WiFi transitions are logged. A lost or changed route stops every started MQTT - client, including one whose disconnect callback arrived first. When the same link returns, +- Ethernet/WiFi transitions are logged. A lost or changed route closes every live MQTT + transport with a bounded disconnect that keeps the client task. When the same link returns, each slot gets one immediate attempt at its current backoff rung (a tripped circuit breaker gets one immediate probe); a switch to the other medium also clears backoff and breakers - WiFi credentials changed at runtime (`set wifi.ssid` / `set wifi.pwd`) are used on the next diff --git a/src/helpers/NetworkLink.cpp b/src/helpers/NetworkLink.cpp index 10dfb23a..6314d181 100644 --- a/src/helpers/NetworkLink.cpp +++ b/src/helpers/NetworkLink.cpp @@ -573,8 +573,10 @@ class AutomaticNetworkLink final : public NetworkLink { _ethernet_retry_attempt.load(std::memory_order_relaxed); if (NetworkPolicy::ethernetInitRetryDue( attempt, now_ms, - _last_ethernet_init_attempt.load(std::memory_order_relaxed))) { + _last_ethernet_init_attempt.load(std::memory_order_relaxed)) && + beginUnlockedMutation()) { startOrRetryEthernet(now_ms, attempt != 0); + endUnlockedMutation(); } } // bootstrap() owns the initial choice. MQTT begin() is intentionally @@ -656,19 +658,25 @@ class AutomaticNetworkLink final : public NetworkLink { } else { _ethernet_no_ip_since.store(0, std::memory_order_relaxed); } + // Restarts rebuild the CH390 netif, so they take the same mutation gate as + // a route switch: an OTA/WebConfig lock taken after the sample above wins. if (!switching_locked && NetworkPolicy::ethernetNoIpRecoveryDue( ethernet_started, ethernet_link_up, _ethernet.isConnected(), - now_ms, _ethernet_no_ip_since.load(std::memory_order_relaxed))) { + now_ms, _ethernet_no_ip_since.load(std::memory_order_relaxed)) && + beginUnlockedMutation()) { ethernet_started = startOrRetryEthernet(now_ms, true); _ethernet_no_ip_since.store(0, std::memory_order_relaxed); + endUnlockedMutation(); } if (!ethernet_started && !switching_locked) { const uint8_t attempt = _ethernet_retry_attempt.load(std::memory_order_relaxed); if (NetworkPolicy::ethernetInitRetryDue( attempt, now_ms, - _last_ethernet_init_attempt.load(std::memory_order_relaxed))) { + _last_ethernet_init_attempt.load(std::memory_order_relaxed)) && + beginUnlockedMutation()) { ethernet_started = startOrRetryEthernet(now_ms, true); + endUnlockedMutation(); } } diff --git a/src/helpers/NetworkPolicy.h b/src/helpers/NetworkPolicy.h index 10d7da69..a4e7953a 100644 --- a/src/helpers/NetworkPolicy.h +++ b/src/helpers/NetworkPolicy.h @@ -29,7 +29,7 @@ enum class NetworkDiagnosticReason : uint8_t { namespace NetworkPolicy { struct MQTTTransitionActions { - bool stop_started_slots; + bool disconnect_started_slots; bool retry_disconnected_slots_now; bool reset_reconnect_backoff; }; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index a9d72775..0f3c67cc 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1986,10 +1986,8 @@ void MQTTBridge::teardownSlot(int index, bool force) { // A stopped client needs connect(): reconnect() is a documented no-op on one, so reaching // it here would strand the slot. The producer is a failed esp_mqtt_client_start(), which -// leaves _started false while initial_connect_done stays set. Not the WiFi-drop teardown, -// which only stops slots still marked connected: a publishing slot's socket fails first, so -// the guard skips it — measured across a 62 s deauth, five slots, zero stops. An idle slot -// with no traffic to fail on is the one case that could still reach here that way. +// leaves _started false while initial_connect_done stays set. Not a network transition, +// which only softDisconnect()s and keeps the client task. void MQTTBridge::reconnectSlotClient(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; @@ -2760,16 +2758,14 @@ bool MQTTBridge::handleNetworkConnection(unsigned long now) { const NetworkPolicy::MQTTTransitionActions actions = NetworkPolicy::mqttActions(transition); - if (actions.stop_started_slots) { - // Broker ownership stays in the bridge. The physical adapter reports the - // edge; the bridge explicitly stops every started slot instead of waiting - // for eventual socket timeouts. Do not gate this on slot.connected: the - // ESP-MQTT disconnect callback can clear that flag before the network edge - // reaches this task, but its client task and transport can still be alive. + if (actions.disconnect_started_slots) { + // Close each live transport now instead of waiting for socket timeouts, but + // keep the esp-mqtt task: softDisconnect() is bounded where a full stop can + // wait forever, and reconnectSlotClient() then reconnects on the new route. for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].client && _slots[i].client->isStarted()) { - MQTT_DEBUG_PRINTLN("MQTT%d stopping for network transition", i + 1); - _slots[i].client->disconnect(); + MQTT_DEBUG_PRINTLN("MQTT%d disconnecting for network transition", i + 1); + _slots[i].client->softDisconnect(); } _slots[i].connected = false; _slots[i].connected_at_ms = 0; diff --git a/test/test_network_policy/test_network_policy.cpp b/test/test_network_policy/test_network_policy.cpp index 7a057918..3558e1e3 100644 --- a/test/test_network_policy/test_network_policy.cpp +++ b/test/test_network_policy/test_network_policy.cpp @@ -4,28 +4,28 @@ TEST(NetworkPolicy, MqttDownDisconnectsSlotsWithoutRequestingImmediateRetry) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Down); - EXPECT_TRUE(actions.stop_started_slots); + EXPECT_TRUE(actions.disconnect_started_slots); EXPECT_FALSE(actions.retry_disconnected_slots_now); EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, MqttUpRetriesWithoutClearingBrokerCircuitBreaker) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Up); - EXPECT_FALSE(actions.stop_started_slots); + EXPECT_FALSE(actions.disconnect_started_slots); EXPECT_TRUE(actions.retry_disconnected_slots_now); EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, NoTransitionHasNoMqttSideEffects) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::None); - EXPECT_FALSE(actions.stop_started_slots); + EXPECT_FALSE(actions.disconnect_started_slots); EXPECT_FALSE(actions.retry_disconnected_slots_now); EXPECT_FALSE(actions.reset_reconnect_backoff); } TEST(NetworkPolicy, LinkSwitchReconnectsMqttSlotsImmediately) { const auto actions = NetworkPolicy::mqttActions(NetworkTransition::Switched); - EXPECT_TRUE(actions.stop_started_slots); + EXPECT_TRUE(actions.disconnect_started_slots); EXPECT_TRUE(actions.retry_disconnected_slots_now); EXPECT_TRUE(actions.reset_reconnect_backoff); } From de19fccfd214c4f675d786ebe3851073fcbd13c5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:28:10 -0700 Subject: [PATCH 78/93] fix(network): cap the per-slot transition disconnect wait at 2 s The old route is already down or switched away when a transition fires, so waiting the default 5 s per slot for a DISCONNECTED event only delays recovery (up to 25 s across five slots). A client whose event is late is aborted by keepalive and retried by the normal backoff loop, the same as the token-renewal path's softDisconnect() + reconnect(). --- src/helpers/bridges/MQTTBridge.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 0f3c67cc..d530cc8a 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2736,6 +2736,8 @@ void MQTTBridge::checkConfigurationMismatch() { } } +static constexpr unsigned long kNetworkTransitionDisconnectMs = 2000; + bool MQTTBridge::handleNetworkConnection(unsigned long now) { const NetworkMedium previous_medium = _network->medium(); _network->updateWifiCredentials(_obs->wifi_ssid, _obs->wifi_password); @@ -2762,10 +2764,12 @@ bool MQTTBridge::handleNetworkConnection(unsigned long now) { // Close each live transport now instead of waiting for socket timeouts, but // keep the esp-mqtt task: softDisconnect() is bounded where a full stop can // wait forever, and reconnectSlotClient() then reconnects on the new route. + // The old route is gone, so a short wait suffices; a client whose event is + // late is aborted by keepalive and picked up by the normal backoff retry. for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].client && _slots[i].client->isStarted()) { MQTT_DEBUG_PRINTLN("MQTT%d disconnecting for network transition", i + 1); - _slots[i].client->softDisconnect(); + _slots[i].client->softDisconnect(kNetworkTransitionDisconnectMs); } _slots[i].connected = false; _slots[i].connected_at_ms = 0; From eb39ca59e7c6b8c34fc2262b1df46760b71e1eb5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:35:49 -0700 Subject: [PATCH 79/93] fix(webconfig): persist the admin password before completing Ethernet onboarding The upstream `password` command saves /prefs.json without reporting the result, and WebConfig overwrites its reply with "OK". If that write failed while the later /mqtt.json write succeeded, wifi.setup_complete was set and the factory password came back after reboot with first-run setup suppressed. onInitialSetupComplete() now re-saves /prefs.json and records completion only when it succeeds; otherwise the batch fails, setup stays open, and no reboot is queued. --- examples/simple_repeater/MyMesh.h | 2 ++ examples/simple_room_server/MyMesh.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 6cc179d8..99394a47 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -508,6 +508,8 @@ public: } void onConfigBatchEnd() override; bool onInitialSetupComplete() override { + // The password command does not report its save; make it durable first. + if (!_cli.savePrefs(_fs)) return false; MQTTPrefs* obs = _cli.getObserverPrefs(); obs->network_setup_complete = 1; if (_cli.saveObserverPrefs(_fs)) return true; diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index f5e0be7e..594450f8 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -498,6 +498,8 @@ public: } void onConfigBatchEnd() override; bool onInitialSetupComplete() override { + // The password command does not report its save; make it durable first. + if (!_cli.savePrefs(_fs)) return false; MQTTPrefs* obs = _cli.getObserverPrefs(); obs->network_setup_complete = 1; if (_cli.saveObserverPrefs(_fs)) return true; From 3c39908720ef2527b1898e6a3e89fd0175da78cb Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 10:53:39 -0700 Subject: [PATCH 80/93] fix(webconfig): read the LAN address after pinning the route startLanMode() took the caller's IP, read before the route lock. A Wi-Fi/Ethernet switch in between left WebConfig locked to the new link while advertising the old link's address. It now reads the selected link's address after lockSwitching(), as startOTAUpdate() already does. --- examples/simple_repeater/MyMesh.cpp | 3 +-- examples/simple_room_server/MyMesh.cpp | 3 +-- src/helpers/esp32/WebConfigServer.cpp | 4 +++- src/helpers/esp32/WebConfigServer.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 29518694..ff70df6f 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1512,8 +1512,7 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { } _webconfig->startSetupMode(reply); } else if (activeNetworkLink().isConnected()) { - _webconfig->startLanMode(activeNetworkLink().localIP(), - !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); + _webconfig->startLanMode(!mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { _webconfig->startSetupMode(reply); } else { diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index f5b41860..35bbd632 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1314,8 +1314,7 @@ bool MyMesh::startWebConfig(bool force_ap, char* reply) { } _webconfig->startSetupMode(reply); } else if (activeNetworkLink().isConnected()) { - _webconfig->startLanMode(activeNetworkLink().localIP(), - !mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); + _webconfig->startLanMode(!mqttNetworkSetupComplete(_cli.getObserverPrefs()), reply); } else if (!mqttNetworkSetupComplete(_cli.getObserverPrefs())) { _webconfig->startSetupMode(reply); } else { diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 510d9b75..bb9bdc36 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -301,7 +301,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { return true; } -bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[]) { +bool WebConfigServer::startLanMode(bool initial_setup, char reply[]) { if (_mode != MODE_OFF || _stopping) { strcpy(reply, "Err: webconfig busy"); return false; @@ -313,6 +313,8 @@ bool WebConfigServer::startLanMode(IPAddress ip, bool initial_setup, char reply[ } activeNetworkLink().lockSwitching(); _network_locked = true; + // Read the address only once the route is pinned, so it names the locked link. + const IPAddress ip = activeNetworkLink().localIP(); if (!activeNetworkLink().isConnected() || ip == IPAddress()) { activeNetworkLink().unlockSwitching(); _network_locked = false; diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index b19ca810..16ff47ba 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -84,7 +84,7 @@ public: static bool isRebootPending(); bool startSetupMode(char reply[]); // open SoftAP + DNS captive portal - bool startLanMode(IPAddress ip, bool initial_setup, char reply[]); + bool startLanMode(bool initial_setup, char reply[]); // bind to the selected network void requestStop(); // stop listening and detach this session void tick(uint32_t now); // call every loop iteration From 946d6c4f1e853f4b5040997e300f5bc93abafa06 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 11:27:01 -0700 Subject: [PATCH 81/93] fix(wifi): keep the legacy power-save default off; store explicit min as 3 Stored 0 was the shipped default from 2026-01-02 to 2026-03-28, and every association path has run it with power save off since. Reading it as MIN_MODEM, as F11 did, would have put every node set up in that window into modem sleep on its next association. 0 now reads as `none`, and `set wifi.powersave min` stores a new value 3. /mqtt.json accepts 0..3; older firmware repairs 3 to `none` on load. Binary snapshots are no longer written, so 3 never reaches a legacy layout. --- src/helpers/MQTTPrefsSerializer.h | 3 ++- src/helpers/WifiPowerSavePolicy.h | 15 +++++++++------ .../test_wifi_power_save_policy.cpp | 14 ++++++++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/helpers/MQTTPrefsSerializer.h b/src/helpers/MQTTPrefsSerializer.h index f783b52c..e256e6a9 100644 --- a/src/helpers/MQTTPrefsSerializer.h +++ b/src/helpers/MQTTPrefsSerializer.h @@ -4,6 +4,7 @@ #include #include #include +#include #ifdef WITH_MQTT_BRIDGE @@ -50,7 +51,7 @@ class MQTTPrefsSerializer : public ConfigSerializer { public: explicit WifiPrefs(MQTTPrefs* prefs) : _prefs(prefs), _power_save(prefs->wifi_power_save) {} bool apply(bool* repaired) { - if (_power_save < 0 || _power_save > 2) { + if (_power_save < 0 || _power_save > WifiPowerSavePolicy::kMaxStoredValue) { _power_save = 1; *repaired = true; } diff --git a/src/helpers/WifiPowerSavePolicy.h b/src/helpers/WifiPowerSavePolicy.h index bdebd57e..bb8a3571 100644 --- a/src/helpers/WifiPowerSavePolicy.h +++ b/src/helpers/WifiPowerSavePolicy.h @@ -11,15 +11,18 @@ // to `min` silently ran with power save off after its first reconnect while // `get wifi.powersave` still said min. // -// Stored values are fleet state — never renumber them. The product default is -// `none`, which is a *default* (MQTTDefaults.h), not a reinterpretation of an -// operator's explicit `min`. +// Stored values are fleet state — never renumber them. 0 was the shipped +// default from 2026-01-02 to 2026-03-28, and every association path ran it with +// power save off, so it keeps meaning `none`; an explicit `min` is stored as 3. +// Older firmware repairs 3 to `none` when it loads /mqtt.json. namespace WifiPowerSavePolicy { enum StoredValue : uint8_t { - kMin = 0, // WIFI_PS_MIN_MODEM - kNone = 1, // WIFI_PS_NONE (default) - kMax = 2, // WIFI_PS_MAX_MODEM + kLegacyDefault = 0, // WIFI_PS_NONE: the old default, never an operator choice + kNone = 1, // WIFI_PS_NONE (default) + kMax = 2, // WIFI_PS_MAX_MODEM + kMin = 3, // WIFI_PS_MIN_MODEM + kMaxStoredValue = kMin, }; // Mirrors wifi_ps_type_t. MQTTBridge.cpp static_asserts these against the SDK. diff --git a/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp b/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp index 66803aa5..35eb39f9 100644 --- a/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp +++ b/test/test_wifi_power_save_policy/test_wifi_power_save_policy.cpp @@ -13,8 +13,18 @@ TEST(WifiPowerSavePolicy, StoredValuesMapToOneModeEach) { EXPECT_EQ(kModeMaxModem, modeFor(kMax)); } +// Stored 0 was the default for nodes set up 2026-01-02..03-28, and they have +// always run with power save off; reading it as `min` would put them to sleep. +TEST(WifiPowerSavePolicy, LegacyDefaultKeepsPowerSaveOff) { + EXPECT_EQ(kModeNone, modeFor(kLegacyDefault)); + EXPECT_STREQ("none", nameFor(kLegacyDefault)); + uint8_t parsed = 0xFF; + ASSERT_TRUE(parseName("min", &parsed)); + EXPECT_NE(kLegacyDefault, parsed); +} + TEST(WifiPowerSavePolicy, NamesRoundTripWithStoredValues) { - for (uint8_t stored = 0; stored <= 2; stored++) { + for (uint8_t stored = kNone; stored <= kMaxStoredValue; stored++) { uint8_t parsed = 0xFF; ASSERT_TRUE(parseName(nameFor(stored), &parsed)) << "stored " << (int)stored; EXPECT_EQ(stored, parsed); @@ -25,7 +35,7 @@ TEST(WifiPowerSavePolicy, NamesRoundTripWithStoredValues) { // A byte outside the stored range must read as the product default, not as // whatever mode happens to sit at that index. TEST(WifiPowerSavePolicy, OutOfRangeStoredValueReadsAsDefault) { - for (int stored = 3; stored <= 255; stored++) { + for (int stored = kMaxStoredValue + 1; stored <= 255; stored++) { EXPECT_EQ(kModeNone, modeFor((uint8_t)stored)) << "stored " << stored; EXPECT_STREQ("none", nameFor((uint8_t)stored)); } From f50a6a3940320a96f23cf70b07490cef9129e6d0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 11:27:01 -0700 Subject: [PATCH 82/93] fix(mqtt): resume after a late stop ack; bound the hourly NTP refresh - A stop that timed out into StopUnproven refused begin(), and only a later begin() honoured the task's late acknowledgement. Nothing called it, so restartBridge() (any `set mqtt...` restart) or an aborted OTA left the observer offline for the rest of the boot. MyMesh now records that a start was refused and restarts the bridge once stopAcknowledgedLate() reports the ack. The OTA-abort alert no longer claims "bridge resumed" when the restart was refused. - refreshNTP() now runs the validated probe with one attempt per server. With two attempts and a 1 s pause it blocked the MQTT task ~18 s every hour on networks that drop UDP/123, where the old async SNTP cost nothing. Loop comment and docs updated to match. --- MQTT_IMPLEMENTATION.md | 2 +- examples/simple_repeater/MyMesh.cpp | 12 +++++++++++- examples/simple_repeater/MyMesh.h | 7 +++++++ examples/simple_room_server/MyMesh.cpp | 9 +++++++++ examples/simple_room_server/MyMesh.h | 7 +++++++ src/helpers/bridges/MQTTBridge.cpp | 25 +++++++++++++++++-------- src/helpers/bridges/MQTTBridge.h | 5 ++++- 7 files changed, 56 insertions(+), 11 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 56f43569..b4671487 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -883,7 +883,7 @@ the radio actually performs in that case. ### NTP Time Synchronization - Automatic time synchronization with NTP servers (required for JWT authentication) - Default primary: `pool.ntp.org`; built-in fallbacks (tried sequentially on failure): `time.google.com`, `time.cloudflare.com`, `time.aws.com`, `time.nist.gov` -- Periodic time updates (every hour) on the effective primary only; system time is kept in UTC +- Periodic time updates (every hour) through the same validated probe, one attempt per server in list order; system time is kept in UTC - Replies are validated before they are trusted: the datagram must be a full-length NTPv3/v4 server reply from the queried address and port, from a synchronised server (stratum 1-15, no leap alarm), echoing the random transmit timestamp of the request, with a plausible epoch. Anything else is discarded and the clock, the RTC and JWT issuance are left alone - Configure and diagnose with `set mqtt.ntp` / `get mqtt.ntp` / `get mqtt.ntp.diag` — see [MQTT Shared Commands](#mqtt-shared-commands) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 75d9fb4e..93dbe279 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1699,6 +1699,15 @@ void MyMesh::loop() { // MQTT processing runs in a separate FreeRTOS task on Core 0, so we don't call bridge.loop() here mesh::Mesh::loop(); +#ifdef WITH_MQTT_BRIDGE + // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; + // once that late ack lands, restart the bridge that was meant to be running. + if (_bridge_resume_pending && bridge && bridge->stopAcknowledgedLate()) { + Serial.println("MQTT: stop acknowledged late - resuming bridge"); + setBridgeState(true); + } +#endif + #ifdef WITH_BRIDGE // bridge.loop() is now handled by FreeRTOS task on Core 0 - no need to call it here #endif @@ -1752,8 +1761,9 @@ void MyMesh::loop() { // resume the bridge instead of flashing under uncertain ownership. if (bridge && !bridge->canFlashAfterStop()) { Serial.println("OTA: aborted, MQTT stop did not complete cleanly - resuming bridge"); - otaAlert("OTA aborted: MQTT stop unclean, bridge resumed"); setBridgeState(true); + otaAlert(bridge->isRunning() ? "OTA aborted: MQTT stop unclean, bridge resumed" + : "OTA aborted: MQTT stop unproven, bridge resumes when it completes"); } else if (!_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); char ota_alert_msg[160]; diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 539a1715..3dd8587b 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -137,6 +137,8 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks ESPNowBridge bridge; #elif defined(WITH_MQTT_BRIDGE) MQTTBridge* bridge; + // begin() was refused after an unproven stop; restart once the task acknowledges. + bool _bridge_resume_pending = false; #endif #ifdef WITH_SNMP MeshSNMPAgent _snmp_agent; @@ -368,6 +370,7 @@ public: bridge->begin(); #ifdef WITH_MQTT_BRIDGE _alerter.setBridge(bridge); + _bridge_resume_pending = !bridge->isRunning() && bridge->isStopUnproven(); #endif } else @@ -375,6 +378,7 @@ public: bridge->end(); #ifdef WITH_MQTT_BRIDGE _alerter.setBridge(nullptr); + _bridge_resume_pending = false; #endif } } @@ -400,6 +404,9 @@ public: bridge->setStatsSources(this, _radio, _cli.getBoard(), _ms); #endif bridge->begin(); +#ifdef WITH_MQTT_BRIDGE + _bridge_resume_pending = !bridge->isRunning() && bridge->isStopUnproven(); +#endif } void restartBridgeSlot(int slot) override { diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index e6bed7d6..956b5959 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1514,6 +1514,15 @@ void MyMesh::loop() { // Check radio FIRST to ensure we don't miss incoming packets // MQTT processing can take time, so we prioritize radio reception mesh::Mesh::loop(); + +#ifdef WITH_MQTT_BRIDGE + // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; + // once that late ack lands, restart the bridge that was meant to be running. + if (_bridge_resume_pending && bridge && bridge->stopAcknowledgedLate()) { + Serial.println("MQTT: stop acknowledged late - resuming bridge"); + setBridgeState(true); + } +#endif #ifdef WITH_MQTT_BRIDGE // bridge.loop() is now handled by FreeRTOS task on Core 0 - no need to call it here #endif diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 47f2f68d..0139fa5a 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -205,6 +205,8 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks #endif #ifdef WITH_MQTT_BRIDGE MQTTBridge* bridge; + // begin() was refused after an unproven stop; restart once the task acknowledges. + bool _bridge_resume_pending = false; #endif #ifdef WITH_SNMP MeshSNMPAgent _snmp_agent; @@ -385,6 +387,7 @@ public: bridge->begin(); #ifdef WITH_MQTT_BRIDGE _alerter.setBridge(bridge); + _bridge_resume_pending = !bridge->isRunning() && bridge->isStopUnproven(); #endif } else @@ -392,6 +395,7 @@ public: bridge->end(); #ifdef WITH_MQTT_BRIDGE _alerter.setBridge(nullptr); + _bridge_resume_pending = false; #endif } } @@ -416,6 +420,9 @@ public: bridge->setStatsSources(this, _radio, _cli.getBoard(), _ms); #endif bridge->begin(); +#ifdef WITH_MQTT_BRIDGE + _bridge_resume_pending = !bridge->isRunning() && bridge->isStopUnproven(); +#endif } void restartBridgeSlot(int slot) override { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index d1c866e2..2f8d69ef 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -464,6 +464,14 @@ void MQTTBridge::applyWifiPowerSave() { bool MQTTBridge::stopUnprovenLatched() { return s_stop_unproven; } +bool MQTTBridge::stopAcknowledgedLate() const { +#ifdef ESP_PLATFORM + return _lifecycle.isStopUnproven() && _stop_acked.load(std::memory_order_acquire); +#else + return false; +#endif +} + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } @@ -1644,9 +1652,9 @@ void MQTTBridge::mqttTaskLoop() { // Periodic configuration check (throttled to avoid spam) checkConfigurationMismatch(); - // Periodic NTP refresh (every hour) — lightweight, non-blocking. - // Uses async SNTP instead of the heavy syncTimeWithNTP() which blocks Core 0 - // for up to 20+ seconds with DNS lookups, UDP sockets, and retry loops. + // Periodic NTP refresh (every hour): a validated probe, one attempt per + // server. Blocks this task for about one probe timeout per server that does + // not answer (see refreshNTP()). if (WiFi.status() == WL_CONNECTED && now - _last_ntp_sync > 3600000) { refreshNTP(); } @@ -4493,10 +4501,11 @@ void MQTTBridge::storeRawRadioData(const uint8_t* raw_data, int len, float snr, // let it set the system clock asynchronously — an acceptance path with NO // validation of its own (see the syncTimeWithNTP() note below), running every // hour for the life of the node. It now goes through the same validated probe -// as every other sync, which costs the MQTT task about a second per attempt and -// touches nothing until a reply passes every check. +// as every other sync and touches nothing until a reply passes every check. +// One attempt per server bounds the blocking walk on a network that drops +// UDP/123 to about one probe timeout per server, instead of three. void MQTTBridge::refreshNTP() { - syncTimeWithNTP(/*force=*/true, /*primary_only=*/false); + syncTimeWithNTP(/*force=*/true, /*primary_only=*/false, /*attempts_per_server=*/1); } // One validated NTP exchange with one named server, on a fresh ephemeral socket. @@ -4599,7 +4608,7 @@ bool MQTTBridge::probeNtpServer(const char* server, uint32_t min_epoch, return accepted; } -bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { +bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only, int attempts_per_server) { if (!WiFi.isConnected()) { MQTT_DEBUG_PRINTLN("Cannot sync time - WiFi not connected"); return false; @@ -4634,7 +4643,7 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { const uint32_t kMinValidEpoch = kNtpMinValidEpoch; const char* ntp_server_used = nullptr; - const int kMaxNtpRetriesPerServer = 2; + const int kMaxNtpRetriesPerServer = attempts_per_server > 0 ? attempts_per_server : 1; for (int s = 0; s < server_count && !ntp_ok; s++) { const char* server = servers[s]; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 8d180a48..e733044f 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -743,6 +743,9 @@ public: // the bridge is down, nothing was released, and it will not restart until the // task acknowledges late (pollLateStopAck) or the node reboots. bool isStopUnproven() const { return _lifecycle.isStopUnproven(); } + // The unproven stop has since been acknowledged, so begin() will release the + // withheld resources and start. Loop task only. + bool stopAcknowledgedLate() const; // Survives end() clearing the diagnostic singleton, so `get mqtt.status` can // still explain why a stopped bridge will not come back without a reboot. static bool stopUnprovenLatched(); @@ -787,7 +790,7 @@ public: * mistyped hostname fails fast instead of blocking through the whole fallback list. * Performs blocking NTP I/O and must only be called from the MQTT task (Core 0). * Other tasks (e.g. the CLI on Core 1) must use requestForcedNtpSync() instead. */ - bool syncTimeWithNTP(bool force = false, bool primary_only = false); + bool syncTimeWithNTP(bool force = false, bool primary_only = false, int attempts_per_server = 2); /** Request a forced NTP sync from another task (e.g. CLI on Core 1). Marshals the * work onto the MQTT task so all NTP I/O stays on Core 0, then blocks up to * timeout_ms for the result. Returns true if the sync succeeded, false on failure, From f72bdbad48f451c353443b1d090f3dfd56fc2636 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 11:48:43 -0700 Subject: [PATCH 83/93] fix(mqtt): record the renewed token as applied after a successful bounce The renewal bounce called reconnect()/connect() directly, so a successful bounce never advanced applied_token_expires_at. The renewal decision reads that value, so a renewed slot stayed due and, on a broker that enforces exp, minted and reconnected every minute for the rest of the token's life. Both the renewal bounce and the corrected-clock bounce now go through reconnectSlotClient(), which starts a stopped client, refuses a quarantined one, moves the slot to Starting and records the token in use. The corrected-clock no-bounce case settles the renewal as the renewal path already does. --- src/helpers/bridges/MQTTBridge.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 2f8d69ef..1e807d41 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2687,7 +2687,6 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns // to avoid internal heap leak/fragmentation from destroy/create cycles MQTT_DEBUG_PRINTLN("MQTT%d token renewal: reconnecting with fresh credentials", index + 1); MQTT_TRACE_HEAP("renewal:before-bounce", index); - esp_err_t bounce_result; if (slot.client->isStarted()) { // Keep the esp-mqtt task alive across the handshake. disconnect() // would stop it, returning its 6 KiB stack into the hole the two @@ -2699,15 +2698,13 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns index + 1, esp_err_to_name(soft)); } MQTT_TRACE_HEAP("renewal:after-disconnect", index); - slot.client->setCredentials(_jwt_username, slot.auth_token); - MQTT_TRACE_HEAP("renewal:after-credentials", index); - bounce_result = slot.client->reconnect(); - } else { - // Client was stopped (teardown/reconfigure). reconnect() is a no-op - // on a stopped client, so this path must start it. - slot.client->setCredentials(_jwt_username, slot.auth_token); - bounce_result = slot.client->connect(); } + slot.client->setCredentials(_jwt_username, slot.auth_token); + MQTT_TRACE_HEAP("renewal:after-credentials", index); + // Via the helper: it starts a stopped client, refuses a quarantined one, + // and on success records the fresh token as the one in use, which is + // what keeps the renewal from coming due again a minute later. + const esp_err_t bounce_result = reconnectSlotClient(index); if (bounce_result != ESP_OK) { // The fresh token is in the buffer but did not reach the // connection: the config transaction failed, or the client would @@ -4784,7 +4781,8 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only, int attempts_per MQTT_DEBUG_PRINTLN("MQTT%d soft disconnect did not complete (%s)", i + 1, esp_err_to_name(soft)); } - const esp_err_t rc = _slots[i].client->reconnect(); + // Via the helper, so the re-created token is recorded as applied. + const esp_err_t rc = reconnectSlotClient(i); if (rc != ESP_OK) { MQTT_DEBUG_PRINTLN("MQTT%d corrected-clock reconnect failed (%s)", i + 1, esp_err_to_name(rc)); @@ -4792,6 +4790,8 @@ bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only, int attempts_per } else { MQTT_DEBUG_PRINTLN("MQTT%d token re-created, no bounce (broker does not enforce exp)", i + 1); + // As in the renewal path: this broker ignores exp, so the renewal is settled. + _slots[i].applied_token_expires_at = _slots[i].token_expires_at; } } } From d180f097ea98bd279befa0fba9ec878d4ce24061 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 13:58:49 -0700 Subject: [PATCH 84/93] fix(mqtt): an explicit bridge stop cancels a pending late-ack resume setBridgeState(false) returned early when the bridge was already down, so `set bridge.enabled off` after a timed-out restart left the resume pending and MQTT restarted against the operator's choice once the task acknowledged. Disabling now clears the pending resume before that early return, and the resume also requires bridge_enabled. The StopUnproven log and `get mqtt.status` no longer say only "reboot to recover", since a late ack now recovers on its own. Hardware (Heltec V4, 5 live slots, 1 s test stop deadline): a restart went StopUnproven, the task acknowledged 27 s later and the bridge resumed with all slots reconnecting; with `set bridge.enabled off` in between it stayed stopped until `set bridge.enabled on`. --- examples/simple_repeater/MyMesh.cpp | 3 ++- examples/simple_repeater/MyMesh.h | 3 +++ examples/simple_room_server/MyMesh.cpp | 3 ++- examples/simple_room_server/MyMesh.h | 3 +++ src/helpers/bridges/MQTTBridge.cpp | 6 +++--- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 93dbe279..f9bdf7a1 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1702,7 +1702,8 @@ void MyMesh::loop() { #ifdef WITH_MQTT_BRIDGE // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; // once that late ack lands, restart the bridge that was meant to be running. - if (_bridge_resume_pending && bridge && bridge->stopAcknowledgedLate()) { + if (_bridge_resume_pending && _prefs.bridge_enabled && bridge && + bridge->stopAcknowledgedLate()) { Serial.println("MQTT: stop acknowledged late - resuming bridge"); setBridgeState(true); } diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 3dd8587b..4b12f6d0 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -353,6 +353,9 @@ public: #endif if (!bridge) return; } +#ifdef WITH_MQTT_BRIDGE + if (!enable) _bridge_resume_pending = false; // an explicit stop cancels a pending resume +#endif if (enable == bridge->isRunning()) return; if (enable) { diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 956b5959..62fe821b 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1518,7 +1518,8 @@ void MyMesh::loop() { #ifdef WITH_MQTT_BRIDGE // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; // once that late ack lands, restart the bridge that was meant to be running. - if (_bridge_resume_pending && bridge && bridge->stopAcknowledgedLate()) { + if (_bridge_resume_pending && _prefs.bridge_enabled && bridge && + bridge->stopAcknowledgedLate()) { Serial.println("MQTT: stop acknowledged late - resuming bridge"); setBridgeState(true); } diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 0139fa5a..34cb2959 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -364,6 +364,9 @@ public: #endif if (!bridge) return; } +#ifdef WITH_MQTT_BRIDGE + if (!enable) _bridge_resume_pending = false; // an explicit stop cancels a pending resume +#endif if (enable == bridge->isRunning()) return; if (enable) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1e807d41..a21a75c6 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -272,7 +272,7 @@ void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPref const char* msgs = (obs && obs->mqtt_status_enabled) ? "on" : "off"; if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) { snprintf(buf, bufsize, "> msgs: %s (bridge %s)", msgs, - s_stop_unproven ? "stopped: previous stop unproven, reboot to recover" + s_stop_unproven ? "stopped: waiting for the previous stop to finish; reboot if it persists" : "not running"); return; } @@ -903,7 +903,7 @@ void MQTTBridge::begin() { // refuse. Recovery is the task finishing, or a reboot. pollLateStopAck(); if (!_lifecycle.mayRestart()) { - MQTT_DEBUG_PRINTLN("MQTT Bridge start refused: previous stop unproven (%s) - reboot to recover", + MQTT_DEBUG_PRINTLN("MQTT Bridge start refused: previous stop unproven (%s) - waiting for the task to acknowledge", MQTTLifecycle::stateName(_lifecycle.state())); return; } @@ -1230,7 +1230,7 @@ void MQTTBridge::end() { s_stop_unproven = _lifecycle.isStopUnproven(); if (_lifecycle.isStopUnproven()) { MQTT_DEBUG_PRINTLN("MQTT Bridge stop UNPROVEN after %lu ms: task did not acknowledge. " - "Nothing released, restart refused, OTA blocked - reboot to recover.", + "Nothing released, OTA blocked; restart waits for its late ack (reboot if it never comes).", (unsigned long)_lifecycle.stopTimeoutMs()); } else { MQTT_DEBUG_PRINTLN("MQTT Bridge stopped (clean)"); From 16562621a2ff9d6bf582010b2f7f76e4a98c4e0b Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 14:22:00 -0700 Subject: [PATCH 85/93] fix(mqtt): gate room-server OTA on an unproven stop; reap late acks - The room server checked canFlashAfterStop() only when the bridge was running at OTA time. After a timed-out restart the bridge reads as stopped while its unacknowledged task may still own TLS/client state, so an OTA could erase and write flash under it. It now refuses while the stop is unproven, after first reaping any late ack. The repeater already gated unconditionally. - MyMesh::loop() now reaps a late stop acknowledgement whenever it lands, releasing the withheld queue and buffers, and restarts only when a resume is pending and the bridge is enabled. Before, a bridge disabled during StopUnproven kept those resources until re-enabled or rebooted. pollLateStopAck() is public for this. - The wrapper's destructor no longer stops an already-stopped client: destroySlotClients() had just stopped it, so every shutdown logged five spurious "esp_mqtt_client_stop failed: ESP_FAIL" errors. Hardware (Heltec V4, 1 s test stop deadline): restart -> StopUnproven -> `set bridge.enabled off`; the late ack was reaped ("releasing withheld resources"), status read "not running", and `set bridge.enabled on` started cleanly without a second release. --- examples/simple_repeater/MyMesh.cpp | 15 ++++++----- examples/simple_room_server/MyMesh.cpp | 25 ++++++++++++------- .../src/PsychicMqttClient.cpp | 3 ++- src/helpers/bridges/MQTTBridge.h | 6 ++--- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index f9bdf7a1..a8a0b201 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1700,12 +1700,15 @@ void MyMesh::loop() { mesh::Mesh::loop(); #ifdef WITH_MQTT_BRIDGE - // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; - // once that late ack lands, restart the bridge that was meant to be running. - if (_bridge_resume_pending && _prefs.bridge_enabled && bridge && - bridge->stopAcknowledgedLate()) { - Serial.println("MQTT: stop acknowledged late - resuming bridge"); - setBridgeState(true); + // A timed-out stop keeps the bridge down until the MQTT task acknowledges it. + // Release the withheld resources whenever that late ack lands, and restart + // only a bridge that is still meant to be running. + if (bridge && bridge->stopAcknowledgedLate()) { + bridge->pollLateStopAck(); + if (_bridge_resume_pending && _prefs.bridge_enabled) { + Serial.println("MQTT: stop acknowledged late - resuming bridge"); + setBridgeState(true); + } } #endif diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 62fe821b..7c13018c 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1516,12 +1516,15 @@ void MyMesh::loop() { mesh::Mesh::loop(); #ifdef WITH_MQTT_BRIDGE - // A timed-out stop keeps the bridge down until the MQTT task acknowledges it; - // once that late ack lands, restart the bridge that was meant to be running. - if (_bridge_resume_pending && _prefs.bridge_enabled && bridge && - bridge->stopAcknowledgedLate()) { - Serial.println("MQTT: stop acknowledged late - resuming bridge"); - setBridgeState(true); + // A timed-out stop keeps the bridge down until the MQTT task acknowledges it. + // Release the withheld resources whenever that late ack lands, and restart + // only a bridge that is still meant to be running. + if (bridge && bridge->stopAcknowledgedLate()) { + bridge->pollLateStopAck(); + if (_bridge_resume_pending && _prefs.bridge_enabled) { + Serial.println("MQTT: stop acknowledged late - resuming bridge"); + setBridgeState(true); + } } #endif #ifdef WITH_MQTT_BRIDGE @@ -1619,14 +1622,18 @@ void MyMesh::loop() { drainOutbound(OTA_TX_DRAIN_TIMEOUT_MS); bool may_flash = true; + if (bridge) bridge->pollLateStopAck(); if (bridge_was_running) { setBridgeState(false); // OTA must not write after a forced/timed-out MQTT shutdown: its TLS/heap // ownership is uncertain until a subsequent clean start/stop cycle. may_flash = bridge && bridge->canFlashAfterStop(); - if (!may_flash) { - Serial.println("OTA: aborted, MQTT stop did not complete cleanly"); - } + } else if (bridge && bridge->isStopUnproven()) { + // Reads as stopped, but an unacknowledged MQTT task may still own TLS/client state. + may_flash = false; + } + if (!may_flash) { + Serial.println("OTA: aborted, MQTT stop did not complete cleanly"); } char ota_reply[160]; diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index ccd11341..01efb194 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -20,7 +20,8 @@ PsychicMqttClient::PsychicMqttClient() : _mqtt_cfg() PsychicMqttClient::~PsychicMqttClient() { - disconnect(); + // Owners stop the client before deleting it; stopping it again only logs ESP_FAIL. + if (_started) disconnect(); if (_client != nullptr) { esp_mqtt_client_destroy(_client); diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index e733044f..b88b35a6 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -602,9 +602,6 @@ private: void logMemoryStatus(); void refreshOriginFromPrefs(); void applyWifiPowerSave(); // one mapping, applied on every association - // Honours a stop acknowledgement that arrived after the deadline: releases the - // withheld resources and makes the bridge restartable. Loop task only. - void pollLateStopAck(); // begin()/end()-scoped PSRAM buffers. Each allocation is independent so a // transient heap shortage degrades to the existing stack fallback instead // of making the bridge unusable. @@ -746,6 +743,9 @@ public: // The unproven stop has since been acknowledged, so begin() will release the // withheld resources and start. Loop task only. bool stopAcknowledgedLate() const; + // Honours a stop acknowledgement that arrived after the deadline: releases the + // withheld resources and makes the bridge restartable. Loop task only. + void pollLateStopAck(); // Survives end() clearing the diagnostic singleton, so `get mqtt.status` can // still explain why a stopped bridge will not come back without a reboot. static bool stopUnprovenLatched(); From 6e7f3cdb52168b15817c5d9e8e125cbdbb750d1c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 15:31:51 -0700 Subject: [PATCH 86/93] fix(network): bound link transitions and restore each medium's DNS Two defects found by an adversarial review of the merge. A link transition no longer stops a slot whose connect attempt is in flight. Stopping one means esp_mqtt_client_stop(), which waits on the SDK's API mutex and its task's stopped event with no bound; the client task notices only when it returns from whatever transport call it is in. During teardown the bridge's StopUnproven timeout contains that, but during a transition nothing does, so a routine WiFi flap could freeze the sole MQTT worker for far longer than the 2 s this path advertises, with the bridge's own stop handshake queued behind it. The attempt is left to resolve instead: unlike a reconfigure, a transition changes neither endpoint nor credentials, so an attempt that completes is credited to the broker it actually reached and drops with the old route. Failing back to Ethernet left the node using the WiFi network's DNS server. lwIP keeps one global server list and IDF 4.4 has no per-interface retention, so the medium that leased last owns DNS for every socket. Each link now remembers the resolver its own DHCP lease installed and puts it back when it is selected again; a medium that never held a lease leaves the current resolver alone. Where the two networks are on different subnets this was a silent outage: the link read as connected while every broker and NTP hostname failed to resolve, until Ethernet's own DHCP renewal happened to fix it. --- MQTT_IMPLEMENTATION.md | 3 + src/helpers/NetworkLink.cpp | 60 ++++++++++++++++++- src/helpers/bridges/MQTTBridge.cpp | 39 +++++++----- .../test_mqtt_client_state.cpp | 31 ++++++---- 4 files changed, 102 insertions(+), 31 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 2184177f..d5eeb8bc 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -880,6 +880,9 @@ the radio actually performs in that case. report a connection that no longer exists. When the same link returns, each slot gets one immediate attempt at its current backoff rung (a tripped circuit breaker gets one immediate probe); a switch to the other medium also clears backoff and breakers +- Each medium's DHCP DNS servers are remembered when it gets its lease and restored when it is + selected again. lwIP keeps one global resolver list, so without this a node that fell back to + WiFi and then failed back to Ethernet would keep asking the WiFi network's DNS server - WiFi credentials changed at runtime (`set wifi.ssid` / `set wifi.pwd`) are used on the next reconnect attempt without a reboot - Packets are queued while a slot is disconnected and flushed when it recovers diff --git a/src/helpers/NetworkLink.cpp b/src/helpers/NetworkLink.cpp index 3b3b981f..3590ffa8 100644 --- a/src/helpers/NetworkLink.cpp +++ b/src/helpers/NetworkLink.cpp @@ -11,6 +11,7 @@ #include #include +#include #if defined(NETWORK_PREFER_ETHERNET) #include "ethernet/ch390/CH390Config.h" @@ -40,6 +41,8 @@ class NetworkLinkBase : public NetworkLink { bool _status_initialized = false; bool _last_connected = false; unsigned long _last_status_check = 0; + static constexpr int kDnsServers = 2; + std::atomic _dns_snapshot[kDnsServers] = {{0}, {0}}; AlertFaultPolicy::OutageSnapshot outage() const { return AlertFaultPolicy::unpackOutageSnapshot( @@ -65,6 +68,7 @@ class NetworkLinkBase : public NetworkLink { (uint32_t)now_ms, reason, outage())); } + public: unsigned long connectedAtMillis() const override { return _connected_at.load(std::memory_order_relaxed); @@ -81,6 +85,39 @@ class NetworkLinkBase : public NetworkLink { AlertFaultPolicy::OutageSnapshot outageSnapshot() const override { return outage(); } + + // Remember the resolver this link's DHCP lease installed, so switching back + // to it can put that resolver back. + // + // lwIP keeps ONE global server list, not one per interface: a DHCP lease on + // any medium calls dns_setserver() and overwrites whatever the other medium + // had. IDF 4.4 has no per-interface retention either — esp_netif_get_dns_info() + // reads the same globals — so the medium that leased last owns DNS for every + // socket. Without this, an Ethernet node that falls back to Wi-Fi and then + // fails back keeps asking the Wi-Fi network's DNS server. When the two are on + // different subnets that server is unreachable from Ethernet, and the node + // reads as connected while every broker and NTP hostname fails to resolve, + // until Ethernet's own DHCP renewal happens to fix it hours later. + void snapshotDns() { + for (int i = 0; i < kDnsServers; i++) { + const ip_addr_t* server = dns_getserver(i); + _dns_snapshot[i].store( + (server != nullptr && !ip_addr_isany(server)) + ? ip4_addr_get_u32(ip_2_ip4(server)) : 0, + std::memory_order_relaxed); + } + } + + // Only servers this link actually leased are restored; a medium that has + // never held a lease leaves the current resolver alone rather than blanking it. + void restoreDns() const { + for (int i = 0; i < kDnsServers; i++) { + const uint32_t addr = _dns_snapshot[i].load(std::memory_order_relaxed); + if (addr == 0) continue; + const ip_addr_t server = IPADDR4_INIT(addr); + dns_setserver(i, &server); + } + } }; class WiFiNetworkLink final : public NetworkLinkBase { @@ -199,7 +236,10 @@ class WiFiNetworkLink final : public NetworkLinkBase { // Already associated when the link started (end()/begin() leaves STA up): // there is no connect transition below to carry the setting, so apply it // here or the node keeps running whatever mode was set before. - if (connected) applyPowerPrefs(wifi_power_save); + if (connected) { + applyPowerPrefs(wifi_power_save); + snapshotDns(); + } } if ((uint32_t)(now_ms - _last_status_check) <= 10000) { @@ -216,6 +256,7 @@ class WiFiNetworkLink final : public NetworkLinkBase { _connected_at.store(now_ms, std::memory_order_relaxed); _reconnect_backoff_attempt = 0; applyPowerPrefs(wifi_power_save); + snapshotDns(); } _last_connected = true; return transitioned ? NetworkTransition::Up : NetworkTransition::None; @@ -308,6 +349,10 @@ class EthernetNetworkLink final : public NetworkLinkBase { _event_state.store(static_cast(EventState::GotIp), std::memory_order_relaxed); noteConnected(millis()); + // Taken here rather than at the maintain() edge: this is the moment + // the lease installed the resolver, before a Wi-Fi fallback lease + // can overwrite it. + snapshotDns(); break; case ARDUINO_EVENT_ETH_DISCONNECTED: _event_state.store(static_cast(EventState::LinkDown), @@ -350,7 +395,10 @@ class EthernetNetworkLink final : public NetworkLinkBase { _status_initialized = true; setOutage(AlertFaultPolicy::applyWifiStatus( now_ms, connected, outage(), false)); - if (connected) noteConnected(now_ms); + if (connected) { + noteConnected(now_ms); + snapshotDns(); + } return NetworkTransition::None; } @@ -364,6 +412,7 @@ class EthernetNetworkLink final : public NetworkLinkBase { _connected_at.store(now_ms, std::memory_order_relaxed); setOutage(AlertFaultPolicy::applyWifiStatus( now_ms, true, outage(), true)); + snapshotDns(); return NetworkTransition::Up; } @@ -525,6 +574,13 @@ class AutomaticNetworkLink final : public NetworkLink { } _selected_down_since.store(0, std::memory_order_relaxed); _selected.store(medium, std::memory_order_release); + // Put back the resolver this medium leased. lwIP's server list is global, + // so whichever medium leased last still owns it here (see snapshotDns()). + if (medium == NetworkMedium::Ethernet) { + _ethernet.restoreDns(); + } else if (medium == NetworkMedium::WiFi) { + _wifi.restoreDns(); + } } bool startOrRetryEthernet(uint32_t now_ms, bool restart) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index d0ce8c02..29edc89e 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2335,31 +2335,38 @@ static constexpr unsigned long kNetworkTransitionDisconnectMs = 2000; // Close a slot's transport because the route under it changed. // -// Same shape as closeLiveClientForReconfigure(), and for the same F04 reason: a -// client that is still Starting has an attempt aimed at the OLD route, and -// softDisconnect() returns immediately on one that is not yet connected, so the -// attempt would run to completion and deliver a CONNECTED event that is -// indistinguishable from the new route's. On a medium switch the old route can -// still be briefly usable, so that event is not hypothetical. Cancelling it -// needs a real stop; reconnectSlotClient() starts a stopped client again. +// Deliberately NOT the reconfigure path, even though both close a live client. +// A reconfigure gives the slot a new endpoint and new credentials, so an +// in-flight attempt that completes afterwards reports a connection to the OLD +// broker under the new configuration, and cancelling it is worth a real stop +// (F04). A link transition changes neither: the attempt is aimed at the same +// broker with the same credentials, so if it completes it is credited to the +// endpoint it actually reached. It is stale only in that its socket sits on a +// route that is going away, and the ordinary DISCONNECTED path handles that. // -// A connected client takes the cheap path instead — bounded, and the esp-mqtt -// task stays, which is what keeps a Wi-Fi flap from recreating every slot's -// task and refragmenting internal heap. The wait is short because the old route -// is already gone; a client whose event is late is aborted by keepalive and -// picked up by the normal backoff retry. +// So an attempt in flight is left alone here. Stopping it would mean calling +// esp_mqtt_client_stop(), which waits on the SDK's API mutex and its task's +// stopped event with no bound (see stopSlotClient()) — the client task notices +// only when it returns from whatever transport call it is in, up to +// network_timeout_ms for a TLS connect. During teardown that is contained by +// the bridge's StopUnproven timeout; during a transition nothing contains it, +// so paying it serially for up to five slots would freeze the sole MQTT worker +// far past the bound this path advertises, and would block the bridge's own +// stop handshake behind it. A routine Wi-Fi flap must not cost that. // -// Nothing here touches a client that is not live, so a Quarantined one (its SDK -// task was never joined) is left alone, as it must be for the rest of the boot. +// A connected client takes the cheap bounded path, and the esp-mqtt task stays, +// which is what keeps a flap from recreating every slot's task and +// refragmenting internal heap. Nothing here touches a client that is not live, +// so a Quarantined one (its SDK task was never joined) is left alone, as it +// must be for the rest of the boot. void MQTTBridge::closeLiveClientForLinkTransition(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; MQTTSlot& slot = _slots[index]; if (slot.client == nullptr || !clientStateIsLive(slot.client_state)) return; if (mqttClientStateHasAttemptInFlight(slot.client_state)) { - MQTT_DEBUG_PRINTLN("MQTT%d link transition during connect - stopping to cancel the attempt", + MQTT_DEBUG_PRINTLN("MQTT%d link transition during connect - leaving the attempt to resolve", index + 1); - stopSlotClient(index); return; } diff --git a/test/test_mqtt_client_state/test_mqtt_client_state.cpp b/test/test_mqtt_client_state/test_mqtt_client_state.cpp index 76cfba73..1ff2b97f 100644 --- a/test/test_mqtt_client_state/test_mqtt_client_state.cpp +++ b/test/test_mqtt_client_state/test_mqtt_client_state.cpp @@ -76,26 +76,31 @@ TEST(MqttClientState, StopIsAcknowledgedOnlyWhenEveryClientIsProvenStopped) { } // A link transition closes each slot's transport because the route under it -// changed. The two predicates above are what decide how, and getting the -// Starting case wrong is not cosmetic: softDisconnect() returns immediately on -// a client that is not yet connected, so that attempt would run to completion -// against the OLD route and deliver a CONNECTED event the new attempt cannot be -// told apart from. -TEST(MqttClientState, LinkTransitionStopsOnlyClientsWithAnAttemptInFlight) { +// changed. These two predicates are what decide which clients it may touch. +// +// The distinction from a reconfigure matters: a reconfigure must cancel an +// in-flight attempt, because that attempt would report the OLD endpoint under +// the new configuration. A transition changes no configuration, so the attempt +// is left to resolve — cancelling it would mean an unbounded +// esp_mqtt_client_stop() on the sole MQTT worker during a routine link flap. +TEST(MqttClientState, LinkTransitionOnlyDisconnectsClientsWithNoAttemptInFlight) { for (MqttClientState s : kAllStates) { - const bool touched = mqttClientStateIsLive(s); - const bool needs_stop = touched && mqttClientStateHasAttemptInFlight(s); + const bool live = mqttClientStateIsLive(s); + const bool disconnected_here = live && !mqttClientStateHasAttemptInFlight(s); // Quarantined is never touched: its SDK task was not joined, so it stays // out of service for the rest of the boot. - if (s == MqttClientState::Quarantined) EXPECT_FALSE(touched) << mqttClientStateName(s); + if (s == MqttClientState::Quarantined) EXPECT_FALSE(live) << mqttClientStateName(s); // Nothing that is already proven stopped is worth a disconnect. - if (mqttClientStateIsProvenStopped(s)) EXPECT_FALSE(touched) << mqttClientStateName(s); + if (mqttClientStateIsProvenStopped(s)) EXPECT_FALSE(live) << mqttClientStateName(s); - EXPECT_EQ(s == MqttClientState::Starting, needs_stop) << mqttClientStateName(s); - // The cheap bounded path, which keeps the esp-mqtt task alive. EXPECT_EQ(s == MqttClientState::Connected || s == MqttClientState::Disconnected, - touched && !needs_stop) << mqttClientStateName(s); + disconnected_here) << mqttClientStateName(s); + // Starting is live, and still must not be disconnected here. + if (s == MqttClientState::Starting) { + EXPECT_TRUE(live); + EXPECT_FALSE(disconnected_here); + } } } From c78186236252d9d19de5415dcdb5a6c7d4d26045 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 15:45:39 -0700 Subject: [PATCH 87/93] fix(network): keep the DNS snapshot typed and on the lwIP thread Follow-up to the DNS restore, from a second review round. lwIP's DNS functions belong to the TCP/IP thread, and both halves were called from elsewhere: the Ethernet snapshot from the Arduino event task, the Wi-Fi snapshot and every restore from the MQTT task. Both now go through esp_netif_tcpip_exec(), which is also what serializes the snapshot itself, so it no longer needs atomics. Neither caller is the TCP/IP thread, so the call cannot deadlock on itself. The snapshot keeps the whole ip_addr_t instead of an IPv4 word. These builds compile lwIP with IPv6 enabled, so reinterpreting a v6 server as v4 would have installed four meaningless bytes as a resolver. RDNSS is disabled here, so a v6 server should not arise today; storing the tagged value means it stays correct if that changes. Restore now writes every slot, empty ones included, so returning to a one-server network cannot leave the other medium's second server behind as a fallback that only fails slowly. A lease that carried no resolver at all is not recorded, so a medium that has never had DNS leaves the current resolver alone instead of wiping it. --- src/helpers/NetworkLink.cpp | 56 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/src/helpers/NetworkLink.cpp b/src/helpers/NetworkLink.cpp index 3590ffa8..ba87b35a 100644 --- a/src/helpers/NetworkLink.cpp +++ b/src/helpers/NetworkLink.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #if defined(NETWORK_PREFER_ETHERNET) @@ -42,7 +43,9 @@ class NetworkLinkBase : public NetworkLink { bool _last_connected = false; unsigned long _last_status_check = 0; static constexpr int kDnsServers = 2; - std::atomic _dns_snapshot[kDnsServers] = {{0}, {0}}; + // Only ever touched on the lwIP thread (see snapshotDns()), so no atomics. + ip_addr_t _dns_snapshot[kDnsServers] = {}; + bool _dns_captured = false; AlertFaultPolicy::OutageSnapshot outage() const { return AlertFaultPolicy::unpackOutageSnapshot( @@ -86,6 +89,7 @@ class NetworkLinkBase : public NetworkLink { return outage(); } + private: // Remember the resolver this link's DHCP lease installed, so switching back // to it can put that resolver back. // @@ -98,26 +102,52 @@ class NetworkLinkBase : public NetworkLink { // different subnets that server is unreachable from Ethernet, and the node // reads as connected while every broker and NTP hostname fails to resolve, // until Ethernet's own DHCP renewal happens to fix it hours later. - void snapshotDns() { + // + // Both halves run through esp_netif_tcpip_exec() because lwIP's DNS functions + // belong to the TCP/IP thread, and these are called from the Arduino event + // task (Ethernet GOT_IP) and the MQTT task (Wi-Fi edge, and select()). That + // also serializes the snapshot itself, so it needs no atomics. Neither caller + // is the TCP/IP thread, so the call cannot deadlock on itself. + // + // The whole ip_addr_t is kept, not an IPv4 word: these builds compile lwIP + // with IPv6 on, and reinterpreting a v6 server as v4 would install four + // meaningless bytes as a resolver. (RDNSS is disabled here, so a v6 server + // should never appear — storing the tagged value means it stays correct if + // that ever changes.) + static esp_err_t snapshotDnsOnTcpipThread(void* ctx) { + NetworkLinkBase* self = static_cast(ctx); + bool any = false; for (int i = 0; i < kDnsServers; i++) { const ip_addr_t* server = dns_getserver(i); - _dns_snapshot[i].store( - (server != nullptr && !ip_addr_isany(server)) - ? ip4_addr_get_u32(ip_2_ip4(server)) : 0, - std::memory_order_relaxed); + self->_dns_snapshot[i] = (server != nullptr) ? *server : *IP_ADDR_ANY; + if (!ip_addr_isany_val(self->_dns_snapshot[i])) any = true; } + // A lease that carried no resolver at all (static configuration, or a + // server that offered none) must not be remembered as "this medium's DNS is + // nothing" — restoring that would wipe a working resolver for no gain. + if (any) self->_dns_captured = true; + return ESP_OK; } - // Only servers this link actually leased are restored; a medium that has - // never held a lease leaves the current resolver alone rather than blanking it. - void restoreDns() const { + static esp_err_t restoreDnsOnTcpipThread(void* ctx) { + NetworkLinkBase* self = static_cast(ctx); + if (!self->_dns_captured) return ESP_OK; + // Every slot is written, empty ones included: returning to a network with + // one server must not leave the other medium's second server behind as a + // fallback that only fails slowly. for (int i = 0; i < kDnsServers; i++) { - const uint32_t addr = _dns_snapshot[i].load(std::memory_order_relaxed); - if (addr == 0) continue; - const ip_addr_t server = IPADDR4_INIT(addr); - dns_setserver(i, &server); + dns_setserver(i, &self->_dns_snapshot[i]); } + return ESP_OK; } + + public: + void snapshotDns() { esp_netif_tcpip_exec(snapshotDnsOnTcpipThread, this); } + + // A medium that has never held a lease with a resolver leaves the current one + // alone rather than blanking it. + void restoreDns() { esp_netif_tcpip_exec(restoreDnsOnTcpipThread, this); } + }; class WiFiNetworkLink final : public NetworkLinkBase { From da6406e92fdd2d83350074ed56e4f19ffebd9617 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 15:57:45 -0700 Subject: [PATCH 88/93] fix(network): rebuild the DHCP hostname when the node is renamed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hostname was built once in begin() from the node name, so `set name` left the node advertising its old name to DHCP until the next reflash. It is now rebuilt on rename through a new CommonCLICallbacks::onNodeNameChanged() hook, and begin() shares the same helper. What this cannot do is rename a live lease. IDF 4.4 states that a hostname changed after the interface is up "would only be reflected once the interface restarts/reconnects", and Arduino 2.x's WiFi.setHostname() does not touch a running netif at all — it writes a static string that the netif reads at bring-up. Forcing the issue would mean bouncing the link, which costs every MQTT slot a reconnect; that is not a reasonable price for a rename, so the new name lands at the next reconnect or boot instead. Only automatic (Ethernet-preferred) links carry a hostname today, exactly as before. Giving plain Wi-Fi observers one would change the DHCP identity of the existing fleet and is a separate decision. --- examples/simple_repeater/MyMesh.cpp | 27 +++++++++++++++++++++----- examples/simple_repeater/MyMesh.h | 13 +++++++++++++ examples/simple_room_server/MyMesh.cpp | 27 +++++++++++++++++++++----- examples/simple_room_server/MyMesh.h | 13 +++++++++++++ src/helpers/CommonCLI.cpp | 1 + src/helpers/CommonCLI.h | 4 ++++ 6 files changed, 75 insertions(+), 10 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index d6a9554d..dfd407b6 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1123,6 +1123,27 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc memset(default_scope.key, 0, sizeof(default_scope.key)); } +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) +// Only an automatic link carries a hostname today (see the header note). The +// value reaches DHCP when the link next starts or reconnects: IDF 4.4 states +// plainly that a hostname changed after the interface is up "would only be +// reflected once the interface restarts/reconnects", and Arduino 2.x's +// WiFi.setHostname() does not even touch a running netif — it writes a static +// string the netif reads at bring-up. So this cannot rename a live lease, and +// deliberately does not bounce the link to force one: dropping an observer's +// route mid-operation costs every MQTT slot a reconnect, which is not a +// reasonable price for a cosmetic rename. +void MyMesh::applyNetworkHostname(const char* when) { + NetworkLink& link = activeNetworkLink(); + if (!link.isAutomatic()) return; + char network_hostname[NetworkHostname::kBufferSize]; + NetworkHostname::build(network_hostname, sizeof(network_hostname), + _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); + link.setHostname(network_hostname); + Serial.printf("Network: hostname %s%s\n", network_hostname, when); +} +#endif + void MyMesh::begin(FILESYSTEM *fs) { mesh::Mesh::begin(); _fs = fs; @@ -1151,11 +1172,7 @@ void MyMesh::begin(FILESYSTEM *fs) { #if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) NetworkLink& boot_network = activeNetworkLink(); if (boot_network.isAutomatic()) { - char network_hostname[NetworkHostname::kBufferSize]; - NetworkHostname::build(network_hostname, sizeof(network_hostname), - _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); - boot_network.setHostname(network_hostname); - Serial.printf("Network: hostname %s\n", network_hostname); + applyNetworkHostname(""); MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index aebfa2e6..344d4932 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -274,10 +274,23 @@ protected: void sendFloodReply(mesh::Packet* packet, unsigned long delay_millis, uint8_t path_hash_size); + // Build the DHCP hostname from the current node name and hand it to the + // link. Only automatic links carry one: a plain Wi-Fi observer keeps the + // framework default, and changing that for the existing fleet is a separate + // decision. + void applyNetworkHostname(const char* when); + public: MyMesh(mesh::MainBoard& board, mesh::Radio& radio, mesh::MillisecondClock& ms, mesh::RNG& rng, mesh::RTCClock& rtc, mesh::MeshTables& tables); void begin(FILESYSTEM* fs); + + // CommonCLICallbacks: `set name` has just rewritten node_name. + void onNodeNameChanged() override { +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) + applyNetworkHostname(" (renamed)"); +#endif + } void sendNodeDiscoverReq(); const char* getFirmwareVer() override { return FIRMWARE_VERSION; } const char* getBuildDate() override { return FIRMWARE_BUILD_DATE; } diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 146c547e..1eba3892 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -943,6 +943,27 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc memset(default_scope.key, 0, sizeof(default_scope.key)); } +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) +// Only an automatic link carries a hostname today (see the header note). The +// value reaches DHCP when the link next starts or reconnects: IDF 4.4 states +// plainly that a hostname changed after the interface is up "would only be +// reflected once the interface restarts/reconnects", and Arduino 2.x's +// WiFi.setHostname() does not even touch a running netif — it writes a static +// string the netif reads at bring-up. So this cannot rename a live lease, and +// deliberately does not bounce the link to force one: dropping an observer's +// route mid-operation costs every MQTT slot a reconnect, which is not a +// reasonable price for a cosmetic rename. +void MyMesh::applyNetworkHostname(const char* when) { + NetworkLink& link = activeNetworkLink(); + if (!link.isAutomatic()) return; + char network_hostname[NetworkHostname::kBufferSize]; + NetworkHostname::build(network_hostname, sizeof(network_hostname), + _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); + link.setHostname(network_hostname); + Serial.printf("Network: hostname %s%s\n", network_hostname, when); +} +#endif + void MyMesh::begin(FILESYSTEM *fs) { mesh::Mesh::begin(); _fs = fs; @@ -952,11 +973,7 @@ void MyMesh::begin(FILESYSTEM *fs) { #if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) NetworkLink& boot_network = activeNetworkLink(); if (boot_network.isAutomatic()) { - char network_hostname[NetworkHostname::kBufferSize]; - NetworkHostname::build(network_hostname, sizeof(network_hostname), - _prefs.node_name, self_id.pub_key, PUB_KEY_SIZE); - boot_network.setHostname(network_hostname); - Serial.printf("Network: hostname %s\n", network_hostname); + applyNetworkHostname(""); MQTTPrefs* obs = _cli.getObserverPrefs(); Serial.printf("Network: probing Ethernet for up to %lums\n", (unsigned long)NETWORK_ETHERNET_BOOT_WAIT_MS); diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index bfcfe362..1870df79 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -281,10 +281,23 @@ protected: void sendFloodReply(mesh::Packet* packet, unsigned long delay_millis, uint8_t path_hash_size); + // Build the DHCP hostname from the current node name and hand it to the + // link. Only automatic links carry one: a plain Wi-Fi observer keeps the + // framework default, and changing that for the existing fleet is a separate + // decision. + void applyNetworkHostname(const char* when); + public: MyMesh(mesh::MainBoard& board, mesh::Radio& radio, mesh::MillisecondClock& ms, mesh::RNG& rng, mesh::RTCClock& rtc, mesh::MeshTables& tables); void begin(FILESYSTEM* fs); + + // CommonCLICallbacks: `set name` has just rewritten node_name. + void onNodeNameChanged() override { +#if defined(WITH_MQTT_BRIDGE) && defined(ESP_PLATFORM) + applyNetworkHostname(" (renamed)"); +#endif + } void addSystemPost(const char* postData); const char* getFirmwareVer() override { return FIRMWARE_VERSION; } diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 244db25b..657157d1 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -1583,6 +1583,7 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep if (isValidName(&config[5])) { StrHelper::strncpy(_prefs->node_name, &config[5], sizeof(_prefs->node_name)); savePrefs(); + _callbacks->onNodeNameChanged(); strcpy(reply, "OK"); } else { strcpy(reply, "Error, bad chars"); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 040e845f..34912525 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -274,6 +274,10 @@ struct LegacyObserverTail { class CommonCLICallbacks { public: virtual void savePrefs() = 0; + // The node name feeds the DHCP hostname, which is built once when the link + // starts. Renaming a running node has to rebuild it or the node keeps + // advertising the old name to DHCP until the next reflash. + virtual void onNodeNameChanged() {} #ifdef WITH_MQTT_BRIDGE virtual bool saveObserverPrefs() = 0; #else From 1bfbbf4bd80689acc57a79d3ed196b12a7a1f653 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 16:46:55 -0700 Subject: [PATCH 89/93] build(mqtt): make the preset table byte-identical across channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preset table is fleet state, not a build detail. A slot's preset is stored in /mqtt.json by name, and firmware that does not know a name does not merely ignore it: MQTTPrefsSerializer repairs it to "none" and CommonCLI writes the repaired file back to flash. A node that rolls back to a channel missing a preset therefore loses that slot permanently, and re-upgrading does not bring it back. The parity gate compared preset names only, deliberately allowing URL, CA and credential drift. It now requires src/helpers/MQTTPresets.h to be byte-identical between the channels, which also catches that drift — the two channels are meant to dial the same brokers. That is only workable if the file holds no channel-specific code, so mqttPresetEnforcesTokenExp() moves to the new MQTTPresetPolicy.h. It was the sole difference between the two channels' copies, and with it moved they match exactly today. Policy keyed off the table belongs there from now on; the table itself stays pure data. The older name-only comparison stays available without --exact for ad-hoc use, and the checker's self-test now covers both modes, including that --exact rejects a config-only change the name check waves through. --- .../workflows/check-mqtt-preset-parity.yml | 12 +- scripts/check_mqtt_preset_parity.py | 107 +++++++++++++++++- src/helpers/MQTTPresetPolicy.h | 34 ++++++ src/helpers/MQTTPresets.h | 18 --- src/helpers/bridges/MQTTBridge.h | 1 + 5 files changed, 145 insertions(+), 27 deletions(-) create mode 100644 src/helpers/MQTTPresetPolicy.h diff --git a/.github/workflows/check-mqtt-preset-parity.yml b/.github/workflows/check-mqtt-preset-parity.yml index e1039a40..7b7d663d 100644 --- a/.github/workflows/check-mqtt-preset-parity.yml +++ b/.github/workflows/check-mqtt-preset-parity.yml @@ -1,7 +1,11 @@ name: Check MQTT Preset Name Parity -# Ensures observer-firmware and observer-firmware-dev share the same built-in -# MQTT preset *names* (config details may differ). See scripts/check_mqtt_preset_parity.py. +# Ensures observer-firmware and observer-firmware-dev ship an identical +# src/helpers/MQTTPresets.h. A slot's preset is stored in /mqtt.json by name, and +# firmware that does not know a name repairs it to "none" and writes the file +# back, so a node that rolls back between channels loses that slot for good. +# Behaviour that differs per channel belongs in MQTTPresetPolicy.h, which is not +# compared. See scripts/check_mqtt_preset_parity.py. permissions: contents: read @@ -81,9 +85,9 @@ jobs: - name: Self-test checker run: python3 scripts/check_mqtt_preset_parity.py --self-test - - name: Compare preset names + - name: Compare preset tables run: | - python3 scripts/check_mqtt_preset_parity.py \ + python3 scripts/check_mqtt_preset_parity.py --exact \ /tmp/preset-parity/prod.h \ /tmp/preset-parity/dev.h \ --label-a observer-firmware \ diff --git a/scripts/check_mqtt_preset_parity.py b/scripts/check_mqtt_preset_parity.py index d213a258..039bc6ee 100755 --- a/scripts/check_mqtt_preset_parity.py +++ b/scripts/check_mqtt_preset_parity.py @@ -1,13 +1,25 @@ #!/usr/bin/env python3 -"""Compare MQTT built-in preset *names* between two MQTTPresets.h files. +"""Compare the MQTT broker preset table between two MQTTPresets.h files. -Only the first string field of each ``MQTT_PRESETS`` entry is compared (set -equality, case-sensitive). URL, auth, CA, keepalive, and credentials are -ignored so channel branches may diverge on config details without failing CI. +With ``--exact`` (what CI uses) the two files must be byte-identical. The table +is fleet state, not just a build detail: a slot's preset is stored in +/mqtt.json by NAME, and firmware that does not know a name does not merely +ignore it — MQTTPrefsSerializer repairs it to "none" and the repaired file is +written back to flash. A node that rolls back from one channel to the other +therefore loses that slot permanently. Byte equality also catches URL, CA and +credential drift, which a name-only check passes silently even though the two +channels are meant to dial the same brokers. + +Channel-specific behaviour belongs in MQTTPresetPolicy.h, which is not compared, +so this file can stay identical while the channels differ elsewhere. + +Without ``--exact`` only the first string field of each ``MQTT_PRESETS`` entry +is compared (set equality, case-sensitive). That is the older, weaker check, +kept for ad-hoc use. Usage:: - python3 scripts/check_mqtt_preset_parity.py FILE_A FILE_B \\ + python3 scripts/check_mqtt_preset_parity.py FILE_A FILE_B --exact \\ --label-a observer-firmware --label-b observer-firmware-dev python3 scripts/check_mqtt_preset_parity.py --self-test @@ -137,6 +149,49 @@ def compare( ] +def compare_exact( + path_a: Path, + path_b: Path, + *, + label_a: str, + label_b: str, +) -> list[str]: + """Return error lines if the two files differ byte for byte.""" + + data_a = path_a.read_bytes() + data_b = path_b.read_bytes() + if data_a == data_b: + return [] + + import difflib + + errors = [f"{label_a} and {label_b} do not match byte for byte."] + try: + diff = list( + difflib.unified_diff( + data_a.decode("utf-8").splitlines(), + data_b.decode("utf-8").splitlines(), + fromfile=label_a, + tofile=label_b, + lineterm="", + n=1, + ) + ) + except UnicodeDecodeError: + errors.append("(binary difference; cannot render a text diff)") + return errors + + # Enough to identify the drift without pasting the whole table into a log. + errors.extend(diff[:60]) + if len(diff) > 60: + errors.append(f"... {len(diff) - 60} more diff line(s)") + errors.append( + "Preset rows must be identical on both channels. Behaviour that differs " + "per channel belongs in MQTTPresetPolicy.h." + ) + return errors + + def load_names(path: Path) -> set[str]: text = path.read_text(encoding="utf-8") names, _ = parse_presets(text, source=str(path)) @@ -237,6 +292,26 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = {{ print("self-test failed: URL scheme was treated as a comment.", file=sys.stderr) return 1 + # --exact: identical bytes pass, any drift fails — including a change + # that a name-only comparison waves through. + if compare_exact(equal_a, equal_a, label_a="a", label_b="a-copy"): + print("self-test failed: a file was not equal to itself.", file=sys.stderr) + return 1 + if compare(load_names(equal_a), load_names(equal_b), label_a="a", label_b="b"): + print("self-test failed: fixture assumption broken.", file=sys.stderr) + return 1 + exact_errs = compare_exact(equal_a, equal_b, label_a="a", label_b="b") + if not exact_errs or "byte for byte" not in exact_errs[0]: + print( + "self-test failed: --exact accepted files that differ only in " + f"config: {exact_errs!r}", + file=sys.stderr, + ) + return 1 + if not any("MQTTPresetPolicy.h" in line for line in exact_errs): + print("self-test failed: --exact failure omitted the remedy.", file=sys.stderr) + return 1 + # Silence unused path in fixture layout. dupes.write_text("// unused\n", encoding="utf-8") @@ -250,6 +325,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("file_b", type=Path, nargs="?", help="Second MQTTPresets.h (e.g. dev)") parser.add_argument("--label-a", default="file_a", help="Label for file_a in reports") parser.add_argument("--label-b", default="file_b", help="Label for file_b in reports") + parser.add_argument( + "--exact", + action="store_true", + help="Require the two files to be byte-identical (what CI enforces)", + ) parser.add_argument("--self-test", action="store_true") args = parser.parse_args(argv) @@ -259,6 +339,23 @@ def main(argv: list[str] | None = None) -> int: if args.file_a is None or args.file_b is None: parser.error("FILE_A and FILE_B are required unless --self-test is set") + if args.exact: + try: + errors = compare_exact( + args.file_a, args.file_b, label_a=args.label_a, label_b=args.label_b + ) + except OSError as error: + print(f"MQTT preset parity check could not run: {error}", file=sys.stderr) + return 2 + if errors: + print("MQTT preset parity check failed:", *errors, sep="\n ", file=sys.stderr) + return 1 + print( + f"MQTT preset parity check passed: {args.label_a} and {args.label_b} " + "have identical preset tables." + ) + return 0 + try: names_a = load_names(args.file_a) names_b = load_names(args.file_b) diff --git a/src/helpers/MQTTPresetPolicy.h b/src/helpers/MQTTPresetPolicy.h new file mode 100644 index 00000000..a5a8ddce --- /dev/null +++ b/src/helpers/MQTTPresetPolicy.h @@ -0,0 +1,34 @@ +#pragma once + +#include "MQTTPresets.h" + +#include + +// Policy that keys off a preset rather than describing one. +// +// MQTTPresets.h is data: it is the broker table, and it is kept byte-identical +// between the observer-firmware and observer-firmware-dev channels so a node +// that rolls back cannot meet a preset name its firmware does not know. An +// unknown name is not merely ignored — MQTTPrefsSerializer repairs it to "none" +// and the repaired /mqtt.json is written back, so the operator's slot is gone +// for good. Channel-specific behaviour therefore lives here instead, where the +// two channels are free to differ. + +// True when the broker tears down a live session once its JWT passes exp, so the +// renewal must proactively bounce the connection to present a fresh token. +// +// Default true, because getting this wrong the safe way costs a re-handshake and +// getting it wrong the unsafe way costs an outage. waev is the exception: its +// operator confirmed (2026-08-11) that their servers do not disconnect on expiry, +// so a live session there needs only its credentials refreshed for the next +// reconnect. waev is also the only preset with a short token_lifetime, so it was +// the only one bouncing often — every ~47 min, and each bounce's re-handshake can +// cost ~10 KB of contiguous internal DRAM on a non-PSRAM board. +// +// Keyed by name rather than a struct field on purpose: adding a field would mean +// re-ordering a dozen positional initialisers in the table, where a mistake is +// silent. +static inline bool mqttPresetEnforcesTokenExp(const MQTTPresetDef* preset) { + if (!preset || !preset->name) return true; // custom/audience slots: assume enforced + return strcmp(preset->name, "waev") != 0; +} diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 8b9d4b1f..5537b031 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -47,24 +47,6 @@ struct MQTTPresetDef { // Braces match topic placeholders ({device}/{iata}); never send this string to the broker. static const char MQTT_USERPASS_USERNAME_PUBKEY[] = "{pubkey}"; -// True when the broker tears down a live session once its JWT passes exp, so the -// renewal must proactively bounce the connection to present a fresh token. -// -// Default true, because getting this wrong the safe way costs a re-handshake and -// getting it wrong the unsafe way costs an outage. waev is the exception: its -// operator confirmed (2026-08-11) that their servers do not disconnect on expiry, -// so a live session there needs only its credentials refreshed for the next -// reconnect. waev is also the only preset with a short token_lifetime, so it was -// the only one bouncing often — every ~47 min, and each bounce's re-handshake can -// cost ~10 KB of contiguous internal DRAM on a non-PSRAM board. -// -// Keyed by name rather than a struct field on purpose: adding a field would mean -// re-ordering a dozen positional initialisers below, where a mistake is silent. -static inline bool mqttPresetEnforcesTokenExp(const MQTTPresetDef* preset) { - if (!preset || !preset->name) return true; // custom/audience slots: assume enforced - return strcmp(preset->name, "waev") != 0; -} - static inline bool mqttPresetUsesDevicePubkeyUsername(const MQTTPresetDef* preset) { return preset && preset->auth_type == MQTT_AUTH_USERPASS && preset->userpass_username && diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 7b662da6..1126da0b 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -10,6 +10,7 @@ #include "helpers/JWTHelper.h" #include "helpers/MQTTPacketFilter.h" #include "helpers/MQTTPresets.h" +#include "helpers/MQTTPresetPolicy.h" #include "helpers/MQTTLifecycle.h" #include "helpers/AlertFaultPolicy.h" #include "helpers/MQTTEffectiveConfig.h" From 0dca31f5c3ca61bc7de61e84804c61c5d2ebf8d0 Mon Sep 17 00:00:00 2001 From: MarekWo Date: Sat, 12 Sep 2026 16:52:44 +0200 Subject: [PATCH 90/93] Add marwoj MQTT broker preset Adds a built-in preset for the Polish MeshCore community broker at mqtt.marwoj.net: MQTT over TLS on port 8883, username/password auth with credentials embedded in firmware, and the standard MeshCore topic layout. The Let's Encrypt chain anchors at the existing ISRG_ROOT_X1 constant, so no new CA certificate is needed. Co-Authored-By: Claude Opus 5 --- MQTT_IMPLEMENTATION.md | 5 +++-- src/helpers/MQTTPresets.h | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index d5eeb8bc..72fdbfc0 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -164,6 +164,7 @@ below documents the current build. | `idahomesh` | `wss://mqtt.idahomesh.org:443/mqtt` | JWT | — | | `ntxmesh` | `wss://ntxmesh.dhovin.me:8883` | JWT | — | | `bsmesh` | `wss://mqtt.bsmesh.de:8885` | JWT | — | +| `marwoj` | `mqtts://mqtt.marwoj.net:8883` | User/pass (in firmware) | — | | `custom` | your own broker | User/pass, or JWT when `mqttN.audience` is set | `set mqttN.server` (see [custom broker setup](#custom-brokers)) | | `none` | (slot disabled) | — | — | @@ -374,7 +375,7 @@ Each slot (1-6) supports the following commands: - `set mqttN.audience` - Clear JWT audience (reverts to username/password auth) - `set mqttN.filter ` - Select payload types uploaded to this slot -**Note:** Custom server/port settings only apply when the slot's preset is `custom`. Username/password also apply to built-in presets that use per-slot credentials (e.g. `inwmesh`); other userpass presets (`tennmesh`, `nashmesh`, `ctmesh`) ship fixed credentials in firmware. +**Note:** Custom server/port settings only apply when the slot's preset is `custom`. Username/password also apply to built-in presets that use per-slot credentials (e.g. `inwmesh`); other userpass presets (`tennmesh`, `nashmesh`, `ctmesh`, `marwoj`) ship fixed credentials in firmware. #### Per-broker packet filters @@ -912,7 +913,7 @@ the radio actually performs in that case. ### Authentication The auth mode is fixed per preset (see [Broker Presets](#broker-presets)). Three modes are used: - **JWT Authentication**: Ed25519-signed tokens for brokers that expect JWT (most WSS presets). For `custom` slots, JWT is used when `audience` is set. -- **Username/Password**: Some presets ship fixed credentials embedded in firmware (`tennmesh`, `nashmesh`, `ctmesh` — plain MQTT, no TLS); others (`inwmesh`, `custom`) take per-slot credentials via `mqttN.username` / `mqttN.password`. +- **Username/Password**: Some presets ship fixed credentials embedded in firmware (`tennmesh`, `nashmesh`, `ctmesh` — plain MQTT, no TLS; `marwoj` — MQTT over TLS); others (`inwmesh`, `custom`) take per-slot credentials via `mqttN.username` / `mqttN.password`. - **None**: `meshrank` (account token carried in the topic) and `eastidahomesh` connect without broker auth. - **Username Format** (JWT): `v1_{UPPERCASE_PUBLIC_KEY}` - **Automatic Token Renewal**: Tokens are renewed before expiration diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 5537b031..000adf84 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -130,7 +130,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 36; +static const int MQTT_PRESET_COUNT = 37; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -179,6 +179,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "idahomesh", "wss://mqtt.idahomesh.org:443/mqtt", "mqtt.idahomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "ntxmesh", "wss://ntxmesh.dhovin.me:8883", "ntxmesh.dhovin.me", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "bsmesh", "wss://mqtt.bsmesh.de:8885", "mqtt.bsmesh.de", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "marwoj", "mqtts://mqtt.marwoj.net:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, "observer-agessaman", "ipRwCEclZkX47K" }, }; // Find a preset by name, returns nullptr if not found From 4dca3688eab860e508b678825026f4a8864c770c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 18:03:09 -0700 Subject: [PATCH 91/93] ci(observer): prune stale manifests and verify the production OTA channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps found while tracing a ThinkNode M7 report of a missing OTA manifest. The M7 itself was healthy in both channels; these are what the investigation turned up around it. Pass --prune to gen-slim-manifests.py in both workflows. The generator only ever added, so an env that stopped being built kept a manifest pointing at a release asset the KEEP_BUILDS=2 prune later deleted — a 404 for `ota update` on any node still running it. The flasher-side commit adds the flag along with an empty-input guard and --prune-limit, which fails the run rather than pruning when a build looks like it silently lost envs. Add "Verify production channel is baked in", which production lacked while beta has had it from the start. The manifest base is a compile-time -D, so a build that lost it or picked up the other channel's is invisible until a node runs `ota check` — and the failure mode is production hardware OTA-ing itself onto beta. Checked against real published binaries: passes the production M7 build, rejects the beta one. OTA_MANIFEST_BASE_URL is now stated explicitly in the production env: block, equal to build.sh's default, so the verify step asserts against the value the build was actually handed instead of a second hardcoded copy that could drift. Neither change triggers a build: .github/** is in both workflows' paths-ignore. --- .../build-observer-firmwares-beta.yml | 8 +++++ .../workflows/build-observer-firmwares.yml | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/build-observer-firmwares-beta.yml b/.github/workflows/build-observer-firmwares-beta.yml index 8618da4b..a81e3484 100644 --- a/.github/workflows/build-observer-firmwares-beta.yml +++ b/.github/workflows/build-observer-firmwares-beta.yml @@ -312,8 +312,16 @@ jobs: # Slim manifests come from the build output in out/ (the assets # actually uploaded to the release), not from config-beta.json — see # gen-slim-manifests.py's --bin-dir mode (flasher repo PR #1). + # --prune deletes manifests for envs this build no longer produces. + # Without it the dir only ever grew: the retired + # LilyGo_TLora_V2_1_1_6_*_observer_mqtt_ manifests sat here for two + # months pointing at release assets that had since been pruned, so any + # node still on that build got a 404 from `ota update`. The generator + # caps the prune (--prune-limit, default 4) and fails the run instead + # of pruning when a build looks like it silently lost envs. python3 flasher/scripts/gen-slim-manifests.py \ --bin-dir out \ + --prune \ --static-path "$STATIC_PATH" \ --out-dir "flasher/$MANIFEST_DIR" \ --base-version "$FIRMWARE_VERSION" \ diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 13d9cda0..81312ea2 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -48,6 +48,12 @@ env: # repo). Baked into the slim OTA manifests' file URLs; must stay consistent # with config.json's staticPath. STATIC_PATH: https://observer-fw.gessaman.com + # Where the firmware fetches .json from — this URL *is* the release + # channel. Stated explicitly (it equals build.sh's default) rather than left + # unset, so "Verify production channel is baked in" below can assert against the + # same value the build was handed instead of a second hardcoded copy. Must match + # build.sh's OTA_MANIFEST_BASE_URL default and the Pages path serving flasher/v. + OTA_MANIFEST_BASE_URL: https://observer.gessaman.com/v jobs: @@ -137,6 +143,23 @@ jobs: FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} + - name: Verify production channel is baked in + shell: bash + run: | + # Mirror of the beta workflow's guard, which production lacked. Fail fast + # rather than publish firmware that would OTA itself onto the wrong + # channel: the manifest base is a compile-time -D, so a build that lost + # it (or picked up beta's) is invisible until a node tries `ota check`. + BIN=$(find .pio/build -name firmware.elf | head -1) + if [ -z "$BIN" ]; then echo "no ELF found to verify" >&2; exit 1; fi + if ! strings "$BIN" | grep -qF "$OTA_MANIFEST_BASE_URL"; then + echo "ERROR: production manifest base missing from $BIN" >&2; exit 1 + fi + if strings "$BIN" | grep -qF 'https://observer.gessaman.com/beta/v'; then + echo "ERROR: beta manifest base present in a production build" >&2; exit 1 + fi + echo "OK: $BIN carries $OTA_MANIFEST_BASE_URL" + - name: Upload Shard Artifact uses: actions/upload-artifact@v4 with: @@ -253,8 +276,15 @@ jobs: # in out/ — the assets actually uploaded to the release — stamping this # build's number. Then persist the counter so the next run increments # from here. + # --prune deletes manifests for envs this build no longer produces, so + # a retired env cannot leave behind a manifest pointing at a release + # asset that later gets pruned (which is a 404 for `ota update` on any + # node still running it). The generator caps the prune (--prune-limit, + # default 4) and fails the run rather than pruning when a build looks + # like it silently lost envs. python3 flasher/scripts/gen-slim-manifests.py \ --bin-dir out \ + --prune \ --static-path "$STATIC_PATH" \ --out-dir flasher/v \ --base-version "$FIRMWARE_VERSION" \ From 3d2d0b5da655d226aa2256acbb2aaedacd67a4c5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 20:51:25 -0700 Subject: [PATCH 92/93] feat(webconfig): let the portal discover board-specific CLI commands The FEM commands moved out of CommonCLI into Board::handleCommand(), and the T-Beam 1W fan control was added there too, but the portal never caught up: `radio.fem.txgain` and the fan commands were missing from the terminal table entirely, while `radio.fem.rxgain` was offered on every board even though a board with no hook now answers "??:" rather than "unsupported". Whether a node answers these is a property of the board, not of the build, so the page cannot know from the firmware version. Ask the board instead: probeBoardCommands() runs each candidate getter once on the loop task at startup and keeps the ones that answer, which needs no per-variant list because Board::handleCommand() already reports whether it handled a command. /api/status names the survivors and the page hides everything else, so adding a command to a variant means one entry in WC_BOARD_CMDS rather than an edit per board. The two FEM keys also become Radio-panel toggles, gated the same way; their `set` reaches the board hook through the existing config batch, so only the allowlist and /api/config needed to grow. webconfig_cli_audit.py now scans variants/*/*Board.cpp for the boards that actually build the portal, checks set-only keys in the reverse direction, and verifies every gate is a command some board answers; the mock gained --board-cmds so both shapes of board are testable. `stop ota` joins NOT_OFFERED: `start ota` cannot run from the portal, so it has nothing to stop. --- docs/cli_commands.md | 21 +++ scripts/webconfig_cli_audit.py | 129 +++++++++++++++--- scripts/webconfig_mock_server.py | 83 ++++++++++- src/helpers/WebConfigKeys.h | 3 +- src/helpers/esp32/WebConfigServer.cpp | 56 +++++++- src/helpers/esp32/WebConfigServer.h | 7 + .../test_webconfig_keys.cpp | 2 + webui/index.html | 59 ++++++-- 8 files changed, 319 insertions(+), 41 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index aec48720..5001f699 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -322,6 +322,27 @@ Elsewhere it replies `Err - neighbors not enabled in this build`. If a --- +#### View the fan state, or change how it is driven, on supported boards +**Usage:** +- `get fan` +- `set fan ` +- `set fan.lo ` +- `set fan.hi ` + +**Parameters:** +- `mode`: `on`|`off`|`auto` +- `celsius`: `fan.lo` is 0-100 and must be below `fan.hi`; `fan.hi` is at most 120 and must be above `fan.lo` + +**Default:** `auto`, with `fan.lo` 45 and `fan.hi` 60 + +**Notes:** +- Currently the T-Beam 1W, the only board with a fan under software control. +- `get fan` reports the mode, the measured temperature, whether the fan is running, and the remaining cooldown, e.g. `auto 52.4C fan=on cd=18s`. The temperature reads `n/a` when the NTC value is implausible. +- In `auto` the fan starts at `fan.hi` and stops at `fan.lo`; the gap between them is what keeps it from chattering around one threshold. +- `fan.lo` and `fan.hi` are set-only: `get fan` reports the mode and current state, not the thresholds. + +--- + ### System #### View or change this node's name diff --git a/scripts/webconfig_cli_audit.py b/scripts/webconfig_cli_audit.py index 3cd24db4..1b3767d4 100644 --- a/scripts/webconfig_cli_audit.py +++ b/scripts/webconfig_cli_audit.py @@ -15,6 +15,7 @@ that come back an error, so the two stay honest about each other. Exits non-zero if anything fails that is not in EXPECTED_FAILURES. Stdlib only. """ +import glob import json import os import re @@ -30,12 +31,7 @@ INDEX_HTML = os.path.join(HERE, "..", "webui", "index.html") # Errors that are the correct answer, not a gap. EXPECTED_FAILURES = { - # Runtime-gated on the real device by Board::canControlLoRaFemLna(); the - # command exists in every build and the board answers for itself. The mock - # board is a Heltec V3, which has no front-end module. - "get radio.fem.rxgain": "unsupported", - "set radio.fem.rxgain on": "unsupported", - # Guarded by the firmware the same way when no alert PSK is configured. + # Guarded by the firmware when no alert PSK is configured. "alert test": "not configured", } @@ -44,8 +40,11 @@ SKIP = {"reboot", "clkreboot", "poweroff", "shutdown", "erase", "start ota", "stop webconfig", "ota update", "start webconfig", "start webconfig ap"} -def table(): - """The commands autocomplete offers, read straight out of the page.""" +def table(board_cmds=()): + """The commands autocomplete offers, read straight out of the page. + + `board_cmds` is what /api/status said this board answers; the CLI_BOARD_KEYS + entries gated on anything else are not offered and so not driven.""" html = open(INDEX_HTML, encoding="utf-8").read() def section(start, end): @@ -53,17 +52,27 @@ def table(): verbs = re.findall(r'\["([^"]+)","', section("var CLI_VERBS=", "var CLI_KEYS=")) keys = re.findall(r'\["([^"]+)","(?:[^"\\]|\\.)*",(\d)', - section("var CLI_KEYS=", "var CLI_SLOT=")) + section("var CLI_KEYS=", "var CLI_BOARD_KEYS=")) fields = re.findall(r'\["(\w+)","', section("var CLI_SLOT=", "var CLI_TYPES=")) gets = ["get " + k for k, mode in keys if mode != "2"] gets += ["get mqtt%d.%s" % (n, f) for n in (1, 3) for f in fields] # Verbs taking an argument need a value the node will accept; those are # covered by the round-trip probes below rather than guessed at here. + gets += ["get " + k for gate, k, mode in board_table() + if mode != "2" and gate in board_cmds] plain = [v for v in verbs if not v.endswith(" ") and v not in SKIP] return gets + plain +def board_table(): + """[(gate, key, mode)] from CLI_BOARD_KEYS — the keys the page offers only + when /api/status says this board answers for their gate.""" + html = open(INDEX_HTML, encoding="utf-8").read() + section = html[html.index("var CLI_BOARD_KEYS="):html.index("// Per-slot keys")] + return re.findall(r'\["([^"]+)","([^"]+)","(?:[^"\\]|\\.)*",(\d)', section) + + # Where top-level commands are implemented. MyMesh handles a few before # delegating to CommonCLI, which is exactly how discover.* stayed missing from # the table for so long: grepping CommonCLI alone does not see them. @@ -73,17 +82,56 @@ COMMAND_SOURCES = [ "examples/simple_repeater/MyMesh.cpp", ] -# Firmware commands the table deliberately does not offer. Everything below -# except tls.bundletest is also rejected by /api/cli (wcCliUnavailable), so the -# portal never pretends to run something it cannot. +# Board::handleCommand() is dispatched BEFORE CommonCLI's own table, and the FEM +# commands now live there rather than in CommonCLI. A board file contributes +# whole commands ("get radio.fem.rxgain"), not the bare verbs the sources above +# yield, so the two are collected separately and merged. +BOARD_SOURCES = "variants/*/*Board.cpp" + +# Firmware commands the table deliberately does not offer. NOT_OFFERED = { "tls.bundletest", # TLS debugging, not an operator command "start ota", # binds port 80, which the portal is already using + "stop ota", # nothing to stop: `start ota` cannot run from here "clock sync", # takes its time from the caller; a web request has none "log", # streams to Serial and stalls the radio ("log start" is offered) "get acl", # streams to Serial, returns nothing } +# Of those, the ones /api/cli does NOT reject at POST (wcCliUnavailable). They +# are left out of the table rather than blocked, because running them is +# harmless — `stop ota` just reports that no OTA server is running, which is +# always true here. Everything else in NOT_OFFERED must come back a 400 with a +# reason, so the portal never pretends to run something it cannot. +NOT_REFUSED = {"tls.bundletest", "stop ota"} + + +def webconfig_variants(): + """Variant directories whose build serves the portal (ESP32 + MQTT bridge). + + Only these boards can put a command in front of this page; a board command + on an nRF52 variant is real but unreachable from here, so it is not a gap. + """ + out = set() + for ini in glob.glob(os.path.join(HERE, "..", "variants", "*", "platformio.ini")): + if "WITH_MQTT_BRIDGE" in open(ini, encoding="utf-8").read(): + out.add(os.path.basename(os.path.dirname(ini))) + return out + + +def board_commands(): + """Whole commands Board::handleCommand() answers, across portal variants.""" + variants = webconfig_variants() + found = set() + for path in glob.glob(os.path.join(HERE, "..", *BOARD_SOURCES.split("/"))): + if os.path.basename(os.path.dirname(path)) not in variants: + continue + src = open(path, encoding="utf-8").read() + body = src[src.find("::handleCommand"):] + for lit in re.findall(r'(?:mem|str)n?cmp\(\s*command\s*,\s*"([^"]+)"', body): + found.add(lit.strip()) + return found + def firmware_commands(): """Top-level command literals the firmware dispatches on.""" @@ -96,7 +144,7 @@ def firmware_commands(): continue for lit in re.findall(r'(?:mem|str)n?cmp\(\s*command\s*,\s*"([^"]+)"', src): found.add(lit.strip()) - return found - NOT_OFFERED + return (found | board_commands()) - NOT_OFFERED ROUND_TRIPS = [ ("set radio.watchdog 30", "get radio.watchdog", "30"), ("set dutycycle 25", "get dutycycle", "25.0"), @@ -105,6 +153,8 @@ ROUND_TRIPS = [ ("set mqtt.neighbors on", "get mqtt.neighbors", "on"), ("set path.hash.mode 2", "get path.hash.mode", "2"), ("set mqtt.iata den", "get mqtt.iata", "DEN"), + # Board commands, exercised only when this board answers for them (see + # BOARD_ROUND_TRIPS below); the settings form drives the two FEM keys. # Secret reads are masked back down for an HTTP caller, in CommonCLI's own # words for a non-serial one (wcIsSecretReadCommand). ("set guest.password hunter2", "get guest.password", "******** (serial only)"), @@ -112,6 +162,14 @@ ROUND_TRIPS = [ ] +# Gated on /api/status's board_cmds, keyed by the gate the page probes for. +BOARD_ROUND_TRIPS = { + "radio.fem.rxgain": [("set radio.fem.rxgain off", "get radio.fem.rxgain", "off")], + "radio.fem.txgain": [("set radio.fem.txgain on", "get radio.fem.txgain", "on")], + "fan": [("set fan on", "get fan", "on 41.0C fan=on cd=0s")], +} + + class Client: def __init__(self, base): self.base = base @@ -119,7 +177,10 @@ class Client: self.cookie = r.headers["Set-Cookie"].split(";")[0] # The node caps a sequence at MAX_BATCH and reports it; chunk to match # rather than hardcoding a number that drifts when the slot is resized. - self.max_cmds = json.load(self._open("/api/status")).get("max_cmds", 24) + status = json.load(self._open("/api/status")) + self.max_cmds = status.get("max_cmds", 24) + # Board::handleCommand() commands this node probed for at startup. + self.board_cmds = [c for c in status.get("board_cmds", "").split(",") if c] def _open(self, path, data=None): headers = {"Content-Type": "application/json"} @@ -172,7 +233,7 @@ def main(): failures = [] - cmds = table() + cmds = table(cli.board_cmds) unexpected = [] for cmd, res in cli.run(cmds): if res["ok"]: @@ -182,6 +243,7 @@ def main(): continue unexpected.append((cmd, res["reply"])) print("commands offered by autocomplete : %d" % len(cmds)) + print("board commands this node answers : %s" % (", ".join(cli.board_cmds) or "none")) print("answered : %d" % (len(cmds) - len(unexpected))) print("sequence cap reported by the node: %d" % cli.max_cmds) for cmd, reply in unexpected: @@ -191,17 +253,39 @@ def main(): # The reverse direction: a command the firmware implements but the table # never offers is invisible to the check above, because the check only ever # drives what the table already knows about. - offered = " ".join(cmds) + " " + " ".join( - re.findall(r'\["([^"]+)","', open(INDEX_HTML, encoding="utf-8").read())) + # A board command is a whole `get`/`set` line, so the get-only list the audit + # drives cannot decide it is offered: `set fan.lo` has no `get` counterpart in + # the table at all, and a board key this mock does not claim is absent from + # `cmds` while still being offerable. Both are expanded from the page here. + html = open(INDEX_HTML, encoding="utf-8").read() + keys = re.findall(r'\["([^"]+)","(?:[^"\\]|\\.)*",(\d)', + html[html.index("var CLI_KEYS="):html.index("var CLI_BOARD_KEYS=")]) + offered = " ".join(cmds) + " " \ + + " ".join("set " + k for k, mode in keys if mode != "1") + " " \ + + " ".join("get %s set %s" % (k, k) for _, k, _ in board_table()) + " " \ + + " ".join(re.findall(r'\["([^"]+)","', html)) missing = sorted(c for c in firmware_commands() if c not in offered) print("\nfirmware commands not in the table: %d" % len(missing)) for c in missing: print(" MISSING %s" % c) failures += [(c, "not offered by autocomplete") for c in missing] - results = cli.run([c for probe in ROUND_TRIPS for c in probe[:2]]) - print("\nround-trips : %d" % len(ROUND_TRIPS)) - for i, (setc, getc, want) in enumerate(ROUND_TRIPS): + # The page asks the node for each gate by name (`get `), so a gate that + # no board getter answers would hide its keys on every board, silently. + fw = firmware_commands() + gates = sorted({gate for gate, _, _ in board_table()}) + bad_gates = [g for g in gates if "get " + g not in fw] + print("\nboard-command gates : %d" % len(gates)) + for g in bad_gates: + print(" FAIL %-30s no board answers `get %s`" % (g, g)) + failures += [(g, "gate has no getter") for g in bad_gates] + + probes = list(ROUND_TRIPS) + for gate in cli.board_cmds: + probes += BOARD_ROUND_TRIPS.get(gate, []) + results = cli.run([c for probe in probes for c in probe[:2]]) + print("\nround-trips : %d" % len(probes)) + for i, (setc, getc, want) in enumerate(probes): setr, getr = results[i * 2][1], results[i * 2 + 1][1] # `get` answers "> value"; compare the value, as the terminal displays it got = re.sub(r"^>\s?", "", getr["reply"]) @@ -214,7 +298,7 @@ def main(): # Commands the portal refuses must be refused clearly, not run and fudged. print("\nrefused with a reason : ", end="") refused = [] - for cmd in sorted(NOT_OFFERED - {"tls.bundletest"}): + for cmd in sorted(NOT_OFFERED - NOT_REFUSED): try: cli._sequence([cmd]) refused.append((cmd, "was accepted, expected a 400")) @@ -222,7 +306,8 @@ def main(): body = json.load(e) if e.code == 400 else {} if e.code != 400 or not body.get("error"): refused.append((cmd, "HTTP %d, expected 400 with a reason" % e.code)) - print("%d/%d" % (len(NOT_OFFERED) - 1 - len(refused), len(NOT_OFFERED) - 1)) + checked = len(NOT_OFFERED) - len(NOT_REFUSED) + print("%d/%d" % (checked - len(refused), checked)) for cmd, why in refused: print(" FAIL %-30s %s" % (cmd, why)) failures += refused diff --git a/scripts/webconfig_mock_server.py b/scripts/webconfig_mock_server.py index 74c6b730..4a36e555 100644 --- a/scripts/webconfig_mock_server.py +++ b/scripts/webconfig_mock_server.py @@ -101,6 +101,9 @@ def default_config(setup_mode): "radio": { "freq": 910.525, "bw": 62.5, "sf": 7, "cr": 5, "tx": 22, "af": 1.0, "rxdelay": 0.0, "txdelay": 0.5, "cad": False, "rxgain": True, + # Stored in NodePrefs like any other radio pref; only the COMMAND + # is board-specific, so the value exists even where nothing drives it. + "fem_rxgain": True, "fem_txgain": False, "repeat": True, "flood_max": 64, "flood_max_advert": 8, "flood_max_unscoped": 8, "loop_detect": "moderate", "name": "MockNode", "lat": 39.7392, "lon": -104.9903, @@ -149,6 +152,9 @@ class State: self.lock = threading.Lock() self.setup_mode = args.setup self.active_slots = args.active_slots + # Board::handleCommand() commands this "board" answers. Empty by default: + # the mock is a Heltec V3, which implements no such hook at all. + self.board_cmds = [c for c in args.board_cmds.split(",") if c] self.cfg = default_config(args.setup) # latched at AP start, like WebConfigServer::_initial_setup self.initial_setup = args.setup and self.cfg["wifi"]["ssid"] == "" @@ -192,6 +198,7 @@ class State: "role": "Repeater", "board": "Heltec V3 (mock)", "uptime_s": int(time.time() - self.start), "runtime_slots": 6, "max_slots": 6, "active_slots": self.active_slots, + "board_cmds": ",".join(self.board_cmds), "max_cmds": CLI_MAX_CMDS, } @@ -253,9 +260,6 @@ def apply_set(cfg, key, val): ADMIN_PASSWORD = val return True, "OK" - if key == "radio.fem.rxgain": - return False, "Error: unsupported" # no FEM on the mock board, see GETTERS - if key == "dutycycle": try: dc = float(val) @@ -340,13 +344,74 @@ def apply_set(cfg, key, val): sec, f = STR_KEYS[key] cfg[sec][f] = val return True, "OK" + gate = BOARD_CMD_GATE.get(key) + if gate: + if gate not in ST.board_cmds: + return False, "unknown config: %s" % key # no such command on this board + return apply_board_set(cfg, key, val) + # Strict fallthrough: this function is the single authority on what can be # set, for the batch and the CLI alike. Accepting unknown keys here once hid # the fact that the CLI could not reach `dutycycle` or `radio.fem.rxgain`. # Verbatim shape from CommonCLI::handleSetCmd's fallthrough. + # + # This is also where the board-specific commands land. `radio.fem.*` and + # `fan*` come from Board::handleCommand(), which is dispatched ahead of + # CommonCLI; the mock board is a Heltec V3 and implements no such hook, so + # they reach this fallthrough exactly as they do on the real thing. return False, "unknown config: %s" % key +# Commands that reach Board::handleCommand() rather than CommonCLI. Which ones a +# node answers is a property of the board, so the portal probes for them at +# startup and reports the answers in /api/status; --board-cmds picks which ones +# this mock claims. `fan.lo` / `fan.hi` have no getter and ride on `fan`. +BOARD_CMD_GATE = { + "radio.fem.rxgain": "radio.fem.rxgain", + "radio.fem.txgain": "radio.fem.txgain", + "fan": "fan", + "fan.lo": "fan", + "fan.hi": "fan", +} +BOARD_STATE = {"fan": "auto", "fan.lo": 45, "fan.hi": 60} +FEM_FIELD = {"radio.fem.rxgain": "fem_rxgain", "radio.fem.txgain": "fem_txgain"} + + +def apply_board_set(cfg, key, val): + if key in FEM_FIELD: + if val not in ("on", "off"): + return False, "Error: state must be on or off" + cfg["radio"][FEM_FIELD[key]] = val == "on" + return True, "OK - LoRa FEM %s gain %s" % ("RX" if "rx" in key else "TX", val) + if key == "fan": + if val not in ("on", "off", "auto"): + return False, "Error: fan must be on, off, or auto" + BOARD_STATE["fan"] = val + return True, "OK - fan %s" % val + try: + n = int(val) + except ValueError: + return False, "Error: expected a number" + if key == "fan.lo" and not (0 <= n <= 100 and n < BOARD_STATE["fan.hi"]): + return False, "Error: fan.lo must be 0..100 and < fan.hi" + if key == "fan.hi" and not (BOARD_STATE["fan.lo"] < n <= 120): + return False, "Error: fan.hi must be > fan.lo and <= 120" + BOARD_STATE[key] = n + return True, "OK - %s %d" % (key, n) + + +def board_get(cfg, key): + """Getter reply, or None when this board does not answer the command.""" + if BOARD_CMD_GATE.get(key) not in ST.board_cmds: + return None + if key == "fan": + return "%s 41.0C fan=%s cd=0s" % ( + BOARD_STATE["fan"], "on" if BOARD_STATE["fan"] == "on" else "off") + if key in FEM_FIELD: + return "on" if cfg["radio"][FEM_FIELD[key]] else "off" + return None # fan.lo / fan.hi are set-only on the board too + + # Payload-type names accepted alongside the decimal form. Mirrors # namedPacketTypes() in src/helpers/MQTTPacketFilter.h; 12-14 are reserved # upstream and stay reachable by number only. @@ -516,10 +581,6 @@ GETTERS = { "mqtt.ntp.diag": lambda c: "last sync: 42s ago via %s (offset +0.011s)" % (c["mqtt"]["ntp"] or "none"), "mqtt.stats": lambda c: ("published: %d\ndropped: 0\nqueue: 0/24\nreconnects: 1" % (100 + int(time.time() - ST.start))), - # Runtime-gated on the real device (Board::canControlLoRaFemLna), not - # compiled out — the command exists everywhere and the board answers for - # itself. The mock board is a Heltec V3, which has no FEM. - "radio.fem.rxgain": lambda c: None, } @@ -547,6 +608,11 @@ def cli_get(cfg, key): def _cli_get_value(cfg, key): + if key in BOARD_CMD_GATE: + val = board_get(cfg, key) + # Not answered: the board has no hook for it, so CommonCLI's own getter + # fallthrough is what replies — exactly as on the real thing. + return (True, val) if val is not None else (False, "??: %s" % key) if key in GETTERS: val = GETTERS[key](cfg) return (True, val) if val is not None else (False, "Error: unsupported") @@ -1053,6 +1119,9 @@ def main(): ap.add_argument("--port", type=int, default=8080) ap.add_argument("--setup", action="store_true", help="first-boot setup wizard mode") ap.add_argument("--active-slots", type=int, default=5, help="server slots to expose (2 or 5)") + ap.add_argument("--board-cmds", default="", + help="comma list of Board::handleCommand() commands to answer, e.g. " + "radio.fem.rxgain,radio.fem.txgain,fan (default: none, like a Heltec V3)") ap.add_argument("--fw-version", default=FW_VERSION, help="version string to report, shaped like build.sh's embedded one") ap.add_argument("--minify", action="store_true", diff --git a/src/helpers/WebConfigKeys.h b/src/helpers/WebConfigKeys.h index 47fd4149..c518be9a 100644 --- a/src/helpers/WebConfigKeys.h +++ b/src/helpers/WebConfigKeys.h @@ -19,7 +19,8 @@ static const char* const WC_ALLOWED_SET_KEYS[] = { // NodePrefs (radio / node) "name", "lat", "lon", "radio", "tx", "af", "rxdelay", "txdelay", - "cad", "radio.rxgain", "repeat", "advert.interval", "flood.advert.interval", + "cad", "radio.rxgain", "radio.fem.rxgain", "radio.fem.txgain", + "repeat", "advert.interval", "flood.advert.interval", "flood.max", "flood.max.advert", "flood.max.unscoped", "loop.detect", // MQTTPrefs (WiFi / MQTT / misc observer) "wifi.ssid", "wifi.pwd", "wifi.powersave", diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 6aab8da0..e14088be 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -93,6 +93,21 @@ static inline bool wcIsDeferredReboot(const char* cmd) { return strncmp(cmd, "reboot", 6) == 0; } +// Commands answered by Board::handleCommand(), which CommonCLI dispatches ahead +// of its own table. Whether a node answers them is a property of the BOARD, not +// of the build, so the page cannot know from the firmware version alone. Rather +// than mirror a per-board list here — which would have to be edited every time a +// variant grows a command — ask the board itself: probeBoardCommands() runs each +// getter once and keeps the ones that answer. +static const char* const WC_BOARD_CMDS[] = { + "radio.fem.rxgain", // front-end module LNA (Heltec V4/V4 R8/Tracker V2, Station G3) + "radio.fem.txgain", // front-end module PA level (Station G3) + "fan", // thermal fan, and with it fan.lo / fan.hi (T-Beam 1W) +}; +static const size_t WC_BOARD_CMD_COUNT = sizeof(WC_BOARD_CMDS) / sizeof(WC_BOARD_CMDS[0]); +static_assert(sizeof(WC_BOARD_CMDS) / sizeof(WC_BOARD_CMDS[0]) <= 8, + "_board_cmds is a uint8_t bitmask"); + // Commands the CLI reaches but the portal cannot honestly serve. Rejected at // POST so nothing in the sequence runs, rather than failing halfway with a // reply that does not explain itself. Returns the reason, or NULL if fine. @@ -455,6 +470,26 @@ void WebConfigServer::finalizeTeardown() { if (_cb) _cb->onWebConfigStopped(); } +// Ask the board which of WC_BOARD_CMDS it answers, once per start. The getters +// are pure reads, and the reply says everything needed: CommonCLI answers a real +// getter "> value", while every no-answer path — the "??:" fallthrough a board +// with no hook reaches, and the "Error: unsupported" a board with the hook but +// not the hardware returns — starts with something else. +// +// Loop task only: execCommand() reaches the CLI, which the async task must not. +void WebConfigServer::probeBoardCommands() { + char cmd[48], reply[160]; + uint8_t mask = 0; + for (size_t i = 0; i < WC_BOARD_CMD_COUNT; i++) { + snprintf(cmd, sizeof(cmd), "get %s", WC_BOARD_CMDS[i]); + reply[0] = 0; + _cb->execCommand(cmd, reply); + if (reply[0] == '>') mask |= (uint8_t)(1 << i); + } + _board_cmds = mask; + _board_cmds_probed = true; +} + void WebConfigServer::tick(uint32_t now) { if (_stopping) { uint32_t refs = handlerRefCount(); @@ -474,6 +509,8 @@ void WebConfigServer::tick(uint32_t now) { } if (_mode == MODE_OFF) return; + if (!_board_cmds_probed) probeBoardCommands(); + if (_mode == MODE_LAN && _initial_setup && _setup_reminder_at != 0 && (int32_t)(now - _setup_reminder_at) >= 0) { Serial.printf("WC: Ethernet setup http://%s/ code %s\n", @@ -743,7 +780,7 @@ void WebConfigServer::handleStatus(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } bool authed = checkAuth(req); - DynamicJsonDocument doc(512); + DynamicJsonDocument doc(640); doc["mode"] = (_mode == MODE_SETUP) ? "setup" : "lan"; doc["auth"] = authed; doc["needs_setup"] = !mqttNetworkSetupComplete(_obs); @@ -757,6 +794,19 @@ void WebConfigServer::handleStatus(AsyncWebServerRequest* req) { doc["build_date"] = _build_date; doc["role"] = _role; doc["board"] = _board_name; + // Board-specific CLI commands this board answers; the page hides the controls + // for everything absent here rather than offering one that cannot work. + char board_cmds[80]; + size_t n = 0; + board_cmds[0] = 0; // the nothing-supported case + for (size_t i = 0; i < WC_BOARD_CMD_COUNT && n < sizeof(board_cmds) - 1; i++) { + if (!(_board_cmds & (1 << i))) continue; + int w = snprintf(&board_cmds[n], sizeof(board_cmds) - n, "%s%s", + n ? "," : "", WC_BOARD_CMDS[i]); + if (w < 0) break; + n += (size_t)w; // snprintf NUL-terminates; a truncating w stops the loop + } + doc["board_cmds"] = board_cmds; doc["uptime_s"] = millis() / 1000; doc["runtime_slots"] = RUNTIME_MQTT_SLOTS; doc["max_slots"] = MAX_MQTT_SLOTS; @@ -867,6 +917,10 @@ void WebConfigServer::handleConfigGet(AsyncWebServerRequest* req) { radio["txdelay"] = _prefs->tx_delay_factor; radio["cad"] = (bool)_prefs->cad_enabled; radio["rxgain"] = (bool)_prefs->rx_boosted_gain; + // FEM gain is driven by Board::handleCommand(), not CommonCLI, so these are + // the stored intent; a board with no front-end module rejects the `set`. + radio["fem_rxgain"] = (bool)_prefs->radio_fem_rxgain; + radio["fem_txgain"] = (bool)_prefs->radio_fem_txgain; radio["repeat"] = !(bool)_prefs->disable_fwd; // CLI `repeat on` == disable_fwd 0 radio["flood_max"] = _prefs->flood_max; radio["flood_max_advert"] = _prefs->flood_max_advert; diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index 16ff47ba..b9083447 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -200,6 +200,12 @@ private: uint32_t _stats_built_at = 0; char _stats_json[1024] = {0}; + // Which Board::handleCommand() commands this board actually answers, as a + // bitmask over WC_BOARD_CMDS. Probed once per start on the loop task and then + // only read, so the async status handler needs no lock for it. + uint8_t _board_cmds = 0; + bool _board_cmds_probed = false; + void createServer(); void registerRoutes(); typedef void (WebConfigServer::*RequestHandler)(AsyncWebServerRequest*); @@ -208,6 +214,7 @@ private: void detachRoutes(); uint32_t handlerRefCount() const; void drainBatch(uint32_t now); + void probeBoardCommands(); void finalizeTeardown(); bool checkAuth(AsyncWebServerRequest* req); static void collectBody(AsyncWebServerRequest* req, uint8_t* data, size_t len, diff --git a/test/test_webconfig_keys/test_webconfig_keys.cpp b/test/test_webconfig_keys/test_webconfig_keys.cpp index 3903453b..9ce017f7 100644 --- a/test/test_webconfig_keys/test_webconfig_keys.cpp +++ b/test/test_webconfig_keys/test_webconfig_keys.cpp @@ -10,6 +10,8 @@ TEST(WebConfigKeys, AllowsKnownScalarKeys) { EXPECT_TRUE(wcIsAllowedSetKey("name")); EXPECT_TRUE(wcIsAllowedSetKey("radio")); EXPECT_TRUE(wcIsAllowedSetKey("repeat")); + EXPECT_TRUE(wcIsAllowedSetKey("radio.fem.rxgain")); // Board::handleCommand(), not CommonCLI + EXPECT_TRUE(wcIsAllowedSetKey("radio.fem.txgain")); EXPECT_TRUE(wcIsAllowedSetKey("wifi.ssid")); EXPECT_TRUE(wcIsAllowedSetKey("mqtt.iata")); EXPECT_TRUE(wcIsAllowedSetKey("mqtt.neighbors")); diff --git a/webui/index.html b/webui/index.html index 3102d352..c1c72e49 100644 --- a/webui/index.html +++ b/webui/index.html @@ -387,6 +387,10 @@ body.tab-cli{padding-bottom:0}
RX boosted gainSX126x receivers only
+
FEM RX gainExternal front-end module LNA +
+
FEM TX gainFront-end PA level. Station G3 needs the PA PL1 jumper removed. +
RepeatForward mesh traffic. Off = listen-only (still observes and publishes).
@@ -576,7 +580,7 @@ body.tab-cli{padding-bottom:0}