From 61955ee843058f82eb3e788cc9668935138e48fa Mon Sep 17 00:00:00 2001 From: Eldoon Nemar Date: Wed, 8 Jul 2026 09:00:57 -0400 Subject: [PATCH 01/84] Add MQTT preset for corecomms CoreComms is a map, analyzer, and mesh health platform created by EastMe.sh --- src/helpers/MQTTPresets.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index e372bf04..26575cc0 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -135,6 +135,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "rflab", "wss://mqtt.rflab.io:443", "mqtt.rflab.io", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "ipnt.uk", "wss://mqtt.ipnt.uk:443", "mqtt.ipnt.uk", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "flmesh", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "corecomms", "wss://mqtt.corecomms.net:443/mqtt", "mqtt.corecomms.net", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, }; // Find a preset by name, returns nullptr if not found From 4cff79695b6dd6220b411445ae8e74aa74bc41c0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Jul 2026 07:53:23 -0700 Subject: [PATCH 02/84] fix(mqtt): raise QoS1 retransmit timeout to stop duplicate /status storms esp-mqtt's default message_retransmit_timeout is 1000 ms: any unacked QoS 1 PUBLISH is resent (byte-identical, DUP=1) every second until the PUBACK arrives or the outbox entry expires (30 s). Status messages are the only QoS 1 publishes; on a congested or recovering uplink where broker acks take several seconds, each 5-minute /status was delivered ~6 times, ~1 s apart, as exact copies (same timestamp and stats). Downstream observers flagged excessive_packet_copies and at least one broker treats it as abuse. Expose message_retransmit_timeout via PsychicMqttClient and set it to 15 s in optimizeMqttClientConfig: one retry still fits inside the 30 s outbox expiry, preserving at-least-once delivery while capping duplicates at one. /packets paths are QoS 0 and were never affected. --- lib/PsychicMqttClient/src/PsychicMqttClient.cpp | 11 +++++++++++ lib/PsychicMqttClient/src/PsychicMqttClient.h | 11 +++++++++++ src/helpers/bridges/MQTTBridge.cpp | 9 +++++++++ 3 files changed, 31 insertions(+) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index c91f3a50..2ae395c8 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -59,6 +59,17 @@ PsychicMqttClient &PsychicMqttClient::setAutoReconnect(bool reconnect) return *this; } +PsychicMqttClient &PsychicMqttClient::setMessageRetransmitTimeout(int timeoutMs) +{ +#if ESP_IDF_VERSION_MAJOR == 5 + _mqtt_cfg.session.message_retransmit_timeout = timeoutMs; +#else + _mqtt_cfg.message_retransmit_timeout = timeoutMs; +#endif + _config_dirty = true; + return *this; +} + PsychicMqttClient &PsychicMqttClient::setClientId(const char *clientId) { #if ESP_IDF_VERSION_MAJOR == 5 diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index 80961d22..c45cf272 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -126,6 +126,17 @@ public: */ PsychicMqttClient &setAutoReconnect(bool reconnect = true); + /** + * @brief Sets the retransmit timeout for unacknowledged QoS 1/2 messages. + * esp-mqtt resends an unacked PUBLISH (DUP flag set) every time this + * timeout elapses, so a value shorter than the broker's ack latency + * produces byte-identical duplicates on the wire. + * + * @param timeoutMs Retransmit timeout in milliseconds. esp-mqtt's default is 1000. + * @return A reference to the PsychicMqttClient instance. + */ + PsychicMqttClient &setMessageRetransmitTimeout(int timeoutMs); + /** * @brief Sets the client ID for the MQTT connection. * diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 76eccfe3..a007ce71 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -3323,6 +3323,15 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ client->setKeepAlive(75); #endif + // QoS 1 retransmit timeout for unacked PUBLISHes (status messages). esp-mqtt's + // 1000 ms default resends a byte-identical duplicate every second whenever the + // broker's PUBACK takes >1s — on a congested or recovering uplink this floods + // subscribers with exact copies of one /status message (observed 6 copies ~1s + // apart after an ISP outage; brokers may drop the session as spam). 15s allows + // one retry before the outbox entry expires (esp-mqtt outbox expiry is 30s), + // preserving at-least-once delivery while capping duplicates at one. + client->setMessageRetransmitTimeout(15000); + // Buffer sizing: 896 is the minimum safe size for JWT clients (CONNECT + 768-byte JWT). // On PSRAM boards, use a uniform size to reduce fragmentation from mixed allocations. // On non-PSRAM boards, use smaller buffers for non-JWT slots to reduce heap usage and From 631a4e509f25555c29170d228f21db5b6579ae91 Mon Sep 17 00:00:00 2001 From: Adam Gessaman Date: Fri, 10 Jul 2026 08:18:24 -0700 Subject: [PATCH 03/84] Increase MQTT preset count from 25 to 26 --- src/helpers/MQTTPresets.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 26575cc0..e32923a1 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -105,7 +105,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 25; +static const int MQTT_PRESET_COUNT = 26; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { From 5a3f5a8cce3cb9523d83f93511a438fe0d2c955c Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Fri, 10 Jul 2026 12:33:15 -0600 Subject: [PATCH 04/84] Update ColoradoMesh connection string Signed-off-by: Nate Harris --- src/helpers/MQTTPresets.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index e32923a1..f3a46731 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -124,7 +124,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "chimesh", "wss://mqtt.chimesh.org:443", "mqtt.chimesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshat.se", "wss://meshcore-mqtt.meshat.se:443", "meshcore-mqtt.meshat.se", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "eastidahomesh", "wss://broker.eastidahomesh.net:443", nullptr, ISRG_ROOT_X1, MQTT_AUTH_NONE, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, - { "coloradomesh", "wss://mqtt.meshcore.coloradomesh.org:1883","mqtt.meshcore.coloradomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "coloradomesh", "wss://mqtt.meshcore.coloradomesh.org:443","mqtt.meshcore.coloradomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "dutchmeshcore-1", "wss://collector1.dutchmeshcore.nl:443/mqtt", "collector1.dutchmeshcore.nl", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "dutchmeshcore-2", "wss://collector2.dutchmeshcore.nl:443/mqtt", "collector2.dutchmeshcore.nl", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshcore-ca-1", "wss://mqtt1.meshcore.ca:443/mqtt", "mqtt1.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, From cd6ad2333abd4b239b454b842e06c216756cb2e1 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Jul 2026 18:51:17 -0700 Subject: [PATCH 05/84] fix(esp32): stop IDF 4.4 ws-transport heap overflow crashing bridge teardown The precompiled IDF 4.4 WebSocket transport (libtcp_transport.a) has an off-by-one in ws_connect(): when a wss:// endpoint answers the upgrade request with >=1024 bytes of HTTP response before the blank-line terminator (typical of a down broker behind a proxy serving a large error page), it writes a NUL one byte past the 1024-byte ws->buffer. Heap poisoning catches the clobbered tail canary (0xbaad5678 -> 0xbaad5600) only when the block is freed in ws_destroy() during esp_mqtt_client_destroy() - i.e. MQTTBridge::end() - so a single down broker made every deferred 'ota update' panic and reboot at teardown, before the download started. Decoded from a Heltec V3 crash backtrace on v1.16.0.11; line numbers match ESP-IDF release/v4.4 exactly. The transport code ships precompiled, so patch at link time instead: [esp32_base] wraps esp_transport_ws_init and the wrapper swaps the fresh buffer for a (WS_BUFFER_SIZE + 1)-byte allocation, making the out-of-bounds index land on owned memory. The oversized handshake then fails cleanly instead of corrupting the heap. Pass-through on IDF 5.x, where upstream already fixed it; delete with the Arduino core 3.x move. Verified: wrap resolves from ESP32WsTransportFix.cpp.o in the observer firmware.map; observer, room-server observer and plain repeater ESP32 targets build. RAK_4631_repeater failure is pre-existing (reproduced on the merge base without these changes). --- platformio.ini | 3 + src/helpers/AlertReporter.cpp | 10 ++- src/helpers/ESP32WsTransportFix.cpp | 130 ++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 src/helpers/ESP32WsTransportFix.cpp diff --git a/platformio.ini b/platformio.ini index e16f7b83..d38402c7 100644 --- a/platformio.ini +++ b/platformio.ini @@ -61,6 +61,9 @@ monitor_filters = esp32_exception_decoder extra_scripts = merge-bin.py build_flags = ${arduino_base.build_flags} -D ESP32_PLATFORM + ; Route esp_transport_ws_init through src/helpers/ESP32WsTransportFix.cpp to + ; fix a heap-overflow in the precompiled IDF 4.4 WS transport (see that file). + -Wl,--wrap=esp_transport_ws_init ; -D ESP32_CPU_FREQ=80 ; change it to your need build_src_filter = ${arduino_base.build_src_filter} diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp index a6841277..4bce1fd0 100644 --- a/src/helpers/AlertReporter.cpp +++ b/src/helpers/AlertReporter.cpp @@ -218,6 +218,12 @@ void AlertReporter::onLoop(unsigned long now_ms) { // already enforces this on set, but a stale prefs file or future field // tweak shouldn't be able to drag the floor below 1 hour and let a // flapping link spam the mesh. + // + // The rate limiter only applies between two real sends: fired_at_ms == 0 + // means "never fired since boot/config change", and treating it as a send + // at millis()==0 would suppress every first alert until uptime reaches + // min_interval (observed as a 30-minute alert.mqtt threshold not reporting + // until 60 minutes after a reboot). uint16_t cfg_min = _obs->alert_min_interval_min; if (cfg_min < 60) cfg_min = 60; unsigned long min_interval_ms = (unsigned long)cfg_min * 60000UL; @@ -232,7 +238,7 @@ void AlertReporter::onLoop(unsigned long now_ms) { if (_wifi.state == OK) { if (wifi_down && down_ms >= thresh_ms && - (now_ms - _wifi.fired_at_ms) >= min_interval_ms) { + (_wifi.fired_at_ms == 0 || (now_ms - _wifi.fired_at_ms) >= min_interval_ms)) { char age[16]; formatAge(down_ms, age, sizeof(age)); uint8_t reason = MQTTBridge::getLastWifiDisconnectReason(); @@ -282,7 +288,7 @@ void AlertReporter::onLoop(unsigned long now_ms) { if (f.state == OK) { if (down && down_ms >= thresh_ms && - (now_ms - f.fired_at_ms) >= min_interval_ms) { + (f.fired_at_ms == 0 || (now_ms - f.fired_at_ms) >= min_interval_ms)) { char age[16]; formatAge(down_ms, age, sizeof(age)); char text[100]; diff --git a/src/helpers/ESP32WsTransportFix.cpp b/src/helpers/ESP32WsTransportFix.cpp new file mode 100644 index 00000000..2e14bb33 --- /dev/null +++ b/src/helpers/ESP32WsTransportFix.cpp @@ -0,0 +1,130 @@ +#ifdef ESP_PLATFORM + +// Link-time workaround for an off-by-one heap overflow in ESP-IDF v4.4's +// WebSocket transport (components/tcp_transport/transport_ws.c), which ships +// PRECOMPILED in the Arduino-ESP32 2.x SDK (libtcp_transport.a) and cannot be +// patched at source level. +// +// The bug (transport_ws.c, ws_connect() response-read loop): +// +// header_len += len; +// ws->buffer[header_len] = '\0'; // header_len can reach WS_BUFFER_SIZE +// } while (... && header_len < WS_BUFFER_SIZE); +// +// ws->buffer is malloc(WS_BUFFER_SIZE) (1024). When a wss:// endpoint answers +// the WebSocket upgrade with >= 1024 bytes of HTTP response before the blank +// line terminator (typical for a down/misconfigured broker behind a proxy or +// CDN that serves a large HTML error page), the final iteration writes one +// '\0' one byte past the block. With heap poisoning enabled that zeroes the +// LSB of the tail canary (0xbaad5678 -> 0xbaad5600); the corruption then sits +// silent until the block is freed — which happens in ws_destroy() during +// esp_mqtt_client_destroy(), i.e. MQTTBridge::end() — and the free asserts: +// +// CORRUPT HEAP: Bad tail at 0x.... Expected 0xbaad5678 got 0xbaad5600 +// assert failed: multi_heap_free multi_heap_poisoning.c:259 +// +// On observer builds that teardown runs at the start of the deferred +// `ota update`, so a single down wss broker made every online OTA panic and +// reboot before the download began (backtrace decoded from a Heltec V3 on +// v1.16.0.11: free <- ws_destroy <- esp_transport_list_destroy <- +// esp_mqtt_client_destroy <- ~PsychicMqttClient <- destroySlotClients <- +// MQTTBridge::end <- MyMesh::setBridgeState <- MyMesh::loop). +// +// Fix: [esp32_base] adds `-Wl,--wrap=esp_transport_ws_init`, so every +// creation of a WS transport (esp-mqtt does one per wss slot) is routed +// through __wrap_esp_transport_ws_init below, which replaces the freshly +// allocated 1024-byte buffer with a (WS_BUFFER_SIZE + 1)-byte one. The +// out-of-bounds index WS_BUFFER_SIZE then lands on our extra byte and the +// handshake fails cleanly ("Upgrade" header not found) instead of corrupting +// the heap. Upstream fixed this in ESP-IDF 5.x, so this file compiles to a +// pass-through there and can be deleted (together with the --wrap flag) when +// the fork moves to Arduino core 3.x. +// +// transport_ws_t below is copied verbatim from ESP-IDF release/v4.4 +// transport_ws.c (the struct is file-private, so it is not in any shipped +// header). Source fidelity was verified against the shipped binary: addr2line +// on the crash backtrace resolves to the exact line numbers of that file +// (e.g. free(ws->buffer) at transport_ws.c:546). Only the first two members +// (path, buffer) are dereferenced here. + +#include "esp_idf_version.h" + +#if ESP_IDF_VERSION_MAJOR == 4 + +#include +#include "sdkconfig.h" +#include "esp_transport.h" +#include "esp_transport_ws.h" + +#ifndef CONFIG_WS_BUFFER_SIZE +#define CONFIG_WS_BUFFER_SIZE 1024 +#endif + +// --- copied from ESP-IDF release/v4.4 components/tcp_transport/transport_ws.c --- +typedef struct { + uint8_t opcode; + char mask_key[4]; + int payload_len; + int bytes_remaining; + bool header_received; +} ws_transport_frame_state_t; + +typedef struct { + char *path; + char *buffer; + char *sub_protocol; + char *user_agent; + char *headers; + bool propagate_control_frames; + ws_transport_frame_state_t frame_state; + esp_transport_handle_t parent; +} transport_ws_t; +// -------------------------------------------------------------------------------- + +extern "C" { + +esp_transport_handle_t __real_esp_transport_ws_init(esp_transport_handle_t parent_handle); + +esp_transport_handle_t __wrap_esp_transport_ws_init(esp_transport_handle_t parent_handle) { + esp_transport_handle_t t = __real_esp_transport_ws_init(parent_handle); + if (t != nullptr) { + transport_ws_t* ws = (transport_ws_t*)esp_transport_get_context_data(t); + if (ws != nullptr && ws->buffer != nullptr) { + // The buffer is untouched at this point (allocated moments ago inside + // __real_esp_transport_ws_init), so a swap is safe. + char* padded = (char*)malloc(CONFIG_WS_BUFFER_SIZE + 1); + if (padded != nullptr) { + free(ws->buffer); + ws->buffer = padded; + } + // On alloc failure keep the original buffer: same behavior as before + // this fix, which is still strictly better than failing init here. + } + } + return t; +} + +} // extern "C" + +#else // ESP_IDF_VERSION_MAJOR != 4 + +// IDF 5.x fixed the overflow upstream; keep a pass-through so the --wrap flag +// (set for all ESP32 envs in [esp32_base]) still links if anything references +// the symbol. + +#include "esp_transport.h" +#include "esp_transport_ws.h" + +extern "C" { + +esp_transport_handle_t __real_esp_transport_ws_init(esp_transport_handle_t parent_handle); + +esp_transport_handle_t __wrap_esp_transport_ws_init(esp_transport_handle_t parent_handle) { + return __real_esp_transport_ws_init(parent_handle); +} + +} // extern "C" + +#endif // ESP_IDF_VERSION_MAJOR + +#endif // ESP_PLATFORM From c119ad2ab9f4f853eb6a0c290e0f8a9602f70315 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Jul 2026 19:03:38 -0700 Subject: [PATCH 06/84] fix(mqtt): gate reconnect-backoff reset on 2 min of connection stability A CONNACK alone reset the backoff ladder, so a flapping broker (accepts then drops within seconds) retried at the 10 s rung forever - each attempt a full ~40 KB TLS session alloc/free on internal heap, a known fragmentation driver. The ladder now clears only after the connection survives 2 minutes (at least one 75 s keepalive round-trip); flapping endpoints degrade to the 300 s rung and then the existing 30-minute circuit-breaker probes, and recover automatically once stable. --- src/helpers/bridges/MQTTBridge.cpp | 30 +++++++++++++++++++++++++++--- src/helpers/bridges/MQTTBridge.h | 2 ++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index a007ce71..df85722c 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1119,8 +1119,18 @@ void MQTTBridge::initSlotClients() { slot.client->onConnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); _slots[index].connected = true; - _slots[index].reconnect_backoff = 0; - _slots[index].max_backoff_failures = 0; + // NOTE: reconnect_backoff / max_backoff_failures are NOT reset here. + // A CONNACK alone doesn't prove the link is healthy — a broker that + // accepts and then drops within seconds would reset the ladder every + // cycle and retry at the 10 s rung forever, and each retry is a full + // TLS session alloc/free (~40 KB of internal-heap churn, a known + // fragmentation driver). The ladder is instead cleared by + // maintainSlotConnection() once the connection has stayed up for + // BACKOFF_STABLE_RESET_MS, so flapping endpoints keep their earned + // backoff level. The breaker itself does clear now: while connected + // the diag/status must not claim the slot gave up, and the next + // disconnect should be governed by the (still-elevated) ladder. + _slots[index].connected_at_ms = millis(); _slots[index].circuit_breaker_tripped = false; _slots[index].last_tls_err = 0; _slots[index].last_tls_stack_err = 0; @@ -1140,6 +1150,7 @@ void MQTTBridge::initSlotClients() { _slots[index].current_outage_started_ms = millis(); } _slots[index].connected = false; + _slots[index].connected_at_ms = 0; // stability clock only runs while connected updateCachedConnectionStatus(); }); slot.client->onError([this, index](esp_mqtt_error_codes error) { @@ -1453,7 +1464,20 @@ void MQTTBridge::maintainSlotConnections() { void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted) { MQTTSlot& slot = _slots[index]; - if (slot.connected) { + // Forgive past failures only after the connection has proven stable. + // 2 minutes covers at least one keepalive round-trip (keepalive is 75 s), + // so a link that can't survive a single keepalive period never resets the + // ladder. Flapping endpoints therefore stay at their earned backoff rung + // (worst case the 300 s rung / 30-minute breaker probes) instead of + // hammering full TLS handshakes at the 10 s rung — see the onConnect + // handler in initSlotClients() for why this doesn't happen on CONNACK. + static const unsigned long BACKOFF_STABLE_RESET_MS = 120000UL; + if (slot.connected && + (slot.reconnect_backoff != 0 || slot.max_backoff_failures != 0) && + slot.connected_at_ms != 0 && + (now_millis - slot.connected_at_ms) >= BACKOFF_STABLE_RESET_MS) { + MQTT_DEBUG_PRINTLN("MQTT%d stable for %lus - clearing reconnect backoff (was level %d)", + index + 1, (now_millis - slot.connected_at_ms) / 1000UL, slot.reconnect_backoff); slot.reconnect_backoff = 0; slot.max_backoff_failures = 0; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 41365312..15db1261 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -89,6 +89,8 @@ private: uint8_t reconnect_backoff; // 0..4 index into backoff table uint8_t max_backoff_failures; // consecutive failures at max backoff level bool circuit_breaker_tripped; // true = stop reconnecting until reconfigured + unsigned long connected_at_ms; // millis() of last successful connect (0 = not connected); + // gates the stability-based backoff reset in maintenance unsigned long last_reconnect_attempt; unsigned long last_log_time; // Throttle disconnect log messages unsigned long last_deferred_log_ms; // Throttle "connect deferred" log spam (Phase 1) From eca1f2c0dc6770529b550b70bfc4eade540b965d Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Jul 2026 19:41:41 -0700 Subject: [PATCH 07/84] fix(mqtt): scale JWT renewal buffer with token lifetime The token's exp claim and the renewal schedule derive from the same value, so the flat 60 s RENEWAL_BUFFER was the entire margin between proactive re-auth and the broker enforcing exp on the live session - one failed renewal attempt (60 s throttle) or a minute of clock skew lost the race, seen as clean-FIN disconnects (tls=0x8008) on the waev preset, whose 55-minute tokens are the only ones short enough to hit enforcement. Buffer is now lifetime/10 clamped to [60 s, 300 s], and the disconnect-now threshold uses the same value so every renewal is a proactive reconnect on the device's schedule; waev re-auths 10 min before its real 60-minute TTL with ~5 retry windows. Document why waev's preset claims 3300 s against the broker's real 3600 s TTL: the 5-minute claim-side gap protects token acceptance against fast device clocks, which the renewal buffer cannot do. --- src/helpers/MQTTPresets.h | 5 +++ src/helpers/bridges/MQTTBridge.cpp | 60 +++++++++++++++++++++++++----- src/helpers/bridges/MQTTBridge.h | 2 + 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index e32923a1..db53d43d 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -115,6 +115,11 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "nz-analyzer", "wss://meshcore-mqtt-1.baird.io:443", "meshcore-mqtt-1.baird.io", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshmapper", "wss://mqtt.meshmapper.net:443/mqtt", "mqtt.meshmapper.net", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshrank", "mqtts://meshrank.net:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_NONE, MQTT_TOPIC_MESHRANK, 0, false, 0, nullptr, nullptr }, + // waev token_lifetime is 3300 (55 min) on purpose: the broker's real JWT TTL is + // 60 min, and claiming less keeps fresh tokens accepted even with ~5 min of fast + // device-clock skew (and off any exp-iat<=3600 boundary strictness). Do NOT + // "fix" this to 3600 — the renewal race is handled separately by + // tokenRenewalBufferSecs() in MQTTBridge, which renews another 5 min earlier. { "waev", "wss://mqtt.waev.app:443/mqtt", "mqtt.waev.app", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 3300, false, 55, nullptr, nullptr }, { "meshomatic", "wss://us-east.meshomatic.net:443/mqtt", "us-east.meshomatic.net", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "cascadiamesh", "wss://mqtt-v1.cascadiamesh.org:443/mqtt", "mqtt-v1.cascadiamesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index df85722c..aa737d98 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1486,15 +1486,19 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns bool slot_uses_jwt = (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) || (!slot.preset && slot.audience[0] != '\0'); if (slot_uses_jwt) { + // Renew (and below, reconnect) this many seconds before the token's exp + // claim. Scaled to the slot's token lifetime — see tokenRenewalBufferSecs + // for why a flat 60 s lost the renewal race against brokers that enforce + // exp on live sessions (waev's 55-minute tokens). + const unsigned long renewal_buffer = tokenRenewalBufferSecs(slotTokenLifetime(index)); bool token_needs_renewal = false; if (!time_synced) { token_needs_renewal = (slot.token_expires_at == 0); } else { - const unsigned long RENEWAL_BUFFER = 60; token_needs_renewal = (slot.token_expires_at == 0) || !(slot.token_expires_at >= 1000000000) || (current_time >= slot.token_expires_at) || - (current_time >= (slot.token_expires_at - RENEWAL_BUFFER)); + (current_time >= (slot.token_expires_at - renewal_buffer)); } // Throttle renewal attempts to once per minute @@ -1509,12 +1513,16 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns if (createSlotAuthToken(index)) { MQTT_DEBUG_PRINTLN("MQTT%d token renewed", index + 1); - const unsigned long DISCONNECT_THRESHOLD = 60; + // Bounce the connection while WE control the timing whenever the old + // token is inside the renewal buffer — waiting for the broker to + // enforce exp mid-session means a FIN plus a trip through the backoff + // ladder instead of one clean reconnect. Same buffer as the renewal + // trigger above, so a renewal implies a proactive reconnect. bool old_token_expired_or_imminent = !time_synced || (old_token_expires_at == 0) || (current_time >= old_token_expires_at) || (time_synced && old_token_expires_at >= 1000000000 && - current_time >= (old_token_expires_at - DISCONNECT_THRESHOLD)); + current_time >= (old_token_expires_at - renewal_buffer)); if (old_token_expired_or_imminent || !slot.client->connected()) { // Disconnect + reconnect with fresh credentials, reusing existing client @@ -1632,6 +1640,43 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns } } +// Effective JWT lifetime for a slot: the preset's token_lifetime (or the 24 h +// default for custom/audience slots), minus the per-slot expiry stagger that +// keeps multiple JWT slots from renewing/reconnecting simultaneously. This is +// the exact value createSlotAuthToken() puts in the token's exp claim, so the +// renewal scheduling in maintainSlotConnection() can be derived from it. +unsigned long MQTTBridge::slotTokenLifetime(int index) const { + const MQTTSlot& slot = _slots[index]; + unsigned long base_lifetime = 86400; // default 24h + if (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT && slot.preset->token_lifetime > 0) { + base_lifetime = slot.preset->token_lifetime; + } + // Stagger token expiry per slot to avoid simultaneous renewal/reconnect. + // Use 5% of lifetime per slot, capped at 300s, so short-lived tokens aren't over-reduced. + unsigned long stagger = index * min((unsigned long)300, base_lifetime / 20); + return base_lifetime - stagger; +} + +// How early (seconds before the token's exp claim) to renew the token AND +// proactively bounce the connection with fresh credentials. exp and the +// renewal schedule are locked together (both derive from slotTokenLifetime), +// so this buffer is the ONLY margin between "device re-authenticates" and +// "broker enforces exp and FIN-closes the session mid-stream" — shortening a +// preset's token_lifetime moves both times together and cannot widen it. +// The old flat 60 s lost that race whenever the device clock ran slow, or a +// single renewal attempt failed (the 60 s renewal throttle then ate the whole +// margin) — observed on the waev preset, whose 55-minute tokens are the only +// ones short enough for brokers to enforce exp against a live session. +// lifetime/10 with a 60 s floor and 300 s cap: 24 h tokens renew 5 min early +// (unchanged in practice), waev renews ~5 min early with ~5 throttled retry +// windows, and degenerate short lifetimes still renew inside their validity. +unsigned long MQTTBridge::tokenRenewalBufferSecs(unsigned long lifetime_secs) { + unsigned long buffer = lifetime_secs / 10; + if (buffer < 60) buffer = 60; + if (buffer > 300) buffer = 300; + return buffer; +} + bool MQTTBridge::createSlotAuthToken(int index) { if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; MQTTSlot& slot = _slots[index]; @@ -1639,10 +1684,8 @@ bool MQTTBridge::createSlotAuthToken(int index) { // Determine JWT audience: preset takes priority, then custom slot audience field const char* audience = nullptr; - unsigned long base_lifetime = 86400; // default 24h if (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) { audience = slot.preset->jwt_audience; - if (slot.preset->token_lifetime > 0) base_lifetime = slot.preset->token_lifetime; } else if (slot.audience[0] != '\0') { audience = slot.audience; } @@ -1672,10 +1715,7 @@ bool MQTTBridge::createSlotAuthToken(int index) { const char* email = (_obs->mqtt_email[0] != '\0') ? _obs->mqtt_email : nullptr; unsigned long current_time = time(nullptr); - // Stagger token expiry per slot to avoid simultaneous renewal/reconnect - // Use 5% of lifetime per slot, capped at 300s, so short-lived tokens aren't over-reduced - unsigned long stagger = index * min((unsigned long)300, base_lifetime / 20); - unsigned long expires_in = base_lifetime - stagger; + unsigned long expires_in = slotTokenLifetime(index); // preset/default lifetime minus per-slot stagger bool time_synced = (current_time >= 1000000000); if (JWTHelper::createAuthToken( diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 15db1261..377ea51c 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -340,6 +340,8 @@ private: void maintainSlotConnections(); // Maintain all slot connections (token renewal, reconnect) void maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted); bool createSlotAuthToken(int index); // Create/renew JWT token for a slot + unsigned long slotTokenLifetime(int index) const; // effective JWT lifetime (preset/default minus slot stagger), seconds + static unsigned long tokenRenewalBufferSecs(unsigned long lifetime_secs); // how early to renew+reconnect before exp bool publishToSlot(int index, const char* topic, const char* payload, bool retained = false, uint8_t qos = 0); bool publishToAllSlots(const char* topic, const char* payload, bool retained = false, uint8_t qos = 0); void publishStatusToSlot(int index); From cdcef6e161cf71cd21ea665b7413839750c775d7 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 15:40:21 -0700 Subject: [PATCH 08/84] feat(mqtt): add dual-stack IPv4/IPv6 support to MQTT bridge --- MQTT_IMPLEMENTATION.md | 20 ++++++++++++++ src/helpers/CommonCLI_Observer.cpp | 23 ++++++++++++++-- src/helpers/bridges/MQTTBridge.cpp | 43 +++++++++++++++++++++++++++++- src/helpers/bridges/MQTTBridge.h | 4 +++ 4 files changed, 87 insertions(+), 3 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 5ba66247..301b281a 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -249,6 +249,26 @@ The MQTT bridge comes with the following defaults for fresh installs (unless ove - **Timezone Offset**: 0 (fallback, no offset, unless `MQTT_DEFAULT_TIMEZONE_OFFSET` is set) - **Repeat (forwarding)**: On (set `repeat off` for receive-only observers) +## IPv6 Support + +Observer builds run **dual-stack**: IPv4 continues to work exactly as before, and the node +*additionally* acquires an IPv6 address when the network supports it. This is fully automatic +and requires no configuration. + +- **How it works**: once WiFi has an IPv4 address, the node enables IPv6 and obtains a global + address via Router Advertisement / SLAAC. A dual-stack router with RA/SLAAC is required; + on IPv4-only networks the node simply stays IPv4-only (graceful degradation). +- **Visibility**: the global IPv6 address appears in `get wifi.status` once assigned, e.g. + `> connected, IP: 192.168.1.42, IPv6: 2001:db8::abcd, RSSI: -62 dBm, uptime: ...`. + The field is omitted when no global address is present (link-local is not reported). IPv6 is + CLI/serial-visible only — it is not shown on the OLED. +- **Custom broker over IPv6**: use a bracketed literal in a full URI + (`set mqttN.server mqtts://[2001:db8::1]:8883`), or a bare literal + (`set mqttN.server 2001:db8::1`) which is bracketed automatically. Hostname presets need no + changes — DNS resolves AAAA records automatically once the stack is dual-stack. +- **Cost**: none to budget. IPv6 is already compiled into the ESP32 Arduino lwIP that every + build links; enabling it at runtime only adds a couple of address slots. + ## CLI Commands ### MQTT Slot Commands diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 413e92b4..22de73cf 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -780,7 +780,26 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf default: status_str = "unknown"; break; } if (status == WL_CONNECTED) { - sprintf(reply, "> %s, IP: %s, RSSI: %d dBm", status_str, WiFi.localIP().toString().c_str(), WiFi.RSSI()); + // reply points at the caller's char[160] command buffer (see main.cpp). + const size_t kReplyBufSize = 160; + sprintf(reply, "> %s, IP: %s", status_str, WiFi.localIP().toString().c_str()); +#ifdef WITH_MQTT_BRIDGE + // Group IPv6 directly after IPv4 when a global/ULA address is assigned. + char v6[46]; + if (MQTTBridge::getGlobalIPv6(v6, sizeof(v6))) { + size_t v6_len = strlen(reply); + if (v6_len < kReplyBufSize) { + snprintf(reply + v6_len, kReplyBufSize - v6_len, ", IPv6: %s", v6); + } + } +#endif + // RSSI right after the IP addresses. + { + size_t rssi_len = strlen(reply); + if (rssi_len < kReplyBufSize) { + snprintf(reply + rssi_len, kReplyBufSize - rssi_len, ", RSSI: %d dBm", WiFi.RSSI()); + } + } #ifdef WITH_MQTT_BRIDGE unsigned long connect_at = MQTTBridge::getWifiConnectedAtMillis(); if (connect_at != 0) { @@ -791,7 +810,7 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf unsigned long m = (uptime_sec % 3600) / 60; unsigned long s = uptime_sec % 60; size_t len = strlen(reply); - const size_t reply_remaining = 128; + const size_t reply_remaining = (len < kReplyBufSize) ? (kReplyBufSize - len) : 0; if (d > 0) { snprintf(reply + len, reply_remaining, ", uptime: %lud %luh %lum %lus", d, h, m, s); } else if (h > 0) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index aa737d98..eeae09d2 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -15,6 +15,7 @@ #ifdef ESP_PLATFORM #include +#include #include #include #include @@ -179,6 +180,10 @@ static unsigned long s_wifi_connected_at = 0; static uint8_t s_wifi_disconnect_reason = 0; static unsigned long s_wifi_disconnect_time = 0; +// Most recent global/unique-local IPv6 address (SLAAC), as a string; empty when none. +// Populated from the ARDUINO_EVENT_WIFI_STA_GOT_IP6 handler; cleared on WiFi disconnect. +static char s_global_ipv6[46] = ""; + #ifdef MQTT_MEMORY_DEBUG // #region agent log static void agentLogHeap(const char* location, const char* message, const char* hypothesisId, @@ -201,6 +206,17 @@ unsigned long MQTTBridge::getWifiConnectedAtMillis() { return s_wifi_connected_at; } +bool MQTTBridge::getGlobalIPv6(char* buf, size_t len) { + if (buf == nullptr || len == 0) return false; + if (s_global_ipv6[0] == '\0') { + buf[0] = '\0'; + return false; + } + strncpy(buf, s_global_ipv6, len - 1); + buf[len - 1] = '\0'; + return true; +} + void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPrefs* obs) { if (buf == nullptr || bufsize == 0) return; const char* msgs = (obs && obs->mqtt_status_enabled) ? "on" : "off"; @@ -829,11 +845,23 @@ void MQTTBridge::initializeWiFiInTask() { switch(event) { case ARDUINO_EVENT_WIFI_STA_GOT_IP: MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); + // Kick off IPv6 link-local + RA/SLAAC (additive; IPv4 path unchanged). Idempotent. + WiFi.enableIpV6(); // Set flag to trigger NTP sync from loop() instead of doing it here if (!_ntp_synced && !_ntp_sync_pending) { _ntp_sync_pending = true; } break; + case ARDUINO_EVENT_WIFI_STA_GOT_IP6: { + // Store only global/unique-local addresses; link-local isn't useful for diagnostics. + esp_ip6_addr_t ip6 = info.got_ip6.ip6_info.ip; + esp_ip6_addr_type_t type = esp_netif_ip6_get_addr_type(&ip6); + if (type == ESP_IP6_ADDR_IS_GLOBAL || type == ESP_IP6_ADDR_IS_UNIQUE_LOCAL) { + snprintf(s_global_ipv6, sizeof(s_global_ipv6), IPV6STR, IPV62STR(ip6)); + MQTT_DEBUG_PRINTLN("WiFi IPv6: %s", s_global_ipv6); + } + break; + } case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; s_wifi_disconnect_time = millis(); @@ -1325,7 +1353,16 @@ void MQTTBridge::setupSlot(int index) { } else if (slot.port == 443) { proto = "wss"; } - snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); + // Bare IPv6 literals (contain ':' but no '.', not already bracketed) must be wrapped + // in [..] so the ":port" suffix isn't mistaken for part of the address. + bool bare_ipv6 = (slot.host[0] != '[') && + (strchr(slot.host, ':') != nullptr) && + (strchr(slot.host, '.') == nullptr); + if (bare_ipv6) { + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://[%s]:%d", proto, slot.host, slot.port); + } else { + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); + } } slot.client->setServer(slot.broker_uri); MQTT_DEBUG_PRINTLN("MQTT%d custom broker URI: %s (host='%s', port=%u)", @@ -2080,6 +2117,9 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { _wifi_disconnected_time = 0; s_wifi_connected_at = now; _wifi_reconnect_backoff_attempt = 0; + // Re-arm IPv6 link-local after a reconnect (covers paths that re-run WiFi.begin + // and skip the GOT_IP event ordering). Idempotent. + WiFi.enableIpV6(); #ifdef ESP_PLATFORM wifi_ps_type_t ps_mode; uint8_t ps_pref = _obs->wifi_power_save; @@ -2106,6 +2146,7 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (_last_wifi_status == WL_CONNECTED) { _wifi_disconnected_time = now; s_wifi_connected_at = 0; + s_global_ipv6[0] = '\0'; // drop stale IPv6 so get wifi.status doesn't report it // Disconnect all slot clients when WiFi drops for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].client && _slots[i].connected) { diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 377ea51c..ee5b8b98 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -430,6 +430,10 @@ public: static unsigned long getWifiConnectedAtMillis(); + // Copies the current global/unique-local IPv6 address (string) into buf. + // Returns false (and writes an empty string) when no global IPv6 is assigned. + static bool getGlobalIPv6(char* buf, size_t len); + /** * Per-slot outage accessors used by AlertReporter to detect prolonged * MQTT broker outages. Indices are 0..RUNTIME_MQTT_SLOTS-1. From ba1dbefb5989cfa77c532f45626d70ba454f5fb0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Jul 2026 23:14:22 -0700 Subject: [PATCH 09/84] =?UTF-8?q?revert(mqtt):=20remove=20IPv6=20dual-stac?= =?UTF-8?q?k=20support=20=E2=80=94=20measured=20heap=20exhaustion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts merge ae045395 (feat/flex-ipv6, PR #25). Enabling IPv6 joins the device into IPv6 multicast/ND processing; on multicast-heavy LANs (e.g. with a Thread/Matter border router advertising a ULA prefix) inbound bursts land in dynamic WiFi RX buffers in internal heap. Measured on a Station G2 with 4 WSS brokers: min-free floor dropped 36 KB -> 18 KB, largest free block pinned below the 16 KB publish threshold for minutes at a time, 523 dropped publishes in 15 minutes. With IPv6 disabled the floor and max-alloc recovered and publish skips stopped. The feature only fed the wifi.status display line — no transport uses IPv6 (all brokers connect over IPv4), so the fleet risk (network- dependent degradation on unknown home LANs) buys nothing. Revisit after the Arduino core 3.x / IDF 5.x move if IPv6 transport is ever needed; the branch remains at feat/flex-ipv6. Kept from the merge: the wifi.status uptime append now computes the actual remaining space in the 160-byte reply buffer instead of assuming a hardcoded 128, fixing a latent overflow of the snprintf bound. --- MQTT_IMPLEMENTATION.md | 20 -------------- src/helpers/CommonCLI_Observer.cpp | 24 +++-------------- src/helpers/bridges/MQTTBridge.cpp | 43 +----------------------------- src/helpers/bridges/MQTTBridge.h | 4 --- 4 files changed, 5 insertions(+), 86 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 301b281a..5ba66247 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -249,26 +249,6 @@ The MQTT bridge comes with the following defaults for fresh installs (unless ove - **Timezone Offset**: 0 (fallback, no offset, unless `MQTT_DEFAULT_TIMEZONE_OFFSET` is set) - **Repeat (forwarding)**: On (set `repeat off` for receive-only observers) -## IPv6 Support - -Observer builds run **dual-stack**: IPv4 continues to work exactly as before, and the node -*additionally* acquires an IPv6 address when the network supports it. This is fully automatic -and requires no configuration. - -- **How it works**: once WiFi has an IPv4 address, the node enables IPv6 and obtains a global - address via Router Advertisement / SLAAC. A dual-stack router with RA/SLAAC is required; - on IPv4-only networks the node simply stays IPv4-only (graceful degradation). -- **Visibility**: the global IPv6 address appears in `get wifi.status` once assigned, e.g. - `> connected, IP: 192.168.1.42, IPv6: 2001:db8::abcd, RSSI: -62 dBm, uptime: ...`. - The field is omitted when no global address is present (link-local is not reported). IPv6 is - CLI/serial-visible only — it is not shown on the OLED. -- **Custom broker over IPv6**: use a bracketed literal in a full URI - (`set mqttN.server mqtts://[2001:db8::1]:8883`), or a bare literal - (`set mqttN.server 2001:db8::1`) which is bracketed automatically. Hostname presets need no - changes — DNS resolves AAAA records automatically once the stack is dual-stack. -- **Cost**: none to budget. IPv6 is already compiled into the ESP32 Arduino lwIP that every - build links; enabling it at runtime only adds a couple of address slots. - ## CLI Commands ### MQTT Slot Commands diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 22de73cf..3b4219f3 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -780,26 +780,7 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf default: status_str = "unknown"; break; } if (status == WL_CONNECTED) { - // reply points at the caller's char[160] command buffer (see main.cpp). - const size_t kReplyBufSize = 160; - sprintf(reply, "> %s, IP: %s", status_str, WiFi.localIP().toString().c_str()); -#ifdef WITH_MQTT_BRIDGE - // Group IPv6 directly after IPv4 when a global/ULA address is assigned. - char v6[46]; - if (MQTTBridge::getGlobalIPv6(v6, sizeof(v6))) { - size_t v6_len = strlen(reply); - if (v6_len < kReplyBufSize) { - snprintf(reply + v6_len, kReplyBufSize - v6_len, ", IPv6: %s", v6); - } - } -#endif - // RSSI right after the IP addresses. - { - size_t rssi_len = strlen(reply); - if (rssi_len < kReplyBufSize) { - snprintf(reply + rssi_len, kReplyBufSize - rssi_len, ", RSSI: %d dBm", WiFi.RSSI()); - } - } + sprintf(reply, "> %s, IP: %s, RSSI: %d dBm", status_str, WiFi.localIP().toString().c_str(), WiFi.RSSI()); #ifdef WITH_MQTT_BRIDGE unsigned long connect_at = MQTTBridge::getWifiConnectedAtMillis(); if (connect_at != 0) { @@ -809,6 +790,9 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf unsigned long h = (uptime_sec % 86400) / 3600; unsigned long m = (uptime_sec % 3600) / 60; unsigned long s = uptime_sec % 60; + // reply points at the caller's char[160] command buffer (see main.cpp); + // compute the actual remaining space instead of assuming 128. + const size_t kReplyBufSize = 160; size_t len = strlen(reply); const size_t reply_remaining = (len < kReplyBufSize) ? (kReplyBufSize - len) : 0; if (d > 0) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index eeae09d2..aa737d98 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -15,7 +15,6 @@ #ifdef ESP_PLATFORM #include -#include #include #include #include @@ -180,10 +179,6 @@ static unsigned long s_wifi_connected_at = 0; static uint8_t s_wifi_disconnect_reason = 0; static unsigned long s_wifi_disconnect_time = 0; -// Most recent global/unique-local IPv6 address (SLAAC), as a string; empty when none. -// Populated from the ARDUINO_EVENT_WIFI_STA_GOT_IP6 handler; cleared on WiFi disconnect. -static char s_global_ipv6[46] = ""; - #ifdef MQTT_MEMORY_DEBUG // #region agent log static void agentLogHeap(const char* location, const char* message, const char* hypothesisId, @@ -206,17 +201,6 @@ unsigned long MQTTBridge::getWifiConnectedAtMillis() { return s_wifi_connected_at; } -bool MQTTBridge::getGlobalIPv6(char* buf, size_t len) { - if (buf == nullptr || len == 0) return false; - if (s_global_ipv6[0] == '\0') { - buf[0] = '\0'; - return false; - } - strncpy(buf, s_global_ipv6, len - 1); - buf[len - 1] = '\0'; - return true; -} - void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPrefs* obs) { if (buf == nullptr || bufsize == 0) return; const char* msgs = (obs && obs->mqtt_status_enabled) ? "on" : "off"; @@ -845,23 +829,11 @@ void MQTTBridge::initializeWiFiInTask() { switch(event) { case ARDUINO_EVENT_WIFI_STA_GOT_IP: MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); - // Kick off IPv6 link-local + RA/SLAAC (additive; IPv4 path unchanged). Idempotent. - WiFi.enableIpV6(); // Set flag to trigger NTP sync from loop() instead of doing it here if (!_ntp_synced && !_ntp_sync_pending) { _ntp_sync_pending = true; } break; - case ARDUINO_EVENT_WIFI_STA_GOT_IP6: { - // Store only global/unique-local addresses; link-local isn't useful for diagnostics. - esp_ip6_addr_t ip6 = info.got_ip6.ip6_info.ip; - esp_ip6_addr_type_t type = esp_netif_ip6_get_addr_type(&ip6); - if (type == ESP_IP6_ADDR_IS_GLOBAL || type == ESP_IP6_ADDR_IS_UNIQUE_LOCAL) { - snprintf(s_global_ipv6, sizeof(s_global_ipv6), IPV6STR, IPV62STR(ip6)); - MQTT_DEBUG_PRINTLN("WiFi IPv6: %s", s_global_ipv6); - } - break; - } case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; s_wifi_disconnect_time = millis(); @@ -1353,16 +1325,7 @@ void MQTTBridge::setupSlot(int index) { } else if (slot.port == 443) { proto = "wss"; } - // Bare IPv6 literals (contain ':' but no '.', not already bracketed) must be wrapped - // in [..] so the ":port" suffix isn't mistaken for part of the address. - bool bare_ipv6 = (slot.host[0] != '[') && - (strchr(slot.host, ':') != nullptr) && - (strchr(slot.host, '.') == nullptr); - if (bare_ipv6) { - snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://[%s]:%d", proto, slot.host, slot.port); - } else { - snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); - } + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); } slot.client->setServer(slot.broker_uri); MQTT_DEBUG_PRINTLN("MQTT%d custom broker URI: %s (host='%s', port=%u)", @@ -2117,9 +2080,6 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { _wifi_disconnected_time = 0; s_wifi_connected_at = now; _wifi_reconnect_backoff_attempt = 0; - // Re-arm IPv6 link-local after a reconnect (covers paths that re-run WiFi.begin - // and skip the GOT_IP event ordering). Idempotent. - WiFi.enableIpV6(); #ifdef ESP_PLATFORM wifi_ps_type_t ps_mode; uint8_t ps_pref = _obs->wifi_power_save; @@ -2146,7 +2106,6 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (_last_wifi_status == WL_CONNECTED) { _wifi_disconnected_time = now; s_wifi_connected_at = 0; - s_global_ipv6[0] = '\0'; // drop stale IPv6 so get wifi.status doesn't report it // Disconnect all slot clients when WiFi drops for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].client && _slots[i].connected) { diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index ee5b8b98..377ea51c 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -430,10 +430,6 @@ public: static unsigned long getWifiConnectedAtMillis(); - // Copies the current global/unique-local IPv6 address (string) into buf. - // Returns false (and writes an empty string) when no global IPv6 is assigned. - static bool getGlobalIPv6(char* buf, size_t len); - /** * Per-slot outage accessors used by AlertReporter to detect prolonged * MQTT broker outages. Indices are 0..RUNTIME_MQTT_SLOTS-1. From b7c145929f8442084eb7410c93981adce2fa0929 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 11 Jul 2026 09:35:40 -0700 Subject: [PATCH 10/84] fix(mqtt): bound esp-mqtt outbox for QoS0 publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QoS0 packet/raw publishes are forced into the esp-mqtt outbox (store=true, async) so packet topics keep flowing, but the outbox has no size bound of its own — esp-mqtt frees entries only on send-ack or ~30s expiry. On a stalled or slow uplink (socket still "connected") QoS0 frames accumulate on internal heap without limit, driving the heap exhaustion/fragmentation seen in the field. Cap the outbox at the application level: PsychicMqttClient::setOutboxLimit() records a per-client byte cap, and publish() drops a QoS0 message (returns -2) when esp_mqtt_client_get_outbox_size() is already at/over the cap, before enqueuing. The bridge's existing processPacketQueue retry/drop path handles the -2 as backpressure. Caps: 16 KiB PSRAM / 8 KiB non-PSRAM (outbox lives on internal heap, so non-PSRAM is the fragmentation-sensitive case). Portable across IDF 4.4 and 5 via esp_mqtt_client_get_outbox_size(); esp-mqtt's own outbox.limit config is not used (its enqueue path does not reliably enforce it for QoS0, and the app-level guard fires before enqueue regardless). Adds getOutboxSize()/getOutboxLimit()/getOutboxDrops() and surfaces per-slot outbox size/cap/drops via a throttled logMemoryStatus() in the MQTT task loop (MQTT_DEBUG-gated) to confirm the bound on-target. --- .../src/PsychicMqttClient.cpp | 50 ++++++++++++++- lib/PsychicMqttClient/src/PsychicMqttClient.h | 42 +++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 63 +++++++++++++++++-- 3 files changed, 146 insertions(+), 9 deletions(-) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index 2ae395c8..69cb35b1 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -529,10 +529,30 @@ int PsychicMqttClient::publish(const char *topic, int qos, bool retain, const ch if (async) { + // QoS0 async publishes are stored in the esp-mqtt outbox (store=true) so that + // packet topics keep flowing — enqueue(store=false) produced false-failure + // semantics that stalled the packet path. The outbox has no size bound of its + // own (esp-mqtt frees entries only on send-ack or time-based expiry), so on a + // stalled uplink QoS0 frames pile up on internal heap without limit. Cap it + // here: if the outbox is already at/over _outbox_limit, drop this QoS0 message + // and report -2 ("outbox full") so the caller can retry/drop. QoS1/2 (durable, + // low-rate) are never gated. + if (qos == 0 && _outbox_limit > 0 && _client != nullptr && + esp_mqtt_client_get_outbox_size(_client) >= _outbox_limit) + { + _outbox_drops++; + static unsigned long last_full_log = 0; + unsigned long now = millis(); + if (now - last_full_log > 5000) + { + ESP_LOGW(TAG, "Outbox at cap (%u bytes); dropping QoS0 message to topic %s", + (unsigned)_outbox_limit, topic); + last_full_log = now; + } + return -2; + } + ESP_LOGV(TAG, "Enqueuing message to topic %s with QoS %d", topic, qos); - // Hotfix: restore legacy outbox behavior for QoS0 async publishes. - // This avoids false-failure semantics from enqueue(store=false) on some - // connected paths where packet topics stop flowing. bool store_in_outbox = true; return esp_mqtt_client_enqueue(_client, topic, payload, length, qos, retain, store_in_outbox); } @@ -557,6 +577,30 @@ esp_mqtt_client_config_t *PsychicMqttClient::getMqttConfig() return &_mqtt_cfg; } +PsychicMqttClient &PsychicMqttClient::setOutboxLimit(size_t bytes) +{ + _outbox_limit = bytes; + return *this; +} + +size_t PsychicMqttClient::getOutboxSize() +{ + if (_client == nullptr) + return 0; + int size = esp_mqtt_client_get_outbox_size(_client); + return size > 0 ? (size_t)size : 0; +} + +size_t PsychicMqttClient::getOutboxLimit() +{ + return _outbox_limit; +} + +unsigned long PsychicMqttClient::getOutboxDrops() +{ + return _outbox_drops; +} + void PsychicMqttClient::_onMqttEventStatic(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data) { // Since this is a static function, we need to cast the first argument (void*) back to the class instance type diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index c45cf272..a9e273ba 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -410,6 +410,42 @@ public: */ esp_mqtt_client_config_t *getMqttConfig(); + /** + * @brief Caps the size of the esp-mqtt outbox for async QoS 0 publishes. + * + * QoS 0 async publishes are forced into the esp-mqtt outbox (store=true) so + * packet topics keep flowing, but the outbox has no size bound of its own — + * it only frees entries on send-ack or time-based expiry. On a stalled uplink + * the entries accumulate on internal heap without limit. When the current + * outbox size is at/over this cap, publish() drops the new QoS 0 message + * (returns -2) instead of enqueuing it, applying backpressure. 0 disables the + * cap. QoS 1/2 publishes are never gated by this. + * + * @param bytes Maximum outbox size in bytes, or 0 to disable. + * @return A reference to the PsychicMqttClient instance. + */ + PsychicMqttClient &setOutboxLimit(size_t bytes); + + /** + * @brief Returns the current esp-mqtt outbox size in bytes (0 if the client + * is not initialized). Useful for diagnostics/backpressure monitoring. + * + * @return Current outbox size in bytes. + */ + size_t getOutboxSize(); + + /** + * @brief Returns the configured outbox cap in bytes (0 = disabled). Lets + * callers confirm the cap is actually applied to this client. + */ + size_t getOutboxLimit(); + + /** + * @brief Returns the cumulative count of QoS0 messages dropped because the + * outbox was at/over the cap. Monotonic; useful for backpressure diagnostics. + */ + unsigned long getOutboxDrops(); + private: esp_mqtt_client_handle_t _client = nullptr; esp_mqtt_client_config_t _mqtt_cfg; @@ -418,6 +454,12 @@ private: bool _stopMqttClient = false; bool _config_dirty = true; + // Runtime cap on the esp-mqtt outbox for QoS 0 async publishes (bytes). + // 0 = disabled. Enforced in publish(); not an esp-mqtt config field. + size_t _outbox_limit = 0; + // Cumulative count of QoS0 publishes dropped because the outbox hit the cap. + unsigned long _outbox_drops = 0; + // Multipart message reassembly. _buffer is lazily allocated at connect() time // to match the configured buffer size, then reused for the client's lifetime. // _topic is inline storage, never heap-allocated. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index aa737d98..1e9b218c 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -898,6 +898,16 @@ void MQTTBridge::mqttTaskLoop() { #endif unsigned long now = millis(); + + // Periodic heap + outbox snapshot (compiles to nothing unless MQTT_DEBUG is set). + // Outbox= is the value to watch: it should sit near 0 on a healthy uplink and + // plateau at the configured cap (not climb) during a stall. + static unsigned long last_mem_log = 0; + if (now - last_mem_log >= 30000) { + last_mem_log = now; + logMemoryStatus(); + } + bool wifi_just_connected = handleWiFiConnection(now); if (wifi_just_connected) { // WiFi recovered — reset last_reconnect_attempt for disconnected slots so they @@ -1742,13 +1752,16 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload return false; } - // QoS 0 for the high-rate packet/raw publish paths: no PUBACK, no outbox store, - // no per-message heap alloc — critical for non-PSRAM fragmentation. QoS 1 is used - // only for low-rate retained status messages where delivery matters. + // QoS 0 for the high-rate packet/raw publish paths: no PUBACK, so delivery is + // best-effort. These still enter the esp-mqtt outbox (store=true, needed so packet + // topics keep flowing), but the client bounds that outbox via setOutboxLimit() to + // stop QoS0 frames accumulating on internal heap during a stalled uplink — over the + // cap, publish() returns -2 and the queue retry/drop path below handles it. QoS 1 is + // used only for low-rate retained status messages where delivery matters. // // esp_mqtt_client_enqueue return convention: QoS 0 returns msg_id == 0 on success // (no tracking since there's no PUBACK); QoS 1/2 return a positive msg_id. Negative - // values (-1 generic failure, -2 outbox full) are the only actual failures. + // values (-1 generic failure, -2 outbox full / over cap) are the only actual failures. int result = slot.client->publish(topic, qos, retained, payload, strlen(payload), true); if (result < 0) { // QoS0 packet/raw publishes are best-effort and may be retried from the @@ -3408,6 +3421,22 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ client->setBufferSize(MQTT_CLIENT_BUFFER_SIZE); + // Bound the esp-mqtt outbox for high-rate QoS0 packet/raw publishes. Those go in + // with store=true (so packet topics keep flowing) but the outbox has no size limit + // of its own — it frees entries only on send-ack or ~30s expiry, so a stalled uplink + // lets QoS0 frames accumulate on internal heap without bound. Above this cap the + // client drops the new QoS0 message (returns -2), and processPacketQueue's existing + // retry/drop path applies backpressure. Non-PSRAM is the fragmentation-sensitive case + // (outbox lives on internal heap), so it gets the tighter cap. Under healthy operation + // the outbox drains to ~0 in ms, so this only bites during a stall. Note: esp-mqtt's + // own outbox.limit config is not used — its enqueue path does not reliably enforce it + // for QoS0, and this app-level guard fires before enqueue regardless of IDF version. +#if defined(BOARD_HAS_PSRAM) + client->setOutboxLimit(16384); +#else + client->setOutboxLimit(8192); +#endif + // Access ESP-IDF config to optimize additional settings esp_mqtt_client_config_t* config = client->getMqttConfig(); if (config) { @@ -3425,8 +3454,30 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ } void MQTTBridge::logMemoryStatus() { - MQTT_DEBUG_PRINTLN("Memory: Free=%d, Max=%d, Queue=%d/%d", - ESP.getFreeHeap(), ESP.getMaxAllocHeap(), _queue_count, MAX_QUEUE_SIZE); + // The esp-mqtt outbox is the structure that grew unbounded before the cap. The cap + // (setOutboxLimit) is enforced PER CLIENT, so log it per slot — a summed total hides + // whether any single slot has blown past its cap. Each entry is size/limit; drops is + // the cumulative count of QoS0 messages rejected at the cap (backpressure firing). + // Under a healthy uplink size sits near 0; under a stall/saturation it should plateau + // at limit (not climb) and drops should increase. + char outbox_detail[160]; + size_t pos = 0; + size_t outbox_total = 0; + outbox_detail[0] = '\0'; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].client) { + size_t sz = _slots[i].client->getOutboxSize(); + outbox_total += sz; + pos += snprintf(outbox_detail + pos, sizeof(outbox_detail) - pos, "%ss%d=%u/%u(d%lu)", + pos ? " " : "", i, (unsigned)sz, + (unsigned)_slots[i].client->getOutboxLimit(), + _slots[i].client->getOutboxDrops()); + if (pos >= sizeof(outbox_detail)) break; + } + } + MQTT_DEBUG_PRINTLN("Memory: Free=%d, Max=%d, Queue=%d/%d, Outbox=%u [%s]", + ESP.getFreeHeap(), ESP.getMaxAllocHeap(), _queue_count, MAX_QUEUE_SIZE, + (unsigned)outbox_total, outbox_detail); } // --------------------------------------------------------------------------- From 53c39dc282f1a40f5325ddf5a205f1cc9499542b Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 11 Jul 2026 09:36:35 -0700 Subject: [PATCH 11/84] fix(mqtt): publish QoS0 synchronously to bypass ~1 msg/s outbox drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The esp-mqtt task drains only one QUEUED outbox item per loop iteration, and each iteration blocks up to MQTT_POLL_READ_TIMEOUT_MS (1s) on esp_transport_poll_read. With little inbound traffic that caps throughput at ~1 message/second per connection, so even a light packet rate (~1.2/s) outruns the drain: the outbox pins at its cap and ~20-30% of QoS0 packets are dropped as backpressure. The poll timeout is a compile-time constant baked into the precompiled esp-mqtt lib, so the async drain rate cannot be raised on the Arduino/IDF 4.4 toolchain. Route QoS0 packet publishes through esp_mqtt_client_publish() (async=false) so they write straight to the socket, bypassing the outbox drain entirely — QoS0 no longer touches the outbox. QoS1 status keeps the async/outbox + retransmit path. The esp-mqtt task releases its API lock before the poll, so a synchronous publish from the (Core-0, prio-1) MQTT task acquires the lock and writes immediately; a stalled socket blocks only that task (mesh RX on Core 1 and the WiFi/TCP stack are unaffected), bounded by a new setNetworkTimeout() lowered to 2500ms so a first stall fails fast and flips the slot to disconnected. The outbox cap from the previous commit stays as a dormant safety net. Retools the MQTT_DEBUG diagnostic from outbox size/drops (now always ~0) to per-slot publish ok/err counts, the live signal for delivery health, with 1-based slot numbering to match the status line. --- .../src/PsychicMqttClient.cpp | 32 +++++++- lib/PsychicMqttClient/src/PsychicMqttClient.h | 28 +++++++ src/helpers/bridges/MQTTBridge.cpp | 82 ++++++++++--------- 3 files changed, 103 insertions(+), 39 deletions(-) diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index 69cb35b1..abe49c88 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -48,6 +48,17 @@ PsychicMqttClient &PsychicMqttClient::setKeepAlive(int keepAlive) return *this; } +PsychicMqttClient &PsychicMqttClient::setNetworkTimeout(int timeoutMs) +{ +#if ESP_IDF_VERSION_MAJOR == 5 + _mqtt_cfg.network.timeout_ms = timeoutMs; +#else + _mqtt_cfg.network_timeout_ms = timeoutMs; +#endif + _config_dirty = true; + return *this; +} + PsychicMqttClient &PsychicMqttClient::setAutoReconnect(bool reconnect) { #if ESP_IDF_VERSION_MAJOR == 5 @@ -554,12 +565,19 @@ int PsychicMqttClient::publish(const char *topic, int qos, bool retain, const ch ESP_LOGV(TAG, "Enqueuing message to topic %s with QoS %d", topic, qos); bool store_in_outbox = true; - return esp_mqtt_client_enqueue(_client, topic, payload, length, qos, retain, store_in_outbox); + int result = esp_mqtt_client_enqueue(_client, topic, payload, length, qos, retain, store_in_outbox); + if (result < 0) _publish_err++; else _publish_ok++; + return result; } else { ESP_LOGV(TAG, "Publishing message to topic %s with QoS %d", topic, qos); - return esp_mqtt_client_publish(_client, topic, payload, length, qos, retain); + // Synchronous write (used for QoS0 packet publishes). A negative result is a real + // send failure (socket error / network_timeout on a stalled link), tracked here so + // callers can surface delivery health without per-message logging. + int result = esp_mqtt_client_publish(_client, topic, payload, length, qos, retain); + if (result < 0) _publish_err++; else _publish_ok++; + return result; } } @@ -601,6 +619,16 @@ unsigned long PsychicMqttClient::getOutboxDrops() return _outbox_drops; } +unsigned long PsychicMqttClient::getPublishOk() +{ + return _publish_ok; +} + +unsigned long PsychicMqttClient::getPublishErr() +{ + return _publish_err; +} + void PsychicMqttClient::_onMqttEventStatic(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data) { // Since this is a static function, we need to cast the first argument (void*) back to the class instance type diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index a9e273ba..42cf8d5c 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -126,6 +126,18 @@ public: */ PsychicMqttClient &setAutoReconnect(bool reconnect = true); + /** + * @brief Sets the network operation timeout in milliseconds. esp-mqtt aborts a + * network read/write (including a synchronous publish's socket write) if it does + * not complete within this window. A lower value bounds how long a synchronous + * QoS0 publish can block on a stalled/half-open socket before failing and letting + * the slot flip to disconnected. esp-mqtt's default is 10000. + * + * @param timeoutMs Network timeout in milliseconds. + * @return A reference to the PsychicMqttClient instance. + */ + PsychicMqttClient &setNetworkTimeout(int timeoutMs); + /** * @brief Sets the retransmit timeout for unacknowledged QoS 1/2 messages. * esp-mqtt resends an unacked PUBLISH (DUP flag set) every time this @@ -446,6 +458,19 @@ public: */ unsigned long getOutboxDrops(); + /** + * @brief Cumulative count of publishes that were accepted (enqueued or written + * successfully). Monotonic. Pair with getPublishErr() for delivery-health stats. + */ + unsigned long getPublishOk(); + + /** + * @brief Cumulative count of publishes that failed (negative return from the + * synchronous write or async enqueue — socket error / network timeout). Monotonic. + * A rising value indicates the uplink is dropping publishes. + */ + unsigned long getPublishErr(); + private: esp_mqtt_client_handle_t _client = nullptr; esp_mqtt_client_config_t _mqtt_cfg; @@ -459,6 +484,9 @@ private: size_t _outbox_limit = 0; // Cumulative count of QoS0 publishes dropped because the outbox hit the cap. unsigned long _outbox_drops = 0; + // Cumulative publish accept/fail counts (any QoS, sync or async path). + unsigned long _publish_ok = 0; + unsigned long _publish_err = 0; // Multipart message reassembly. _buffer is lazily allocated at connect() time // to match the configured buffer size, then reused for the client's lifetime. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1e9b218c..1682c1de 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1752,17 +1752,23 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload return false; } - // QoS 0 for the high-rate packet/raw publish paths: no PUBACK, so delivery is - // best-effort. These still enter the esp-mqtt outbox (store=true, needed so packet - // topics keep flowing), but the client bounds that outbox via setOutboxLimit() to - // stop QoS0 frames accumulating on internal heap during a stalled uplink — over the - // cap, publish() returns -2 and the queue retry/drop path below handles it. QoS 1 is - // used only for low-rate retained status messages where delivery matters. + // Publish path by QoS: + // - QoS 0 (high-rate packets/raw): SYNCHRONOUS (async=false → esp_mqtt_client_publish), + // which writes straight to the socket. The async/outbox path drains only one queued + // item per esp-mqtt task loop (~1 msg/s/conn, gated by the 1s poll_read), so under + // even light packet load the outbox pins at its cap and drops ~20-30%. A synchronous + // write bypasses that drain ceiling entirely and does not store in the outbox. It can + // block the (Core-0, prio-1) MQTT task on a stalled socket, but only up to + // network_timeout_ms (lowered in optimizeMqttClientConfig); mesh RX (Core 1) and the + // WiFi/TCP stack (higher-prio system tasks) are unaffected, and a failed write flips + // the slot to disconnected so subsequent packets skip it. + // - QoS 1 (low-rate retained status): async, so it keeps the durable outbox + retransmit. // - // esp_mqtt_client_enqueue return convention: QoS 0 returns msg_id == 0 on success - // (no tracking since there's no PUBACK); QoS 1/2 return a positive msg_id. Negative - // values (-1 generic failure, -2 outbox full / over cap) are the only actual failures. - int result = slot.client->publish(topic, qos, retained, payload, strlen(payload), true); + // Return convention: QoS 0 sync publish returns msg_id == 0 on success (no PUBACK + // tracking). Negative values (-1 write/failure) are the only actual failures; the queue + // retry/drop path below handles them. + bool async = (qos > 0); + int result = slot.client->publish(topic, qos, retained, payload, strlen(payload), async); if (result < 0) { // QoS0 packet/raw publishes are best-effort and may be retried from the // bridge queue; avoid logging transient first-attempt failures here. @@ -3421,16 +3427,17 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ client->setBufferSize(MQTT_CLIENT_BUFFER_SIZE); - // Bound the esp-mqtt outbox for high-rate QoS0 packet/raw publishes. Those go in - // with store=true (so packet topics keep flowing) but the outbox has no size limit - // of its own — it frees entries only on send-ack or ~30s expiry, so a stalled uplink - // lets QoS0 frames accumulate on internal heap without bound. Above this cap the - // client drops the new QoS0 message (returns -2), and processPacketQueue's existing - // retry/drop path applies backpressure. Non-PSRAM is the fragmentation-sensitive case - // (outbox lives on internal heap), so it gets the tighter cap. Under healthy operation - // the outbox drains to ~0 in ms, so this only bites during a stall. Note: esp-mqtt's - // own outbox.limit config is not used — its enqueue path does not reliably enforce it - // for QoS0, and this app-level guard fires before enqueue regardless of IDF version. + // Bound how long a synchronous QoS0 publish (see publishToSlot) can block the MQTT + // task on a stalled/half-open socket before esp-mqtt aborts the write. Default is 10s; + // 2.5s lets a first stall resolve fast (write fails → slot flips to disconnected → + // subsequent packets skip it) without holding up publishing to the other slots. Mesh + // RX (Core 1) and the WiFi/TCP stack are unaffected by this block regardless. + client->setNetworkTimeout(2500); + + // Dormant safety net: cap the esp-mqtt outbox for any residual async QoS0 path. QoS0 + // packets now publish synchronously (store=false, no outbox), so this normally never + // engages, but it bounds internal-heap growth if a QoS0 message ever takes the async + // path. Non-PSRAM (outbox on internal heap) gets the tighter cap. #if defined(BOARD_HAS_PSRAM) client->setOutboxLimit(16384); #else @@ -3454,30 +3461,31 @@ void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_ } void MQTTBridge::logMemoryStatus() { - // The esp-mqtt outbox is the structure that grew unbounded before the cap. The cap - // (setOutboxLimit) is enforced PER CLIENT, so log it per slot — a summed total hides - // whether any single slot has blown past its cap. Each entry is size/limit; drops is - // the cumulative count of QoS0 messages rejected at the cap (backpressure firing). - // Under a healthy uplink size sits near 0; under a stall/saturation it should plateau - // at limit (not climb) and drops should increase. - char outbox_detail[160]; + // QoS0 packets now publish synchronously, so the outbox stays ~0 and is only a sanity + // check (a non-zero total would mean the QoS1 status path is backing up or the dormant + // async cap engaged). The live signal is per-slot publish health: ok = cumulative + // accepted writes, err = cumulative failures (socket error / network_timeout on a + // stalled link). A rising err on a slot means that broker's uplink is dropping packets; + // ok climbing with err flat is healthy delivery. + char pub_detail[200]; size_t pos = 0; size_t outbox_total = 0; - outbox_detail[0] = '\0'; + pub_detail[0] = '\0'; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (_slots[i].client) { - size_t sz = _slots[i].client->getOutboxSize(); - outbox_total += sz; - pos += snprintf(outbox_detail + pos, sizeof(outbox_detail) - pos, "%ss%d=%u/%u(d%lu)", - pos ? " " : "", i, (unsigned)sz, - (unsigned)_slots[i].client->getOutboxLimit(), - _slots[i].client->getOutboxDrops()); - if (pos >= sizeof(outbox_detail)) break; + outbox_total += _slots[i].client->getOutboxSize(); + if (_slots[i].enabled) { + pos += snprintf(pub_detail + pos, sizeof(pub_detail) - pos, "%ss%d=%lu/%lu", + pos ? " " : "", i + 1, + _slots[i].client->getPublishOk(), + _slots[i].client->getPublishErr()); + if (pos >= sizeof(pub_detail)) break; + } } } - MQTT_DEBUG_PRINTLN("Memory: Free=%d, Max=%d, Queue=%d/%d, Outbox=%u [%s]", + MQTT_DEBUG_PRINTLN("Memory: Free=%d, Max=%d, Queue=%d/%d, Outbox=%u | pub(ok/err) %s", ESP.getFreeHeap(), ESP.getMaxAllocHeap(), _queue_count, MAX_QUEUE_SIZE, - (unsigned)outbox_total, outbox_detail); + (unsigned)outbox_total, pub_detail); } // --------------------------------------------------------------------------- From 1eaa680e26d2de4002d3d51cc162d4df5b997834 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 11 Jul 2026 12:14:46 -0700 Subject: [PATCH 12/84] chore(mqtt): silence periodic stats log for production; add get mqtt.stats CLI The 30s "MQTT: Memory" line was useful during the outbox/sync-publish investigation but is spam for production. Gate the periodic logMemoryStatus() call in the MQTT task loop behind MQTT_MEMORY_DEBUG (a dedicated diagnostics flag, not enabled by plain MQTT_DEBUG or production builds) and stop the heltec_v3 variant from force-enabling MQTT_MEMORY_DEBUG on its observer_mqtt envs (now commented out to match heltec_v4). logMemoryStatus() itself is kept intact for opt-in debugging. Expose the same data on demand via a new `get mqtt.stats` CLI command backed by MQTTBridge::formatMqttStatsReply(): free/max heap, queue depth, outbox total, and per-slot publish ok/err counts (1-based, matching the msgs: line). Fits the 160-byte reply buffer at 6 slots; returns "(bridge not running)" when down. --- src/helpers/CommonCLI_Observer.cpp | 2 ++ src/helpers/bridges/MQTTBridge.cpp | 48 ++++++++++++++++++++++++++++-- src/helpers/bridges/MQTTBridge.h | 3 ++ variants/heltec_v3/platformio.ini | 6 ++-- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 3b4219f3..695aacef 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -695,6 +695,8 @@ bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* conf start = (int)_atoi(start_arg); } formatMQTTPresetListReply(reply, 160, start); + } else if (memcmp(config, "mqtt.stats", 10) == 0) { + MQTTBridge::formatMqttStatsReply(reply, 160); } else if (memcmp(config, "mqtt.status", 11) == 0) { MQTTBridge::formatMqttStatusReply(reply, 160, &_mqtt_prefs); } else if (memcmp(config, "mqtt.packets", 12) == 0) { diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 1682c1de..15104a65 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -250,6 +250,45 @@ void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPref snprintf(buf + pos, bufsize - pos, ", q:%d", q); } +// On-demand publish-health + heap snapshot for the `get mqtt.stats` CLI command. +// Same data as the (MQTT_MEMORY_DEBUG-only) periodic logMemoryStatus() line, but +// returned as a reply instead of logged. Per-slot "sN=ok/err": ok = cumulative +// accepted publishes, err = cumulative failures (socket error / network timeout). +// Outbox should read ~0 (QoS0 publishes synchronously); a rising err isolates a +// broker whose uplink is dropping writes. +void MQTTBridge::formatMqttStatsReply(char* buf, size_t bufsize) { + if (buf == nullptr || bufsize == 0) return; + if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) { + snprintf(buf, bufsize, "> (bridge not running)"); + return; + } + MQTTBridge* b = s_mqtt_bridge_instance; + + int q = 0; +#ifdef ESP_PLATFORM + if (b->_packet_queue_handle != nullptr) { + q = (int)uxQueueMessagesWaiting(b->_packet_queue_handle); + } +#else + q = b->_queue_count; +#endif + + size_t outbox_total = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (b->_slots[i].client) outbox_total += b->_slots[i].client->getOutboxSize(); + } + + int pos = snprintf(buf, bufsize, "> Free=%d Max=%d q:%d/%d Outbox=%u |", + (int)ESP.getFreeHeap(), (int)ESP.getMaxAllocHeap(), + q, MAX_QUEUE_SIZE, (unsigned)outbox_total); + for (int i = 0; i < RUNTIME_MQTT_SLOTS && pos < (int)bufsize - 1; i++) { + if (!b->_slots[i].enabled || !b->_slots[i].client) continue; + pos += snprintf(buf + pos, bufsize - pos, " s%d=%lu/%lu", i + 1, + b->_slots[i].client->getPublishOk(), + b->_slots[i].client->getPublishErr()); + } +} + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } @@ -899,14 +938,17 @@ void MQTTBridge::mqttTaskLoop() { unsigned long now = millis(); - // Periodic heap + outbox snapshot (compiles to nothing unless MQTT_DEBUG is set). - // Outbox= is the value to watch: it should sit near 0 on a healthy uplink and - // plateau at the configured cap (not climb) during a stall. + // Periodic heap + publish-health snapshot. Gated behind MQTT_MEMORY_DEBUG (a + // dedicated diagnostics flag, NOT enabled on production or plain MQTT_DEBUG builds) + // so it stays off by default — the same data is available on demand via the + // `get mqtt.stats` CLI command (formatMqttStatsReply / logMemoryStatus()). + #ifdef MQTT_MEMORY_DEBUG static unsigned long last_mem_log = 0; if (now - last_mem_log >= 30000) { last_mem_log = now; logMemoryStatus(); } + #endif bool wifi_just_connected = handleWiFiConnection(now); if (wifi_just_connected) { diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 377ea51c..69782bd4 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -468,6 +468,9 @@ public: * (for LoRa). Returns false if the bridge is not running. */ bool ntpDiag(char* reply, size_t reply_size, bool verbose); static void formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPrefs* obs); + /** On-demand publish-health + heap snapshot for `get mqtt.stats` (per-slot ok/err, + * outbox size, free/max heap, queue depth). */ + static void formatMqttStatsReply(char* buf, size_t bufsize); /** True when WiFi is set and at least one MQTT slot can run (preset + custom host if needed). */ static bool isConfigValid(const MQTTPrefs* obs); static void formatSlotDiagReply(char* buf, size_t bufsize, int slot_index); diff --git a/variants/heltec_v3/platformio.ini b/variants/heltec_v3/platformio.ini index 4bd3819a..2d13237b 100644 --- a/variants/heltec_v3/platformio.ini +++ b/variants/heltec_v3/platformio.ini @@ -127,7 +127,8 @@ build_flags = -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 - -D MQTT_MEMORY_DEBUG=1 +; Periodic 30s heap/pub-stats serial log — enable only for debugging (use `get mqtt.stats` on demand instead). +; -D MQTT_MEMORY_DEBUG=1 ; Keep default observer profile less verbose to reduce runtime contention. ; -D MESH_PACKET_LOGGING=1 ; -D MESH_DEBUG=1 @@ -486,7 +487,8 @@ build_flags = -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 - -D MQTT_MEMORY_DEBUG=1 +; Periodic 30s heap/pub-stats serial log — enable only for debugging (use `get mqtt.stats` on demand instead). +; -D MQTT_MEMORY_DEBUG=1 ; Keep default observer profile less verbose to reduce runtime contention. ; -D MESH_PACKET_LOGGING=1 ; -D MESH_DEBUG=1 From d7a7e1b6428a6781ba2c03f0273cf1e2814f709f Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 11 Jul 2026 21:37:44 -0700 Subject: [PATCH 13/84] feat(boards): add Heltec tracker MQTT observer builds Credit: @yellowcooln (PR #12). Adds Wireless Tracker v1.1 board def and MQTT observer envs for v1.1/v2; gate FEM control so v1.1 builds without KCT8103L pins. --- MQTT_IMPLEMENTATION.md | 6 + boards/heltec_tracker_v1_1.json | 40 ++++ variants/heltec_tracker_v2/LoRaFEMControl.cpp | 10 + variants/heltec_tracker_v2/platformio.ini | 194 ++++++++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 boards/heltec_tracker_v1_1.json diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 5ba66247..5bc75004 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -145,6 +145,12 @@ pio run -e Heltec_v3_repeater_observer_mqtt # Heltec V4 pio run -e heltec_v4_repeater_observer_mqtt +# Heltec Wireless Tracker v1.1 / v2 +pio run -e heltec_tracker_v1_1_repeater_observer_mqtt +pio run -e heltec_tracker_v1_1_room_server_observer_mqtt +pio run -e heltec_tracker_v2_repeater_observer_mqtt +pio run -e heltec_tracker_v2_room_server_observer_mqtt + # Station G2 pio run -e Station_G2_repeater_observer_mqtt diff --git a/boards/heltec_tracker_v1_1.json b/boards/heltec_tracker_v1_1.json new file mode 100644 index 00000000..a9ec5851 --- /dev/null +++ b/boards/heltec_tracker_v1_1.json @@ -0,0 +1,40 @@ +{ + "build": { + "arduino": { + "ldscript": "esp32s3_out.ld", + "partitions": "default_8MB.csv" + }, + "core": "esp32", + "extra_flags": [ + "-DARDUINO_USB_CDC_ON_BOOT=1", + "-DARDUINO_USB_MODE=0", + "-DARDUINO_RUNNING_CORE=1", + "-DARDUINO_EVENT_RUNNING_CORE=1" + ], + "f_cpu": "240000000L", + "f_flash": "80000000L", + "flash_mode": "qio", + "hwids": [["0x303A", "0x1001"]], + "mcu": "esp32s3", + "variant": "heltec_tracker_v2" + }, + "connectivity": ["wifi", "bluetooth", "lora"], + "debug": { + "default_tool": "esp-builtin", + "onboard_tools": ["esp-builtin"], + "openocd_target": "esp32s3.cfg" + }, + "frameworks": ["arduino", "espidf"], + "name": "Heltec Wireless Tracker v1.1", + "upload": { + "flash_size": "8MB", + "maximum_ram_size": 327680, + "maximum_size": 8388608, + "use_1200bps_touch": true, + "wait_for_upload_port": true, + "require_upload_port": true, + "speed": 921600 + }, + "url": "https://heltec.org/project/wireless-tracker/", + "vendor": "Heltec" +} diff --git a/variants/heltec_tracker_v2/LoRaFEMControl.cpp b/variants/heltec_tracker_v2/LoRaFEMControl.cpp index b846465d..83f6ff03 100644 --- a/variants/heltec_tracker_v2/LoRaFEMControl.cpp +++ b/variants/heltec_tracker_v2/LoRaFEMControl.cpp @@ -5,6 +5,7 @@ void LoRaFEMControl::init(void) { +#if defined(P_LORA_PA_POWER) && defined(P_LORA_KCT8103L_PA_CSD) && defined(P_LORA_KCT8103L_PA_CTX) pinMode(P_LORA_PA_POWER, OUTPUT); digitalWrite(P_LORA_PA_POWER, HIGH); rtc_gpio_hold_dis((gpio_num_t)P_LORA_PA_POWER); @@ -16,32 +17,40 @@ void LoRaFEMControl::init(void) pinMode(P_LORA_KCT8103L_PA_CTX, OUTPUT); digitalWrite(P_LORA_KCT8103L_PA_CTX, lna_enabled ? LOW : HIGH); setLnaCanControl(true); +#endif } void LoRaFEMControl::setSleepModeEnable(void) { +#if defined(P_LORA_KCT8103L_PA_CSD) // shutdown the PA digitalWrite(P_LORA_KCT8103L_PA_CSD, LOW); +#endif } void LoRaFEMControl::setTxModeEnable(void) { +#if defined(P_LORA_KCT8103L_PA_CSD) && defined(P_LORA_KCT8103L_PA_CTX) digitalWrite(P_LORA_KCT8103L_PA_CSD, HIGH); digitalWrite(P_LORA_KCT8103L_PA_CTX, HIGH); +#endif } void LoRaFEMControl::setRxModeEnable(void) { +#if defined(P_LORA_KCT8103L_PA_CSD) && defined(P_LORA_KCT8103L_PA_CTX) digitalWrite(P_LORA_KCT8103L_PA_CSD, HIGH); if (lna_enabled) { digitalWrite(P_LORA_KCT8103L_PA_CTX, LOW); } else { digitalWrite(P_LORA_KCT8103L_PA_CTX, HIGH); } +#endif } void LoRaFEMControl::setRxModeEnableWhenMCUSleep(void) { +#if defined(P_LORA_KCT8103L_PA_CSD) && defined(P_LORA_KCT8103L_PA_CTX) digitalWrite(P_LORA_KCT8103L_PA_CSD, HIGH); rtc_gpio_hold_en((gpio_num_t)P_LORA_KCT8103L_PA_CSD); if (lna_enabled) { @@ -50,6 +59,7 @@ void LoRaFEMControl::setRxModeEnableWhenMCUSleep(void) digitalWrite(P_LORA_KCT8103L_PA_CTX, HIGH); } rtc_gpio_hold_en((gpio_num_t)P_LORA_KCT8103L_PA_CTX); +#endif } void LoRaFEMControl::setLNAEnable(bool enabled) diff --git a/variants/heltec_tracker_v2/platformio.ini b/variants/heltec_tracker_v2/platformio.ini index 688b1c7d..512a4543 100644 --- a/variants/heltec_tracker_v2/platformio.ini +++ b/variants/heltec_tracker_v2/platformio.ini @@ -58,6 +58,56 @@ lib_deps = ${sensor_base.lib_deps} adafruit/Adafruit ST7735 and ST7789 Library @ ^1.11.0 +[Heltec_tracker_v1_1] +extends = Heltec_tracker_v2 +board = heltec_tracker_v1_1 +build_flags = + ${esp32_base.build_flags} + ${sensor_base.build_flags} + -I variants/heltec_tracker_v2 + -D HELTEC_TRACKER_V1_1 + -D ESP32_CPU_FREQ=240 + -D USE_SX1262 + -D RADIO_CLASS=CustomSX1262 + -D WRAPPER_CLASS=CustomSX1262Wrapper + -D P_LORA_TX_LED=18 + -D P_LORA_DIO_1=14 + -D P_LORA_NSS=8 + -D P_LORA_RESET=12 + -D P_LORA_BUSY=13 + -D P_LORA_SCLK=9 + -D P_LORA_MISO=11 + -D P_LORA_MOSI=10 + -D LORA_TX_POWER=22 + -D SX126X_DIO2_AS_RF_SWITCH=true + -D SX126X_DIO3_TCXO_VOLTAGE=1.8 + -D SX126X_CURRENT_LIMIT=140 + -D SX126X_RX_BOOSTED_GAIN=1 + -D SX126X_REGISTER_PATCH=1 + -D PIN_BOARD_SDA=6 + -D PIN_BOARD_SCL=17 + -D PIN_USER_BTN=0 + -D PIN_TFT_SDA=42 ; SDIN + -D PIN_TFT_SCL=41 ; SCLK + -D PIN_TFT_DC=40 ; RS (register select) + -D PIN_TFT_RST=39 ; RES + -D PIN_TFT_CS=38 + -D USE_PIN_TFT=1 + -D PIN_VEXT_EN=3 ; Vext is connected to VDD which is also connected to OLED & GPS + -D PIN_VEXT_EN_ACTIVE=HIGH + -D PIN_TFT_LEDA_CTL=21 ; LEDK (switches on/off via mosfet to create the ground) + -D DISPLAY_ROTATION=1 + -D PIN_GPS_RX=34 + -D PIN_GPS_TX=33 + -D PIN_GPS_RESET=35 + -D PIN_GPS_RESET_ACTIVE=LOW + -D GPS_BAUD_RATE=115200 + -D ENV_INCLUDE_GPS=1 + -D PIN_ADC_CTRL=2 + -D PIN_VBAT_READ=1 +build_src_filter = ${Heltec_tracker_v2.build_src_filter} +lib_deps = ${Heltec_tracker_v2.lib_deps} + [env:heltec_tracker_v2_repeater] extends = Heltec_tracker_v2 build_flags = @@ -119,6 +169,150 @@ lib_deps = ${Heltec_tracker_v2.lib_deps} ${esp32_ota.lib_deps} +[env:heltec_tracker_v1_1_repeater_observer_mqtt] +extends = Heltec_tracker_v1_1 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_v1_1.build_flags} + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MAX_MQTT_BROKERS=3 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm +build_src_filter = ${Heltec_tracker_v1_1.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_tracker_v1_1.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + +[env:heltec_tracker_v2_repeater_observer_mqtt] +extends = Heltec_tracker_v2 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_v2.build_flags} + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MAX_MQTT_BROKERS=3 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm +build_src_filter = ${Heltec_tracker_v2.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_tracker_v2.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + +[env:heltec_tracker_v1_1_room_server_observer_mqtt] +extends = Heltec_tracker_v1_1 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_v1_1.build_flags} + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"Heltec Tracker Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D WITH_MQTT_BRIDGE=1 + -D MAX_MQTT_BROKERS=3 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm +build_src_filter = ${Heltec_tracker_v1_1.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_tracker_v1_1.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + +[env:heltec_tracker_v2_room_server_observer_mqtt] +extends = Heltec_tracker_v2 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_v2.build_flags} + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"Heltec Tracker Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D WITH_MQTT_BRIDGE=1 + -D MAX_MQTT_BROKERS=3 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm +build_src_filter = ${Heltec_tracker_v2.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_tracker_v2.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + [env:heltec_tracker_v2_terminal_chat] extends = Heltec_tracker_v2 build_flags = From dfee21a0c9d2ff5c4d4d4a5d10011d3cd1490d4e Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 16 Jul 2026 17:00:22 -0700 Subject: [PATCH 14/84] feat(webconfig): implement web configuration portal for ESP32 --- .gitignore | 1 + examples/simple_repeater/MyMesh.cpp | 120 ++++ examples/simple_repeater/MyMesh.h | 46 +- examples/simple_repeater/UITask.cpp | 47 ++ examples/simple_room_server/MyMesh.cpp | 120 ++++ examples/simple_room_server/MyMesh.h | 46 +- examples/simple_room_server/UITask.cpp | 47 ++ platformio.ini | 5 +- scripts/generate_webconfig_html.py | 79 ++ src/helpers/CommonCLI.h | 12 + src/helpers/CommonCLI_Observer.cpp | 15 + src/helpers/bridges/MQTTBridge.cpp | 31 + src/helpers/bridges/MQTTBridge.h | 11 + src/helpers/esp32/WebConfigServer.cpp | 760 ++++++++++++++++++++ src/helpers/esp32/WebConfigServer.h | 169 +++++ webui/index.html | 954 +++++++++++++++++++++++++ 16 files changed, 2460 insertions(+), 3 deletions(-) create mode 100644 scripts/generate_webconfig_html.py create mode 100644 src/helpers/esp32/WebConfigServer.cpp create mode 100644 src/helpers/esp32/WebConfigServer.h create mode 100644 webui/index.html diff --git a/.gitignore b/.gitignore index 29fa3674..10d7e1b8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ compile_commands.json venv/ # Script-generated cert bundles (see scripts/generate_cert_bundle.py) src/certs/x509_crt_bundle.bin +src/helpers/esp32/WebConfigHtml.h ssl_certs/cacert.pem platformio.local.ini .cursor/* diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 88a44fb5..63fd50a8 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1068,6 +1068,16 @@ void MyMesh::begin(FILESYSTEM *fs) { _alerter.setBridge(bridge); #endif +#if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) + // First-boot setup portal: raised only when no WiFi has ever been configured, + // so an OTA onto a deployed (configured) node can never open an AP. + if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + char wc_reply[160]; + startWebConfig(false, wc_reply); + Serial.println(wc_reply); + } +#endif + radio_driver.setParams(_prefs.freq, _prefs.bw, _prefs.sf, _prefs.cr); radio_driver.setTxPower(_prefs.tx_power_dbm); @@ -1304,6 +1314,106 @@ void MyMesh::clearStats() { ((SimpleMeshTables *)getTables())->resetStats(); } +#ifdef WITH_WEBCONFIG +bool MyMesh::startWebConfig(bool force_ap, char* reply) { + if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" + : "Err: webconfig already running"); + return true; + } + if (!_webconfig) { + _webconfig = new WebConfigServer(&_prefs, _cli.getObserverPrefs(), this, + self_id.pub_key, getFirmwareVer(), getRole(), + _cli.getBoard()->getManufacturerName()); + } + if (force_ap) { + // The setup AP owns WiFi outright; refuse while the bridge holds the STA. + if (bridge && bridge->isRunning()) { + strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); + return true; + } + _webconfig->startSetupMode(reply); + } else if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + _webconfig->startSetupMode(reply); // unconfigured: same portal as first boot + } else { + _webconfig->startLanMode(reply); // reports "WiFi not connected" if down + } + return true; +} + +bool MyMesh::stopWebConfig(char* reply) { + if (!_webconfig || !_webconfig->isRunning()) { + strcpy(reply, "Err: webconfig not running"); + return true; + } + _webconfig->requestStop(); + strcpy(reply, "OK - webconfig stopping"); + return true; +} + +void MyMesh::onConfigBatchEnd() { + _wc_batch_active = false; + if (_wc_restart_pending) { + // A full restart re-applies every slot config; drop the per-slot requests. + _wc_restart_pending = false; + _wc_slot_restart_mask = 0; + restartBridge(); + } else if (_wc_slot_restart_mask) { + uint8_t mask = _wc_slot_restart_mask; + _wc_slot_restart_mask = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (mask & (1u << i)) restartBridgeSlot(i); + } + } +} + +// Stats snapshot for GET /api/stats. Runs on the loop task (from tick()); +// same sources as the REQ_TYPE_GET_STATUS reply and `get mqtt.stats`. +void MyMesh::buildStatsJson(char* buf, size_t buf_size) { + char ip[20] = ""; + int wifi_rssi = 0; + if (WiFi.status() == WL_CONNECTED) { + strncpy(ip, WiFi.localIP().toString().c_str(), sizeof(ip) - 1); + wifi_rssi = WiFi.RSSI(); + } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { + strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); + } + int pos = snprintf(buf, buf_size, + "{\"uptime_s\":%lu,\"batt_mv\":%u," + "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," + "\"noise\":%d,\"rssi\":%d,\"snr\":%.1f," + "\"airtime_s\":%lu,\"rx_airtime_s\":%lu," + "\"recv\":%lu,\"sent\":%lu,\"rx_err\":%lu," + "\"sent_flood\":%lu,\"sent_direct\":%lu,\"recv_flood\":%lu,\"recv_direct\":%lu," + "\"tx_queue\":%d,\"wifi_rssi\":%d,\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", + (unsigned long)(uptime_millis / 1000), (unsigned)board.getBattMilliVolts(), + (unsigned long)ESP.getFreeHeap(), (unsigned long)ESP.getMinFreeHeap(), + (unsigned long)ESP.getMaxAllocHeap(), + (int)_radio->getNoiseFloor(), (int)radio_driver.getLastRSSI(), + radio_driver.getLastSNR(), + (unsigned long)(getTotalAirTime() / 1000), (unsigned long)(getReceiveAirTime() / 1000), + (unsigned long)radio_driver.getPacketsRecv(), (unsigned long)radio_driver.getPacketsSent(), + (unsigned long)radio_driver.getPacketsRecvErrors(), + (unsigned long)getNumSentFlood(), (unsigned long)getNumSentDirect(), + (unsigned long)getNumRecvFlood(), (unsigned long)getNumRecvDirect(), + (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, ip, + bridge ? bridge->getQueueSize() : 0); + if (pos < 0 || pos >= (int)buf_size - 3) return; // truncated; snprintf terminated it + bool first = true; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + MQTTBridge::SlotStatusSnapshot s; + if (!MQTTBridge::getSlotStatusSnapshot(i, &s)) continue; + int n = snprintf(buf + pos, buf_size - pos, + "%s{\"n\":%d,\"name\":\"%s\",\"state\":\"%s\",\"ok\":%lu,\"err\":%lu}", + first ? "" : ",", i + 1, s.name, s.state, s.publish_ok, s.publish_err); + if (n < 0 || n >= (int)(buf_size - pos)) break; + pos += n; + first = false; + } + snprintf(buf + pos, buf_size - pos, "]}"); +} +#endif + void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply) { if (region_load_active) { if (StrHelper::isBlank(command)) { // empty/blank line, signal to terminate 'load' operation @@ -1447,6 +1557,16 @@ void MyMesh::loop() { } #endif +#ifdef WITH_WEBCONFIG + if (_webconfig) { + _webconfig->tick(millis()); + if (!_webconfig->isRunning() && !_webconfig->isStopping()) { + delete _webconfig; // teardown finished (or start failed): reclaim the heap + _webconfig = NULL; + } + } +#endif + // is pending dirty contacts write needed? if (dirty_contacts_expiry && millisHasNowPassed(dirty_contacts_expiry)) { acl.save(_fs); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index d2a45b81..127ad7d1 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -27,6 +27,7 @@ #ifdef WITH_MQTT_BRIDGE #include "helpers/bridges/MQTTBridge.h" #define WITH_BRIDGE +#include "helpers/esp32/WebConfigServer.h" // defines WITH_WEBCONFIG on ESP32 #endif #ifdef WITH_SNMP @@ -88,7 +89,11 @@ struct NeighbourInfo { #define PACKET_LOG_FILE "/packet_log" -class MyMesh : public mesh::Mesh, public CommonCLICallbacks { +class MyMesh : public mesh::Mesh, public CommonCLICallbacks +#ifdef WITH_WEBCONFIG + , public WebConfigServer::Callbacks +#endif +{ FILESYSTEM* _fs; uint32_t last_millis; uint64_t uptime_millis; @@ -135,6 +140,12 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { #ifdef WITH_MQTT_BRIDGE AlertReporter _alerter; #endif +#ifdef WITH_WEBCONFIG + WebConfigServer* _webconfig = NULL; // heap-allocated while running, freed on stop + bool _wc_batch_active = false; // coalesce bridge restarts during a config batch + bool _wc_restart_pending = false; + uint8_t _wc_slot_restart_mask = 0; +#endif void putNeighbour(const mesh::Identity& id, uint32_t timestamp, float snr); uint8_t handleLoginReq(const mesh::Identity& sender, const uint8_t* secret, uint32_t sender_timestamp, const uint8_t* data, bool is_flood); @@ -297,6 +308,12 @@ public: void restartBridge() override { if (!bridge || !bridge->isRunning()) return; +#ifdef WITH_WEBCONFIG + if (_wc_batch_active) { // coalesced: applied once in onConfigBatchEnd() + _wc_restart_pending = true; + return; + } +#endif bridge->end(); // Set device metadata before restarting bridge (same as in begin()) char device_id[65]; @@ -315,6 +332,12 @@ public: void restartBridgeSlot(int slot) override { #ifdef WITH_MQTT_BRIDGE if (!bridge || !bridge->isRunning()) return; +#ifdef WITH_WEBCONFIG + if (_wc_batch_active && slot >= 0 && slot < 8) { + _wc_slot_restart_mask |= (uint8_t)(1u << slot); + return; + } +#endif bridge->setSlotPreset(slot, _cli.getObserverPrefs()->mqtt_slot_preset[slot]); #else (void)slot; @@ -350,6 +373,27 @@ public: } #endif +#ifdef WITH_WEBCONFIG + // CommonCLICallbacks: `start webconfig [ap]` / `stop webconfig` + bool startWebConfig(bool force_ap, char* reply) override; + bool stopWebConfig(char* reply) override; + + // WebConfigServer::Callbacks - all invoked from tick() on the loop task + void execCommand(char* cmd, char* reply) override { + handleCommand(0, cmd, reply); + } + void rebootNow() override { + _cli.getBoard()->reboot(); + } + void onConfigBatchStart() override { + _wc_batch_active = true; + _wc_restart_pending = false; + _wc_slot_restart_mask = 0; + } + void onConfigBatchEnd() override; + void buildStatsJson(char* buf, size_t buf_size) override; +#endif + // To check if there is pending work bool hasPendingWork() const; diff --git a/examples/simple_repeater/UITask.cpp b/examples/simple_repeater/UITask.cpp index 713c5bbb..3efaff67 100644 --- a/examples/simple_repeater/UITask.cpp +++ b/examples/simple_repeater/UITask.cpp @@ -8,6 +8,7 @@ #ifdef WITH_MQTT_BRIDGE #include +#include // defines WITH_WEBCONFIG on ESP32 #endif #define AUTO_OFF_MILLIS 20000 // 20 seconds @@ -77,6 +78,43 @@ void UITask::renderCurrScreen() { _display->setCursor((_display->width() - typeWidth) / 2, 48); _display->print(node_type); } else { // home screen +#ifdef WITH_WEBCONFIG + if (WebConfigServer::isRebootPending()) { + // save confirmed on-device: show ground truth even if the browser + // lost its connection before the confirmation reached it + _display->setTextSize(1); + _display->setColor(DisplayDriver::GREEN); + _display->setCursor(0, 14); + _display->print("Config saved!"); + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 30); + _display->print("Rebooting..."); + return; + } + char wc_ssid[33], wc_ip[16]; + if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + // setup portal active: show join instructions instead of the home screen + _display->setTextSize(1); + _display->setColor(DisplayDriver::GREEN); + _display->setCursor(0, 0); + _display->print("Observer WiFi Setup"); + + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 14); + _display->print("Join WiFi:"); + _display->setColor(DisplayDriver::YELLOW); + _display->setCursor(6, 24); + _display->print(wc_ssid); + + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 40); + _display->print("Then browse to:"); + _display->setColor(DisplayDriver::YELLOW); + _display->setCursor(6, 50); + _display->print(wc_ip); + return; + } +#endif // node name _display->setCursor(0, 0); _display->setTextSize(1); @@ -126,6 +164,15 @@ void UITask::loop() { } #endif +#ifdef WITH_WEBCONFIG + // While the setup portal is up there's no user button to wake the screen + // reliably - keep it on so the join instructions stay visible. + if (WebConfigServer::getSetupInfo(NULL, 0, NULL, 0)) { + if (!_display->isOn()) _display->turnOn(); + _auto_off = millis() + AUTO_OFF_MILLIS; + } +#endif + if (_display->isOn()) { if (millis() >= _next_refresh) { _display->startFrame(); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 81389ab1..b63036fb 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -783,6 +783,16 @@ void MyMesh::begin(FILESYSTEM *fs) { _alerter.begin(&_prefs, _cli.getObserverPrefs(), this, this); _alerter.setBridge(bridge); #endif + +#if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) + // First-boot setup portal: raised only when no WiFi has ever been configured, + // so an OTA onto a deployed (configured) node can never open an AP. + if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + char wc_reply[160]; + startWebConfig(false, wc_reply); + Serial.println(wc_reply); + } +#endif } void MyMesh::sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size) { @@ -949,6 +959,106 @@ void MyMesh::formatPacketStatsReply(char *reply) { getNumRecvFlood(), getNumRecvDirect()); } +#ifdef WITH_WEBCONFIG +bool MyMesh::startWebConfig(bool force_ap, char* reply) { + if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" + : "Err: webconfig already running"); + return true; + } + if (!_webconfig) { + _webconfig = new WebConfigServer(&_prefs, _cli.getObserverPrefs(), this, + self_id.pub_key, getFirmwareVer(), getRole(), + _cli.getBoard()->getManufacturerName()); + } + if (force_ap) { + // The setup AP owns WiFi outright; refuse while the bridge holds the STA. + if (bridge && bridge->isRunning()) { + strcpy(reply, "Err: MQTT bridge is running - 'set bridge off' first"); + return true; + } + _webconfig->startSetupMode(reply); + } else if (_cli.getObserverPrefs()->wifi_ssid[0] == 0) { + _webconfig->startSetupMode(reply); // unconfigured: same portal as first boot + } else { + _webconfig->startLanMode(reply); // reports "WiFi not connected" if down + } + return true; +} + +bool MyMesh::stopWebConfig(char* reply) { + if (!_webconfig || !_webconfig->isRunning()) { + strcpy(reply, "Err: webconfig not running"); + return true; + } + _webconfig->requestStop(); + strcpy(reply, "OK - webconfig stopping"); + return true; +} + +void MyMesh::onConfigBatchEnd() { + _wc_batch_active = false; + if (_wc_restart_pending) { + // A full restart re-applies every slot config; drop the per-slot requests. + _wc_restart_pending = false; + _wc_slot_restart_mask = 0; + restartBridge(); + } else if (_wc_slot_restart_mask) { + uint8_t mask = _wc_slot_restart_mask; + _wc_slot_restart_mask = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (mask & (1u << i)) restartBridgeSlot(i); + } + } +} + +// Stats snapshot for GET /api/stats. Runs on the loop task (from tick()); +// same sources as the stats CLI replies and `get mqtt.stats`. +void MyMesh::buildStatsJson(char* buf, size_t buf_size) { + char ip[20] = ""; + int wifi_rssi = 0; + if (WiFi.status() == WL_CONNECTED) { + strncpy(ip, WiFi.localIP().toString().c_str(), sizeof(ip) - 1); + wifi_rssi = WiFi.RSSI(); + } else if (_webconfig && _webconfig->mode() == WebConfigServer::MODE_SETUP) { + strncpy(ip, WiFi.softAPIP().toString().c_str(), sizeof(ip) - 1); + } + int pos = snprintf(buf, buf_size, + "{\"uptime_s\":%lu,\"batt_mv\":%u," + "\"heap_free\":%lu,\"heap_min\":%lu,\"heap_max_alloc\":%lu," + "\"noise\":%d,\"rssi\":%d,\"snr\":%.1f," + "\"airtime_s\":%lu,\"rx_airtime_s\":%lu," + "\"recv\":%lu,\"sent\":%lu,\"rx_err\":%lu," + "\"sent_flood\":%lu,\"sent_direct\":%lu,\"recv_flood\":%lu,\"recv_direct\":%lu," + "\"tx_queue\":%d,\"wifi_rssi\":%d,\"ip\":\"%s\",\"mqtt_queue\":%d,\"slots\":[", + (unsigned long)(uptime_millis / 1000), (unsigned)board.getBattMilliVolts(), + (unsigned long)ESP.getFreeHeap(), (unsigned long)ESP.getMinFreeHeap(), + (unsigned long)ESP.getMaxAllocHeap(), + (int)_radio->getNoiseFloor(), (int)radio_driver.getLastRSSI(), + radio_driver.getLastSNR(), + (unsigned long)(getTotalAirTime() / 1000), (unsigned long)(getReceiveAirTime() / 1000), + (unsigned long)radio_driver.getPacketsRecv(), (unsigned long)radio_driver.getPacketsSent(), + (unsigned long)radio_driver.getPacketsRecvErrors(), + (unsigned long)getNumSentFlood(), (unsigned long)getNumSentDirect(), + (unsigned long)getNumRecvFlood(), (unsigned long)getNumRecvDirect(), + (int)_mgr->getOutboundCount(0xFFFFFFFF), wifi_rssi, ip, + bridge ? bridge->getQueueSize() : 0); + if (pos < 0 || pos >= (int)buf_size - 3) return; // truncated; snprintf terminated it + bool first = true; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + MQTTBridge::SlotStatusSnapshot s; + if (!MQTTBridge::getSlotStatusSnapshot(i, &s)) continue; + int n = snprintf(buf + pos, buf_size - pos, + "%s{\"n\":%d,\"name\":\"%s\",\"state\":\"%s\",\"ok\":%lu,\"err\":%lu}", + first ? "" : ",", i + 1, s.name, s.state, s.publish_ok, s.publish_err); + if (n < 0 || n >= (int)(buf_size - pos)) break; + pos += n; + first = false; + } + snprintf(buf + pos, buf_size - pos, "]}"); +} +#endif + void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply) { if (region_load_active) { if (StrHelper::isBlank(command)) { // empty/blank line, signal to terminate 'load' operation @@ -1113,6 +1223,16 @@ void MyMesh::loop() { MESH_DEBUG_PRINTLN("Radio params restored"); } +#ifdef WITH_WEBCONFIG + if (_webconfig) { + _webconfig->tick(millis()); + if (!_webconfig->isRunning() && !_webconfig->isStopping()) { + delete _webconfig; // teardown finished (or start failed): reclaim the heap + _webconfig = NULL; + } + } +#endif + // is pending dirty contacts write needed? if (dirty_contacts_expiry && millisHasNowPassed(dirty_contacts_expiry)) { acl.save(_fs, MyMesh::saveFilter); diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index cd556be3..466fb09d 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -28,6 +28,7 @@ #ifdef WITH_MQTT_BRIDGE #include "helpers/bridges/MQTTBridge.h" #define WITH_BRIDGE +#include "helpers/esp32/WebConfigServer.h" // defines WITH_WEBCONFIG on ESP32 #endif /* ------------------------------ Config -------------------------------- */ @@ -94,7 +95,11 @@ struct PostInfo { char text[MAX_POST_TEXT_LEN+1]; }; -class MyMesh : public mesh::Mesh, public CommonCLICallbacks { +class MyMesh : public mesh::Mesh, public CommonCLICallbacks +#ifdef WITH_WEBCONFIG + , public WebConfigServer::Callbacks +#endif +{ FILESYSTEM* _fs; uint32_t last_millis; uint64_t uptime_millis; @@ -129,6 +134,12 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { #ifdef WITH_MQTT_BRIDGE AlertReporter _alerter; #endif +#ifdef WITH_WEBCONFIG + WebConfigServer* _webconfig = NULL; // heap-allocated while running, freed on stop + bool _wc_batch_active = false; // coalesce bridge restarts during a config batch + bool _wc_restart_pending = false; + uint8_t _wc_slot_restart_mask = 0; +#endif void addPost(ClientInfo* client, const char* postData); void pushPostToClient(ClientInfo* client, PostInfo& post); @@ -282,6 +293,12 @@ public: void restartBridge() override { if (!bridge || !bridge->isRunning()) return; +#ifdef WITH_WEBCONFIG + if (_wc_batch_active) { // coalesced: applied once in onConfigBatchEnd() + _wc_restart_pending = true; + return; + } +#endif bridge->end(); char device_id[65]; mesh::LocalIdentity self_id = getSelfId(); @@ -299,6 +316,12 @@ public: void restartBridgeSlot(int slot) override { #ifdef WITH_MQTT_BRIDGE if (!bridge || !bridge->isRunning()) return; +#ifdef WITH_WEBCONFIG + if (_wc_batch_active && slot >= 0 && slot < 8) { + _wc_slot_restart_mask |= (uint8_t)(1u << slot); + return; + } +#endif bridge->setSlotPreset(slot, _cli.getObserverPrefs()->mqtt_slot_preset[slot]); #else (void)slot; @@ -324,4 +347,25 @@ public: return bridge->ntpDiag(reply, reply_size, verbose); } #endif + +#ifdef WITH_WEBCONFIG + // CommonCLICallbacks: `start webconfig [ap]` / `stop webconfig` + bool startWebConfig(bool force_ap, char* reply) override; + bool stopWebConfig(char* reply) override; + + // WebConfigServer::Callbacks - all invoked from tick() on the loop task + void execCommand(char* cmd, char* reply) override { + handleCommand(0, cmd, reply); + } + void rebootNow() override { + _cli.getBoard()->reboot(); + } + void onConfigBatchStart() override { + _wc_batch_active = true; + _wc_restart_pending = false; + _wc_slot_restart_mask = 0; + } + void onConfigBatchEnd() override; + void buildStatsJson(char* buf, size_t buf_size) override; +#endif }; diff --git a/examples/simple_room_server/UITask.cpp b/examples/simple_room_server/UITask.cpp index cad436e5..ddc222c3 100644 --- a/examples/simple_room_server/UITask.cpp +++ b/examples/simple_room_server/UITask.cpp @@ -8,6 +8,7 @@ #ifdef WITH_MQTT_BRIDGE #include +#include // defines WITH_WEBCONFIG on ESP32 #endif #define AUTO_OFF_MILLIS 20000 // 20 seconds @@ -77,6 +78,43 @@ void UITask::renderCurrScreen() { _display->setCursor((_display->width() - typeWidth) / 2, 48); _display->print(node_type); } else { // home screen +#ifdef WITH_WEBCONFIG + if (WebConfigServer::isRebootPending()) { + // save confirmed on-device: show ground truth even if the browser + // lost its connection before the confirmation reached it + _display->setTextSize(1); + _display->setColor(DisplayDriver::GREEN); + _display->setCursor(0, 14); + _display->print("Config saved!"); + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 30); + _display->print("Rebooting..."); + return; + } + char wc_ssid[33], wc_ip[16]; + if (WebConfigServer::getSetupInfo(wc_ssid, sizeof(wc_ssid), wc_ip, sizeof(wc_ip))) { + // setup portal active: show join instructions instead of the home screen + _display->setTextSize(1); + _display->setColor(DisplayDriver::GREEN); + _display->setCursor(0, 0); + _display->print("Observer WiFi Setup"); + + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 14); + _display->print("Join WiFi:"); + _display->setColor(DisplayDriver::YELLOW); + _display->setCursor(6, 24); + _display->print(wc_ssid); + + _display->setColor(DisplayDriver::LIGHT); + _display->setCursor(0, 40); + _display->print("Then browse to:"); + _display->setColor(DisplayDriver::YELLOW); + _display->setCursor(6, 50); + _display->print(wc_ip); + return; + } +#endif // node name _display->setCursor(0, 0); _display->setTextSize(1); @@ -126,6 +164,15 @@ void UITask::loop() { } #endif +#ifdef WITH_WEBCONFIG + // While the setup portal is up there's no user button to wake the screen + // reliably - keep it on so the join instructions stay visible. + if (WebConfigServer::getSetupInfo(NULL, 0, NULL, 0)) { + if (!_display->isOn()) _display->turnOn(); + _auto_off = millis() + AUTO_OFF_MILLIS; + } +#endif + if (_display->isOn()) { if (millis() >= _next_refresh) { _display->startFrame(); diff --git a/platformio.ini b/platformio.ini index d38402c7..b2d0c662 100644 --- a/platformio.ini +++ b/platformio.ini @@ -58,7 +58,9 @@ build_src_filter = extends = arduino_base platform = platformio/espressif32@6.11.0 monitor_filters = esp32_exception_decoder -extra_scripts = merge-bin.py +extra_scripts = + pre:scripts/generate_webconfig_html.py + merge-bin.py build_flags = ${arduino_base.build_flags} -D ESP32_PLATFORM ; Route esp_transport_ws_init through src/helpers/ESP32WsTransportFix.cpp to @@ -66,6 +68,7 @@ build_flags = ${arduino_base.build_flags} -Wl,--wrap=esp_transport_ws_init ; -D ESP32_CPU_FREQ=80 ; change it to your need build_src_filter = ${arduino_base.build_src_filter} + + ; empty TU unless WITH_MQTT_BRIDGE [esp32_ota] lib_deps = diff --git a/scripts/generate_webconfig_html.py b/scripts/generate_webconfig_html.py new file mode 100644 index 00000000..d587fa5d --- /dev/null +++ b/scripts/generate_webconfig_html.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Pre-build script: gzip webui/index.html into a PROGMEM C header so the +# webconfig portal can serve the page straight from flash with +# Content-Encoding: gzip. The generated header is .gitignored; this script +# regenerates it whenever the source page (or this script) is newer. +# +# Output: src/helpers/esp32/WebConfigHtml.h +# WEBCONFIG_HTML_GZ[] - gzipped page (PROGMEM) +# WEBCONFIG_HTML_GZ_LEN - byte length +# WEBCONFIG_HTML_ETAG - quoted strong ETag (sha256 prefix of the gz body) + +import gzip +import hashlib +import os +import sys + +Import("env") # noqa: F821 + +SOURCE = os.path.join("webui", "index.html") +OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h") +# __file__ is not defined inside PIO/SCons-executed extra_scripts +SCRIPT = os.path.join("scripts", "generate_webconfig_html.py") + + +def status(msg): + sys.stderr.write("WebConfig HTML: %s\n" % msg) + + +def needs_rebuild(): + if not os.path.isfile(OUTPUT): + return True + out_mtime = os.path.getmtime(OUTPUT) + if os.path.getmtime(SOURCE) > out_mtime: + return True + if os.path.isfile(SCRIPT) and os.path.getmtime(SCRIPT) > out_mtime: + return True + return False + + +def main(): + if not os.path.isfile(SOURCE): + status("ERROR: %s not found" % SOURCE) + sys.exit(2) + + if not needs_rebuild(): + return + + with open(SOURCE, "rb") as f: + raw = f.read() + + # mtime=0 keeps the gzip output (and therefore the ETag) deterministic + gz = gzip.compress(raw, compresslevel=9, mtime=0) + etag = hashlib.sha256(gz).hexdigest()[:16] + + lines = [] + lines.append("// Auto-generated by scripts/generate_webconfig_html.py from %s" % SOURCE.replace(os.sep, "/")) + lines.append("// DO NOT EDIT - edit webui/index.html instead.") + lines.append("#pragma once") + lines.append("#include ") + lines.append("#include ") + lines.append("") + lines.append("const uint32_t WEBCONFIG_HTML_GZ_LEN = %d;" % len(gz)) + lines.append('const char WEBCONFIG_HTML_ETAG[] = "\\"%s\\"";' % etag) + lines.append("const uint8_t WEBCONFIG_HTML_GZ[] PROGMEM = {") + for i in range(0, len(gz), 16): + chunk = gz[i:i + 16] + lines.append(" " + "".join("0x%02x," % b for b in chunk)) + lines.append("};") + lines.append("") + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w") as f: + f.write("\n".join(lines)) + + status("%s -> %s (%d bytes raw, %d bytes gzipped)" % (SOURCE, OUTPUT, len(raw), len(gz))) + + +main() diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index ef4a797c..f0c0f479 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -343,6 +343,18 @@ public: return false; }; + // Browser-based config portal (ESP32 WITH_MQTT_BRIDGE builds override). + // force_ap=true requests the SoftAP setup portal even when WiFi is configured. + // Returns true if handled (reply filled either way when true). + virtual bool startWebConfig(bool force_ap, char* reply) { + (void)force_ap; (void)reply; + return false; + }; + virtual bool stopWebConfig(char* reply) { + (void)reply; + return false; + }; + // Probe all configured NTP servers for connectivity (verbose=serial console gets a // detailed table; otherwise reply gets a compact " ok|fail" list). virtual bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) { diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 695aacef..75d14d48 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -979,6 +979,21 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, strcpy(reply, "ERR: online OTA not supported on this build"); #endif return true; + } else if (memcmp(command, "start webconfig", 15) == 0 && (command[15] == 0 || command[15] == ' ')) { + // Web config portal: `start webconfig` binds to the LAN IP (or raises the + // setup AP when WiFi is unconfigured); `start webconfig ap` forces the AP. + bool force_ap = (command[15] == ' ' && strcmp(&command[16], "ap") == 0); + if (command[15] == ' ' && !force_ap) { + strcpy(reply, "ERR: usage start webconfig [ap]"); + } else if (!_callbacks->startWebConfig(force_ap, reply)) { + strcpy(reply, "ERR: webconfig not supported on this build"); + } + return true; + } else if (strcmp(command, "stop webconfig") == 0) { + if (!_callbacks->stopWebConfig(reply)) { + strcpy(reply, "ERR: webconfig not supported on this build"); + } + return true; } else if (memcmp(command, "alert test", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { // Send a one-off test alert on the configured alert channel. const char* extra = command[10] == ' ' ? &command[11] : ""; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 15104a65..c7e7e66d 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -289,6 +289,37 @@ void MQTTBridge::formatMqttStatsReply(char* buf, size_t bufsize) { } } +// Structured per-slot status for the webconfig stats endpoint. Same state +// derivation as formatMqttStatusReply above. Returns false for out-of-range, +// unconfigured, or bridge-not-running slots. +bool MQTTBridge::getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out) { + if (out == nullptr || slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return false; + if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) return false; + MQTTBridge* b = s_mqtt_bridge_instance; + const MQTTSlot& slot = b->_slots[slot_index]; + + if (!slot.enabled && slot.preset) { + out->name = slot.preset->name; + out->state = "inactive"; + } else if (!slot.enabled) { + return false; // unconfigured slot + } else { + out->name = slot.preset ? slot.preset->name : "custom"; + if (!b->isSlotReady(slot_index)) { + out->state = "wait"; + } else if (slot.connected) { + out->state = "ok"; + } else if (slot.circuit_breaker_tripped) { + out->state = "fail"; + } else { + out->state = "disc"; + } + } + out->publish_ok = slot.client ? slot.client->getPublishOk() : 0; + out->publish_err = slot.client ? slot.client->getPublishErr() : 0; + return true; +} + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 69782bd4..ca065b6c 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -471,6 +471,17 @@ public: /** On-demand publish-health + heap snapshot for `get mqtt.stats` (per-slot ok/err, * outbox size, free/max heap, queue depth). */ static void formatMqttStatsReply(char* buf, size_t bufsize); + /** Structured per-slot snapshot for the webconfig stats endpoint. Same state + * logic as formatMqttStatusReply, same cross-task read semantics. Returns + * false when the bridge is not running, the index is out of range, or the + * slot is unconfigured (skip it). */ + struct SlotStatusSnapshot { + const char* name; // preset name or "custom" + const char* state; // "inactive" | "wait" | "ok" | "fail" | "disc" + unsigned long publish_ok; + unsigned long publish_err; + }; + static bool getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out); /** True when WiFi is set and at least one MQTT slot can run (preset + custom host if needed). */ static bool isConfigValid(const MQTTPrefs* obs); static void formatSlotDiagReply(char* buf, size_t bufsize, int slot_index); diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp new file mode 100644 index 00000000..41e06307 --- /dev/null +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -0,0 +1,760 @@ +#if defined(ESP_PLATFORM) && defined(WITH_MQTT_BRIDGE) + +#include "WebConfigServer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "WebConfigHtml.h" + +// Placeholder sent instead of stored secrets; POSTs carrying it are dropped +// so an untouched password field never overwrites the stored value. +static const char SECRET_SENTINEL[] = "********"; + +// Keys the web UI may drive through the CLI `set` handlers. Everything else +// is rejected, so a crafted request can't reach arbitrary commands (`erase`, +// `password`, ...) through the batch. +static const char* const ALLOWED_SET_KEYS[] = { + // NodePrefs (radio / node) + "name", "lat", "lon", "radio", "tx", "af", "rxdelay", "txdelay", + "cad", "radio.rxgain", "advert.interval", "flood.advert.interval", + // MQTTPrefs (WiFi / MQTT / misc observer) + "wifi.ssid", "wifi.pwd", "wifi.powersave", + "mqtt.origin", "mqtt.iata", "mqtt.status", "mqtt.packets", "mqtt.raw", + "mqtt.tx", "mqtt.rx", "mqtt.interval", "mqtt.ntp", "mqtt.owner", "mqtt.email", + "timezone", "timezone.offset", "snmp", "snmp.community", +}; +static const char* const ALLOWED_SLOT_KEYS[] = { + "preset", "server", "port", "username", "password", "token", "topic", "audience", +}; + +static bool isAllowedSetKey(const char* key) { + for (size_t i = 0; i < sizeof(ALLOWED_SET_KEYS) / sizeof(ALLOWED_SET_KEYS[0]); i++) { + if (strcmp(key, ALLOWED_SET_KEYS[i]) == 0) return true; + } + // mqtt<1-6>. + if (memcmp(key, "mqtt", 4) == 0 && key[4] >= '1' && key[4] <= ('0' + MAX_MQTT_SLOTS) + && key[5] == '.') { + for (size_t i = 0; i < sizeof(ALLOWED_SLOT_KEYS) / sizeof(ALLOWED_SLOT_KEYS[0]); i++) { + if (strcmp(&key[6], ALLOWED_SLOT_KEYS[i]) == 0) return true; + } + } + return false; +} + +static bool isSecretKey(const char* key) { + if (strcmp(key, "wifi.pwd") == 0) return true; + if (memcmp(key, "mqtt", 4) == 0 && key[5] == '.' + && (strcmp(&key[6], "password") == 0 || strcmp(&key[6], "token") == 0)) return true; + return false; +} + +// Constant-time-ish comparison so login timing doesn't leak a prefix match. +static bool fixedTimeEquals(const char* a, const char* b, size_t max_len) { + size_t la = strnlen(a, max_len), lb = strnlen(b, max_len); + uint8_t diff = (la == lb) ? 0 : 1; + for (size_t i = 0; i < max_len; i++) { + char ca = (i < la) ? a[i] : 0; + char cb = (i < lb) ? b[i] : 0; + diff |= (uint8_t)(ca ^ cb); + } + return diff == 0; +} + +// RAII lock; tolerates a null mutex (allocation failure) by not locking. +struct WCLock { + SemaphoreHandle_t h; + explicit WCLock(SemaphoreHandle_t s) : h(s) { if (h) xSemaphoreTake(h, portMAX_DELAY); } + ~WCLock() { if (h) xSemaphoreGive(h); } +}; + +WebConfigServer* WebConfigServer::_active = NULL; + +WebConfigServer::WebConfigServer(NodePrefs* prefs, MQTTPrefs* obs, Callbacks* callbacks, + const uint8_t* pub_key, const char* fw_ver, + const char* role, const char* board_name) + : _prefs(prefs), _obs(obs), _cb(callbacks), _pub_key(pub_key), + _fw_ver(fw_ver), _role(role), _board_name(board_name) { + _mux = xSemaphoreCreateMutex(); + _active = this; +} + +WebConfigServer::~WebConfigServer() { + if (_active == this) _active = NULL; + if (_mux) vSemaphoreDelete(_mux); +} + +bool WebConfigServer::isRebootPending() { + WebConfigServer* w = _active; + return w != NULL && w->_reboot_at != 0 && w->_batch_reboot && + w->_batch_state == BATCH_DONE; +} + +bool WebConfigServer::getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t ip_len) { + WebConfigServer* w = _active; + if (w == NULL || w->_mode != MODE_SETUP || w->_stopping) return false; + if (ssid && ssid_len > 0) { + strncpy(ssid, w->_ap_ssid, ssid_len - 1); + ssid[ssid_len - 1] = 0; + } + if (ip && ip_len > 0) { + snprintf(ip, ip_len, "%s", WiFi.softAPIP().toString().c_str()); + } + return true; +} + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- + +bool WebConfigServer::startSetupMode(char reply[]) { + if (_mode != MODE_OFF || _stopping) { + strcpy(reply, "Err: webconfig busy"); + return false; + } + // AP_STA (not pure AP) so the WiFi scan for the SSID picker works while + // the AP is up. STA stays unconnected - the bridge won't touch WiFi + // while wifi_ssid is empty, and `start webconfig ap` requires it stopped. + WiFi.mode(WIFI_AP_STA); + snprintf(_ap_ssid, sizeof(_ap_ssid), "MeshCore-Setup-%02X%02X", _pub_key[0], _pub_key[1]); +#ifdef WEBCONFIG_AP_PASSWORD + bool ap_ok = WiFi.softAP(_ap_ssid, WEBCONFIG_AP_PASSWORD); +#else + bool ap_ok = WiFi.softAP(_ap_ssid); +#endif + if (!ap_ok) { + WiFi.mode(WIFI_OFF); + strcpy(reply, "Err: failed to start AP"); + return false; + } + delay(100); // let the AP netif settle before reading its IP + IPAddress ip = WiFi.softAPIP(); + + _dns = new DNSServer(); + _dns->start(53, "*", ip); // captive portal: every name resolves to us + + createServer(); + _mode = MODE_SETUP; + _was_setup_ap = true; + _last_activity = millis(); + WiFi.scanNetworks(true); // pre-populate the SSID picker + + sprintf(reply, "WebConfig AP started: join '%s' then open http://%s/", _ap_ssid, ip.toString().c_str()); + return true; +} + +bool WebConfigServer::startLanMode(char reply[]) { + if (_mode != MODE_OFF || _stopping) { + strcpy(reply, "Err: webconfig busy"); + return false; + } + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "Err: WiFi not connected"); + return false; + } + createServer(); + _mode = MODE_LAN; + _last_activity = millis(); + + int pos = sprintf(reply, "WebConfig started: http://%s/ (admin password login)", + WiFi.localIP().toString().c_str()); + if (heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL) < 60 * 1024) { + sprintf(reply + pos, " WARN: low heap"); + } + return true; +} + +void WebConfigServer::createServer() { + _server = new AsyncWebServer(80); + // iOS caches plain-HTTP GETs aggressively (keyed by URL, surviving even a + // device reflash behind the same IP), which poisons /api/config/result and + // friends with stale responses from earlier sessions. Forbid caching on + // every response; the HTML is small enough to refetch per visit. + DefaultHeaders::Instance().addHeader("Cache-Control", "no-store"); + registerRoutes(); + _server->begin(); +} + +void WebConfigServer::requestStop() { + if (_mode == MODE_OFF && !_stopping) return; + if (_server) _server->end(); + if (_dns) _dns->stop(); + _mode = MODE_OFF; + _stopping = true; + // Deleting an AsyncWebServer with live connections is a known crash source; + // give in-flight responses a grace period before freeing. + _delete_at = millis() + 2000; + if (_delete_at == 0) _delete_at = 1; +} + +void WebConfigServer::finalizeTeardown() { + delete _server; + _server = NULL; + delete _dns; + _dns = NULL; + if (_was_setup_ap) { + WiFi.softAPdisconnect(true); + // Nothing else owns WiFi when we raised the AP: either the node is + // unconfigured, or `start webconfig ap` required the bridge stopped. + if (_obs->wifi_ssid[0] == 0) { + WiFi.mode(WIFI_OFF); + } else { + WiFi.mode(WIFI_STA); + } + _was_setup_ap = false; + } + _stopping = false; + _delete_at = 0; + _reboot_at = 0; + _batch_state = BATCH_IDLE; + _batch_next = 0; + _batch_reboot_armed = false; + _session_token[0] = 0; + _stats_json[0] = 0; + if (_cb) _cb->onWebConfigStopped(); +} + +void WebConfigServer::tick(uint32_t now) { + if (_stopping) { + if (_delete_at && (int32_t)(now - _delete_at) >= 0) finalizeTeardown(); + return; + } + if (_mode == MODE_OFF) return; + + if (_dns) _dns->processNextRequest(); + + if (_batch_state == BATCH_PENDING) drainBatch(now); + + if (_reboot_at && (int32_t)(now - _reboot_at) >= 0) { + Serial.printf("WC: rebooting now (%s)\n", _batch_reboot_armed ? "confirmed" : "fallback"); + _cb->rebootNow(); // does not return + } + + if ((int32_t)(_diag_until - now) > 0 && (now - _diag_last) >= 1000) { + _diag_last = now; + Serial.printf("WC: diag sta=%d heap=%u batch=%d/%d state=%d\n", + (int)WiFi.softAPgetStationNum(), (unsigned)ESP.getFreeHeap(), + (int)_batch_next, (int)_batch_count, (int)_batch_state); + } + + // Refresh the stats snapshot only while a client is actually polling. + if ((int32_t)(_stats_wanted_until - now) > 0 && (now - _stats_built_at) >= 2000) { + WCLock lock(_mux); + _cb->buildStatsJson(_stats_json, sizeof(_stats_json)); + _stats_built_at = now; + } + + // Idle timeout: only the setup AP auto-stops (a deployed node must not be + // left broadcasting an open AP). LAN mode runs until `stop webconfig`. + if (_mode == MODE_SETUP && WiFi.softAPgetStationNum() == 0 && + (now - _last_activity) > WEBCONFIG_AP_IDLE_TIMEOUT_MS) { + requestStop(); + } +} + +void WebConfigServer::drainBatch(uint32_t now) { + // One command per call, spaced out. Each `set` persists prefs with a flash + // write, and flash writes stall the WiFi task (flash cache off); running a + // whole batch back-to-back starves the softAP of beacons long enough for + // clients (iPhones especially) to drop off mid-save. + if (_batch_next == 0) { + _cb->onConfigBatchStart(); + } else if (_batch_next < _batch_count && (int32_t)(now - _batch_last_cmd) < 25) { + return; // let the WiFi task breathe between flash writes + } + if (_batch_next < _batch_count) { + BatchEntry& e = _batch[_batch_next++]; + e.reply[0] = 0; + uint32_t t0 = millis(); + _cb->execCommand(e.cmd, e.reply); + if (e.reply[0] == 0) strcpy(e.reply, "OK"); + _batch_last_cmd = millis(); + Serial.printf("WC: cmd %d/%d '%s' took %lums\n", (int)_batch_next, (int)_batch_count, + e.key, (unsigned long)(_batch_last_cmd - t0)); + if (_batch_next < _batch_count) return; // more commands next tick + } + _cb->onConfigBatchEnd(); + WCLock lock(_mux); + _batch_state = BATCH_DONE; + if (_batch_reboot) { + // Fallback only: the real 3 s reboot timer is armed when the client reads + // /api/config/result (handleConfigResult), so the browser gets its + // confirmation before the AP/WiFi drops. This covers a client that + // disconnected and never polls — generous enough for a phone that got + // bounced off the AP mid-save to rejoin and fetch its confirmation. + _reboot_at = now + 30000; + if (_reboot_at == 0) _reboot_at = 1; + } +} + +// --------------------------------------------------------------------------- +// Routes / auth +// --------------------------------------------------------------------------- + +// Diagnostic trace of every request that reaches the server (async_tcp task). +// Distinguishes "client stopped sending" from "server stopped accepting" when +// a save's confirmation polls go missing on hardware. +static void wcLogReq(AsyncWebServerRequest* r) { + Serial.printf("WC: http %s %s\n", r->methodToString(), r->url().c_str()); +} + +void WebConfigServer::registerRoutes() { + _server->on("/", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleRoot(r); }); + _server->on("/api/status", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleStatus(r); }); + _server->on("/api/presets", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handlePresets(r); }); + _server->on("/api/login", HTTP_POST, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleLogin(r); }, + NULL, collectBody); + _server->on("/api/logout", HTTP_POST, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleLogout(r); }); + // NB: plain-string routes match sub-paths too ("/api/config" matches + // "/api/config/result") and handlers run in registration order, so the more + // specific route MUST be registered first or it never fires. This was why + // save confirmations were lost: result polls were answered with config JSON. + _server->on("/api/config/result", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleConfigResult(r); }); + _server->on("/api/config", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleConfigGet(r); }); + _server->on("/api/config", HTTP_POST, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleConfigPost(r); }, + NULL, collectBody); + _server->on("/api/stats", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleStats(r); }); + _server->on("/api/scan", HTTP_GET, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleScan(r); }); + _server->on("/api/reboot", HTTP_POST, [this](AsyncWebServerRequest* r) { wcLogReq(r); handleReboot(r); }); + _server->on("/api/portal/exit", HTTP_POST, [this](AsyncWebServerRequest* r) { wcLogReq(r); handlePortalExit(r); }); + _server->onNotFound([this](AsyncWebServerRequest* r) { wcLogReq(r); handleNotFound(r); }); +} + +// Accumulate a small JSON body into request->_tempObject (freed automatically +// by the request destructor). Oversized bodies are left unbuffered and +// rejected in the completion handler. +void WebConfigServer::collectBody(AsyncWebServerRequest* req, uint8_t* data, size_t len, + size_t index, size_t total) { + if (total == 0 || total > MAX_BODY) return; + if (index == 0) { + req->_tempObject = malloc(total + 1); + if (req->_tempObject) ((char*)req->_tempObject)[total] = 0; + } + if (req->_tempObject) memcpy((uint8_t*)req->_tempObject + index, data, len); +} + +bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { + _last_activity = millis(); + if (_mode == MODE_SETUP) return true; // physical proximity implied, nothing configured + if (_mode != MODE_LAN) return false; + if (_session_token[0] == 0) return false; + if (!req->hasHeader("Cookie")) return false; + const String& cookies = req->getHeader("Cookie")->value(); + int idx = cookies.indexOf("wcs="); + if (idx < 0 || (int)cookies.length() < idx + 4 + 32) return false; + String token = cookies.substring(idx + 4, idx + 4 + 32); + uint32_t now = millis(); + if ((uint32_t)(now - _session_last_seen) > WEBCONFIG_SESSION_TTL_MS) return false; + if (!fixedTimeEquals(token.c_str(), _session_token, 32)) return false; + _session_last_seen = now; // sliding expiry + return true; +} + +// --------------------------------------------------------------------------- +// Handlers (async_tcp task - no CLI/prefs writes, no radio access) +// --------------------------------------------------------------------------- + +void WebConfigServer::handleRoot(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + _last_activity = millis(); + if (req->hasHeader("If-None-Match") && + req->getHeader("If-None-Match")->value() == WEBCONFIG_HTML_ETAG) { + req->send(304); + return; + } + AsyncWebServerResponse* res = + req->beginResponse(200, "text/html", WEBCONFIG_HTML_GZ, WEBCONFIG_HTML_GZ_LEN); + res->addHeader("Content-Encoding", "gzip"); + res->addHeader("ETag", WEBCONFIG_HTML_ETAG); + req->send(res); +} + +void WebConfigServer::handleStatus(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + bool authed = checkAuth(req); + + DynamicJsonDocument doc(512); + doc["mode"] = (_mode == MODE_SETUP) ? "setup" : "lan"; + doc["auth"] = authed; + doc["needs_setup"] = (_obs->wifi_ssid[0] == 0); + doc["name"] = (const char*)_prefs->node_name; + char node_id[17]; + for (int i = 0; i < 8; i++) sprintf(&node_id[i * 2], "%02x", _pub_key[i]); + doc["node_id"] = node_id; + doc["fw"] = _fw_ver; + doc["role"] = _role; + doc["board"] = _board_name; + doc["uptime_s"] = millis() / 1000; + doc["runtime_slots"] = RUNTIME_MQTT_SLOTS; + doc["max_slots"] = MAX_MQTT_SLOTS; + + AsyncResponseStream* res = req->beginResponseStream("application/json"); + serializeJson(doc, *res); + req->send(res); +} + +void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + _last_activity = millis(); + if (_mode == MODE_SETUP) { // no auth in setup mode + req->send(200, "application/json", "{\"ok\":true}"); + return; + } + uint32_t now = millis(); + if (_login_lock_until && (int32_t)(now - _login_lock_until) < 0) { + req->send(429, "application/json", "{\"error\":\"locked, retry in 30s\"}"); + return; + } + const char* body = (const char*)req->_tempObject; + DynamicJsonDocument doc(256); + if (!body || deserializeJson(doc, body) != DeserializationError::Ok) { + req->send(400, "application/json", "{\"error\":\"bad request\"}"); + return; + } + const char* pwd = doc["password"] | ""; + if (!fixedTimeEquals(pwd, _prefs->password, sizeof(_prefs->password))) { + if (++_login_fails >= 5) { + _login_lock_until = now + 30000; + if (_login_lock_until == 0) _login_lock_until = 1; + _login_fails = 0; + } + req->send(401, "application/json", "{\"error\":\"wrong password\"}"); + return; + } + _login_fails = 0; + _login_lock_until = 0; + for (int i = 0; i < 4; i++) sprintf(&_session_token[i * 8], "%08lx", (unsigned long)esp_random()); + _session_last_seen = now; + + AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); + char cookie[80]; + sprintf(cookie, "wcs=%s; HttpOnly; SameSite=Lax; Path=/", _session_token); + res->addHeader("Set-Cookie", cookie); + req->send(res); +} + +void WebConfigServer::handleLogout(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + _session_token[0] = 0; + AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); + res->addHeader("Set-Cookie", "wcs=; Max-Age=0; Path=/"); + req->send(res); +} + +void WebConfigServer::handleConfigGet(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + if (!checkAuth(req)) { req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + + DynamicJsonDocument doc(6144); + { + WCLock lock(_mux); + + JsonObject radio = doc.createNestedObject("radio"); + // round via double so float error doesn't leak into the JSON + // (910.525f would otherwise serialize as 910.5250244) + radio["freq"] = (double)roundf(_prefs->freq * 1000.0f) / 1000.0; + radio["bw"] = (double)roundf(_prefs->bw * 100.0f) / 100.0; + radio["sf"] = _prefs->sf; + radio["cr"] = _prefs->cr; + radio["tx"] = _prefs->tx_power_dbm; + radio["af"] = _prefs->airtime_factor; + radio["rxdelay"] = _prefs->rx_delay_base; + radio["txdelay"] = _prefs->tx_delay_factor; + radio["cad"] = (bool)_prefs->cad_enabled; + radio["rxgain"] = (bool)_prefs->rx_boosted_gain; + radio["name"] = (const char*)_prefs->node_name; + radio["lat"] = _prefs->node_lat; + radio["lon"] = _prefs->node_lon; + radio["advert_interval"] = _prefs->advert_interval * 2; // stored as mins/2 + radio["flood_advert_interval"] = _prefs->flood_advert_interval; // hours + + JsonObject wifi = doc.createNestedObject("wifi"); + wifi["ssid"] = (const char*)_obs->wifi_ssid; + wifi["pwd"] = _obs->wifi_password[0] ? SECRET_SENTINEL : ""; + wifi["powersave"] = _obs->wifi_power_save == 0 ? "min" + : _obs->wifi_power_save == 2 ? "max" : "none"; + + JsonObject mqtt = doc.createNestedObject("mqtt"); + mqtt["origin"] = (const char*)_obs->mqtt_origin; + mqtt["iata"] = (const char*)_obs->mqtt_iata; + mqtt["status"] = (bool)_obs->mqtt_status_enabled; + mqtt["packets"] = (bool)_obs->mqtt_packets_enabled; + mqtt["raw"] = (bool)_obs->mqtt_raw_enabled; + mqtt["tx"] = _obs->mqtt_tx_enabled == 2 ? "advert" + : _obs->mqtt_tx_enabled == 1 ? "on" : "off"; + mqtt["rx"] = (bool)_obs->mqtt_rx_enabled; + mqtt["interval"] = _obs->mqtt_status_interval / 60000; // CLI takes minutes + mqtt["timezone"] = (const char*)_obs->timezone_string; + mqtt["timezone_offset"] = _obs->timezone_offset; + mqtt["ntp"] = (const char*)_obs->mqtt_ntp_server; + mqtt["owner"] = (const char*)_obs->mqtt_owner_public_key; + mqtt["email"] = (const char*)_obs->mqtt_email; + mqtt["snmp"] = (bool)_obs->snmp_enabled; + mqtt["snmp_community"] = (const char*)_obs->snmp_community; + + JsonArray slots = mqtt.createNestedArray("slots"); + for (int i = 0; i < MAX_MQTT_SLOTS; i++) { + JsonObject s = slots.createNestedObject(); + s["preset"] = (const char*)_obs->mqtt_slot_preset[i]; + s["server"] = (const char*)_obs->mqtt_slot_host[i]; + s["port"] = _obs->mqtt_slot_port[i]; + s["username"] = (const char*)_obs->mqtt_slot_username[i]; + s["password"] = _obs->mqtt_slot_password[i][0] ? SECRET_SENTINEL : ""; + s["token"] = _obs->mqtt_slot_token[i][0] ? SECRET_SENTINEL : ""; + s["topic"] = (const char*)_obs->mqtt_slot_topic[i]; + s["audience"] = (const char*)_obs->mqtt_slot_audience[i]; + } + } + + AsyncResponseStream* res = req->beginResponseStream("application/json"); + serializeJson(doc, *res); + req->send(res); +} + +void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + if (!checkAuth(req)) { req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + if (req->contentLength() > MAX_BODY) { + req->send(413, "application/json", "{\"error\":\"body too large\"}"); + return; + } + const char* body = (const char*)req->_tempObject; + DynamicJsonDocument doc(6144); + if (!body || deserializeJson(doc, body) != DeserializationError::Ok) { + req->send(400, "application/json", "{\"error\":\"bad json\"}"); + return; + } + bool reboot_after = doc["reboot"] | false; + JsonObject set = doc["set"]; + + WCLock lock(_mux); + // A DONE batch stays readable until the next POST claims the slot, so a + // client that lost the result response can re-poll instead of failing. + if (_batch_state == BATCH_PENDING) { + req->send(409, "application/json", "{\"error\":\"busy\"}"); + return; + } + + int count = 0; + for (JsonPair kv : set) { + const char* key = kv.key().c_str(); + const char* val = kv.value().as(); + if (!val || !isAllowedSetKey(key)) { + char err[96]; + snprintf(err, sizeof(err), "{\"error\":\"bad key\",\"key\":\"%.32s\"}", key); + req->send(400, "application/json", err); + return; + } + if (isSecretKey(key) && strcmp(val, SECRET_SENTINEL) == 0) continue; // unchanged + if (count >= MAX_BATCH) { + req->send(400, "application/json", "{\"error\":\"too many changes\"}"); + return; + } + BatchEntry& e = _batch[count]; + strncpy(e.key, key, sizeof(e.key) - 1); + e.key[sizeof(e.key) - 1] = 0; + // Build "set ", stripping CR/LF so a value can't smuggle in + // a second command. + int pos = snprintf(e.cmd, sizeof(e.cmd), "set %s ", key); + for (const char* p = val; *p && pos < (int)sizeof(e.cmd) - 1; p++) { + if (*p == '\r' || *p == '\n') continue; + e.cmd[pos++] = *p; + } + e.cmd[pos] = 0; + count++; + } + if (count == 0 && !reboot_after) { + req->send(400, "application/json", "{\"error\":\"no changes\"}"); + return; + } + _batch_count = count; + _batch_next = 0; + _batch_reboot = reboot_after; + _batch_reboot_armed = false; + _batch_state = BATCH_PENDING; // tick() picks it up on the loop task + uint32_t du = millis() + 60000; + if (du == 0) du = 1; + _diag_until = du; + Serial.printf("WC: config POST accepted, %d cmds, reboot=%d\n", count, (int)reboot_after); + + char msg[64]; + snprintf(msg, sizeof(msg), "{\"state\":\"pending\",\"count\":%d}", count); + req->send(202, "application/json", msg); +} + +void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { Serial.println("WC: result read -> 503 (mode off)"); req->send(503); return; } + if (!checkAuth(req)) { Serial.println("WC: result read -> 401"); req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + + // Entry print BEFORE the lock (racy state read is fine for diag): if this + // fires but no branch print follows, the handler is blocked on _mux. + Serial.printf("WC: result entry mode=%d state=%d\n", (int)_mode, (int)_batch_state); + WCLock lock(_mux); + if (_batch_state == BATCH_IDLE) { + Serial.println("WC: result read -> idle"); + req->send(200, "application/json", "{\"state\":\"idle\"}"); + return; + } + if (_batch_state == BATCH_PENDING) { + req->send(200, "application/json", "{\"state\":\"pending\"}"); + return; + } + Serial.printf("WC: result read -> done (reboot=%d armed=%d)\n", + (int)_batch_reboot, (int)_batch_reboot_armed); + DynamicJsonDocument doc(6144); + doc["state"] = "done"; + doc["reboot"] = _batch_reboot; + JsonArray results = doc.createNestedArray("results"); + for (int i = 0; i < _batch_count; i++) { + JsonObject r = results.createNestedObject(); + r["key"] = (const char*)_batch[i].key; + r["reply"] = (const char*)_batch[i].reply; + } + // State stays DONE (re-readable) until the next POST claims the slot. + if (_batch_reboot && !_batch_reboot_armed) { + // Confirmation delivered — reboot 3 s from now (replaces the 15 s + // drain-time fallback) so the UI can show its countdown first. Armed + // once; re-reads must not keep pushing the deadline out. + _batch_reboot_armed = true; + _reboot_at = millis() + 3000; + if (_reboot_at == 0) _reboot_at = 1; + } + + AsyncResponseStream* res = req->beginResponseStream("application/json"); + serializeJson(doc, *res); + req->send(res); +} + +void WebConfigServer::handleStats(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + if (!checkAuth(req)) { req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + uint32_t until = millis() + 15000; + if (until == 0) until = 1; + _stats_wanted_until = until; // tick() refreshes the snapshot while polled + + WCLock lock(_mux); + if (_stats_json[0] == 0) { + req->send(200, "application/json", "{\"state\":\"pending\"}"); + return; + } + req->send(200, "application/json", _stats_json); +} + +void WebConfigServer::handleScan(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + if (!checkAuth(req)) { req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + + int n = WiFi.scanComplete(); + if (req->hasParam("rescan") && n >= 0) { + WiFi.scanDelete(); + n = WIFI_SCAN_FAILED; + } + if (n == WIFI_SCAN_FAILED) { + WiFi.scanNetworks(true); + req->send(200, "application/json", "{\"state\":\"scanning\"}"); + return; + } + if (n < 0) { // WIFI_SCAN_RUNNING + req->send(200, "application/json", "{\"state\":\"scanning\"}"); + return; + } + DynamicJsonDocument doc(3072); + doc["state"] = "done"; + JsonArray nets = doc.createNestedArray("networks"); + for (int i = 0; i < n && i < 20; i++) { + JsonObject net = nets.createNestedObject(); + net["ssid"] = WiFi.SSID(i); + net["rssi"] = WiFi.RSSI(i); + net["enc"] = WiFi.encryptionType(i) != WIFI_AUTH_OPEN; + } + AsyncResponseStream* res = req->beginResponseStream("application/json"); + serializeJson(doc, *res); + req->send(res); +} + +void WebConfigServer::handlePresets(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + _last_activity = millis(); + DynamicJsonDocument doc(3072); + JsonArray arr = doc.createNestedArray("presets"); + for (int i = 0; i < MQTT_PRESET_COUNT; i++) { + const MQTTPresetDef& p = MQTT_PRESETS[i]; + JsonObject o = arr.createNestedObject(); + o["name"] = p.name; + // What the UI must collect for this preset to connect + if (p.topic_style == MQTT_TOPIC_MESHRANK) { + o["needs"] = "token"; + } else if (mqttPresetNeedsSlotCredentials(&p)) { + o["needs"] = "userpass"; + } else { + o["needs"] = "none"; + } + } + AsyncResponseStream* res = req->beginResponseStream("application/json"); + serializeJson(doc, *res); + req->send(res); +} + +void WebConfigServer::handleReboot(AsyncWebServerRequest* req) { + if (_mode == MODE_OFF) { req->send(503); return; } + if (!checkAuth(req)) { req->send(401, "application/json", "{\"error\":\"auth\"}"); return; } + _reboot_at = millis() + 1500; + if (_reboot_at == 0) _reboot_at = 1; + req->send(200, "application/json", "{\"ok\":true}"); +} + +void WebConfigServer::handlePortalExit(AsyncWebServerRequest* req) { + // Setup mode only: switch captive probes to native "success" replies so the + // OS sign-in sheet can be dismissed (iOS: "Done") without dropping the WiFi; + // the user then continues at http:/// in their real browser, + // which survives the phone sleeping (the captive sheet does not). + if (_mode != MODE_SETUP) { req->send(404); return; } + _captive_release = true; + _last_activity = millis(); + char body[64]; + snprintf(body, sizeof(body), "{\"ok\":true,\"url\":\"http://%s/\"}", + WiFi.softAPIP().toString().c_str()); + req->send(200, "application/json", body); +} + +void WebConfigServer::handleNotFound(AsyncWebServerRequest* req) { + // Captive-portal probes (/generate_204, /hotspot-detect.html, /ncsi.txt, + // /connecttest.txt, ...) all land here; a redirect to the portal makes the + // phone pop its sign-in sheet. + if (_mode == MODE_SETUP && req->method() == HTTP_GET) { + if (_captive_release) { + // Answer each OS's connectivity check natively so the sheet reports + // success and can be closed. Deliberately does NOT bump _last_activity: + // background probes must not hold the portal open past the idle timeout. + const String& url = req->url(); + if (url.indexOf("generate_204") >= 0 || url.indexOf("gen_204") >= 0) { + req->send(204); // Android + } else if (url.indexOf("hotspot-detect") >= 0 || url.indexOf("success") >= 0) { + req->send(200, "text/html", // Apple CNA + "SuccessSuccess"); + } else if (url.indexOf("ncsi.txt") >= 0) { + req->send(200, "text/plain", "Microsoft NCSI"); // Windows + } else if (url.indexOf("connecttest.txt") >= 0) { + req->send(200, "text/plain", "Microsoft Connect Test"); + } else { + req->send(404); + } + return; + } + _last_activity = millis(); + req->redirect(String("http://") + WiFi.softAPIP().toString() + "/"); + return; + } + req->send(404); +} + +#endif // ESP_PLATFORM && WITH_MQTT_BRIDGE diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h new file mode 100644 index 00000000..1dd5f31f --- /dev/null +++ b/src/helpers/esp32/WebConfigServer.h @@ -0,0 +1,169 @@ +#pragma once + +// Browser-based configuration portal for ESP32 MQTT observer builds. +// +// Two modes: +// - SETUP: open SoftAP + captive portal, raised automatically on first boot +// when no WiFi is configured (wifi_ssid empty), or manually via +// `start webconfig ap`. Save -> reboot; auto-stops after an idle timeout. +// - LAN: bound to the existing STA connection (owned by the MQTT bridge +// task), started via `start webconfig`, admin-password login required. +// +// Concurrency model: AsyncWebServer handlers run on the async_tcp task and +// must never touch the CLI, prefs persistence, or the radio. Config writes +// are marshaled into a single-slot command batch that tick() - called from +// MyMesh::loop() on the Arduino loop task - drains through the existing CLI +// `set` handlers. Prefs-struct reads and batch state are guarded by _mux. +// +// No persisted state: everything here is RAM-only, so prefs-file layouts +// (fleet-critical) are untouched. + +#if defined(ESP_PLATFORM) && defined(WITH_MQTT_BRIDGE) + +#define WITH_WEBCONFIG 1 + +#include +#include +#include + +class AsyncWebServer; +class AsyncWebServerRequest; +class DNSServer; +struct NodePrefs; +struct MQTTPrefs; + +#ifndef WEBCONFIG_AP_IDLE_TIMEOUT_MS + #define WEBCONFIG_AP_IDLE_TIMEOUT_MS (10UL * 60UL * 1000UL) +#endif +#ifndef WEBCONFIG_SESSION_TTL_MS + #define WEBCONFIG_SESSION_TTL_MS (20UL * 60UL * 1000UL) +#endif + +class WebConfigServer { +public: + enum Mode : uint8_t { MODE_OFF = 0, MODE_SETUP, MODE_LAN }; + + class Callbacks { + public: + // Run one CLI command. Called from tick() only (loop task); reply is 160 bytes. + virtual void execCommand(char* cmd, char* reply) = 0; + virtual void rebootNow() = 0; + // Bracket a config batch so bridge restarts triggered by individual + // `set` handlers can be coalesced into one. + virtual void onConfigBatchStart() {} + virtual void onConfigBatchEnd() {} + // Fill buf with the stats JSON snapshot. Called from tick() (loop task). + virtual void buildStatsJson(char* buf, size_t buf_size) = 0; + // Teardown finished (server + DNS freed, WiFi mode restored). + virtual void onWebConfigStopped() {} + }; + + WebConfigServer(NodePrefs* prefs, MQTTPrefs* obs, Callbacks* callbacks, + const uint8_t* pub_key, const char* fw_ver, + const char* role, const char* board_name); + ~WebConfigServer(); + + // For the device display: true while a setup-mode portal is active. + // Fills the AP SSID and portal IP; either buffer may be NULL to just poll. + // Call from the loop task only (same task that changes the mode). + static bool getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t ip_len); + + // For the device display: true once a config save completed and the node is + // about to reboot — ground truth for the user even if the browser lost its + // connection before the confirmation arrived. Loop task only. + static bool isRebootPending(); + + bool startSetupMode(char reply[]); // open SoftAP + DNS captive portal + bool startLanMode(char reply[]); // bind to existing STA connection + void requestStop(); // stop listening now, free after grace period + void tick(uint32_t now); // call every loop iteration + + Mode mode() const { return _mode; } + bool isRunning() const { return _mode != MODE_OFF; } + bool isStopping() const { return _stopping; } + +private: + static const int MAX_BATCH = 24; + static const size_t MAX_BODY = 4096; + enum BatchState : uint8_t { BATCH_IDLE = 0, BATCH_PENDING, BATCH_DONE }; + struct BatchEntry { + char key[24]; // allowlisted `set` key (echoed back to the UI) + char cmd[160]; // full CLI command (may contain secrets - never echoed) + char reply[160]; + }; + + NodePrefs* _prefs; + MQTTPrefs* _obs; + Callbacks* _cb; + const uint8_t* _pub_key; + const char* _fw_ver; + const char* _role; + const char* _board_name; + + AsyncWebServer* _server = NULL; + DNSServer* _dns = NULL; + SemaphoreHandle_t _mux; + Mode _mode = MODE_OFF; + bool _stopping = false; + bool _was_setup_ap = false; + char _ap_ssid[33] = {0}; + + // Most-recently-created instance, for the display's getSetupInfo() poll. + static WebConfigServer* _active; + + // Command batch: filled by async_tcp under _mux, drained by tick(). + volatile BatchState _batch_state = BATCH_IDLE; + uint8_t _batch_count = 0; + uint8_t _batch_next = 0; // drain progress (one command per tick) + uint32_t _batch_last_cmd = 0; + bool _batch_reboot = false; + bool _batch_reboot_armed = false; + BatchEntry _batch[MAX_BATCH]; + + // LAN-mode session (single slot; new login evicts the old session) + char _session_token[33] = {0}; + uint32_t _session_last_seen = 0; + uint8_t _login_fails = 0; + uint32_t _login_lock_until = 0; + + // Once set (via /api/portal/exit), OS captive probes get native "success" + // replies so the phone's sign-in sheet can be dismissed without dropping the + // WiFi, letting the user continue in their real browser. async_tcp-task only. + volatile bool _captive_release = false; + + // Save-path diagnostics: 1 Hz serial trace for 60 s after each config POST + // (AP station count, heap, batch state) to pinpoint client drops on hardware. + volatile uint32_t _diag_until = 0; + uint32_t _diag_last = 0; + + volatile uint32_t _last_activity = 0; + uint32_t _reboot_at = 0; // 0 = none scheduled + uint32_t _delete_at = 0; // deferred teardown deadline + volatile uint32_t _stats_wanted_until = 0; + uint32_t _stats_built_at = 0; + char _stats_json[1024] = {0}; + + void createServer(); + void registerRoutes(); + void drainBatch(uint32_t now); + void finalizeTeardown(); + bool checkAuth(AsyncWebServerRequest* req); + static void collectBody(AsyncWebServerRequest* req, uint8_t* data, size_t len, + size_t index, size_t total); + + void handleRoot(AsyncWebServerRequest* req); + void handleStatus(AsyncWebServerRequest* req); + void handleLogin(AsyncWebServerRequest* req); + void handleLogout(AsyncWebServerRequest* req); + void handleConfigGet(AsyncWebServerRequest* req); + void handleConfigPost(AsyncWebServerRequest* req); + void handleConfigResult(AsyncWebServerRequest* req); + void handleStats(AsyncWebServerRequest* req); + void handleScan(AsyncWebServerRequest* req); + void handlePresets(AsyncWebServerRequest* req); + void handleReboot(AsyncWebServerRequest* req); + void handlePortalExit(AsyncWebServerRequest* req); + void handleNotFound(AsyncWebServerRequest* req); +}; + +#endif // ESP_PLATFORM && WITH_MQTT_BRIDGE diff --git a/webui/index.html b/webui/index.html new file mode 100644 index 00000000..a9024350 --- /dev/null +++ b/webui/index.html @@ -0,0 +1,954 @@ + + + + + +MeshCore Config + + + +
+ + + + +
+

MeshCore

+
connecting…
+
+ +
+
+ + +
+
+

Admin login

+

Enter this node's admin password (the same one used for remote CLI admin).

+
+
+
+ +
+
+
+
+ + +
+
In the WiFi sign-in popup? It closes if your phone sleeps. + Open in your browser to keep your progress.
+
+ +
+
+

Step 1 · WiFi

+

Connect this node to your WiFi network so it can reach the MQTT servers.

+
+
+ + +
+
+ +
Leave blank for an open network. 2.4 GHz networks only.
+
+ +
Shown on the mesh and in MQTT status messages.
+
+ +
+ +
+
+

Step 2 · Radio

+

Nodes only hear each other on identical radio settings. Pick the preset your local mesh uses.

+
+ +
+
+ +
Maximum legal power varies by region — check local rules.
+
Need settings that aren't listed? Choose Keep current settings, finish setup, then fine-tune in the Advanced editor.
+
+
+ + +
+
+ +
+
+

Step 3 · MQTT

+
+ +
How this observer identifies itself in published status/packet messages. Informational only — not used for topics or authentication. Prefilled with the node name.
+
+ +
Nearest airport code, e.g. DEN. Used in topic paths.
+
+ +
Optional — 64-char hex public key of your companion node. Included in auth JWTs so services that support it can let you claim this node.
+
+ +
Optional — also included in auth JWTs for claiming this node on some services.
+

Servers

+
+
+
+ + +
+
+ +
+
+

Step 4 · Review & save

+
+
+
Saving reboots the node into normal operation. To change settings later, run start webconfig from the serial console.
+
+ + +
+
+
+ +

Advanced editor

+
+ + +
+
+ + + + +
+ +
+
+

Node

+
+
+
+
+
+
+
+

LoRa radio reboot to apply

+
Frequency, bandwidth, SF and CR must match your mesh exactly — a wrong value takes this node off the air until fixed over serial.
+
+
+
+
+
+
+
+
+
+
+
+
+
+

Advanced

+
+
+
+
+
+
CAD (listen before transmit)Channel activity detection +
+
RX boosted gainSX126x receivers only +
+
+
+
0 = off
+
+
0 = off, else 3–168
+
+
+
+ +
+
+

Identity

+
+
How this observer identifies itself in published messages; informational only. Blank = node name.
+
+
+
64-char hex public key of the owner's companion node (optional).
+
+
+
+

Publishing

+
Status messagesPeriodic node status +
+
Packet metadataDecoded packet summaries +
+
Raw packetsFull undecoded frames +
+
RX from MQTTAccept downlink packets +
+
+
+
+
+
+
+
+

Servers

+
+
+
+

Time & SNMP

+
+
Enter none to clear.
+
+
+
+
+
SNMP agentRestart required +
+
+
+
+ +
+
+

WiFi connection

+
Changing WiFi restarts the connection — this page will drop and the node reappears on the new network. Find its new IP from your router or the serial console.
+
+
+ + +
+
+
+
+
+
+

Maintenance

+
+ + +
+
+
+ +
+
+

Device

+
+
+
+

Free heap (KB)

+

Noise floor (dBm)

+
+
+

MQTT servers

+
No data yet.
+
+
+
+
+ + +
+ + + +
+ + +
+
Nearby networks + +
+
+
+ +
+ + +
+
+

Continue in your browser

+
    +
  1. Tap Done (or Cancel) in the corner of this window.
    + If asked, choose Use Without Internet / Stay connected.
    2. +
  2. Open Safari or Chrome on this device.
    3. +
  3. Go to http://192.168.4.1
    4. +
+
Stay connected to the MeshCore-Setup WiFi network. Your progress here isn't carried over — the setup restarts in the browser, where it survives the screen locking.
+

Go back

+
+
+ + +
+
+ +

Rebooting…

+

+

+
+
+ + + + From 639c07a414cc750c35953559d925c65e487d137f Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 16 Jul 2026 18:35:21 -0700 Subject: [PATCH 15/84] feat(webconfig): enhance web configuration with flood and loop settings Add new configuration options for flood traffic management and loop detection in the web interface. This includes parameters for maximum flood hops, maximum advert hops, and loop detection modes, improving the control over mesh network behavior. --- src/helpers/esp32/WebConfigServer.cpp | 9 ++++- webui/index.html | 50 ++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 41e06307..60578df1 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -26,7 +26,8 @@ static const char SECRET_SENTINEL[] = "********"; static const char* const ALLOWED_SET_KEYS[] = { // NodePrefs (radio / node) "name", "lat", "lon", "radio", "tx", "af", "rxdelay", "txdelay", - "cad", "radio.rxgain", "advert.interval", "flood.advert.interval", + "cad", "radio.rxgain", "repeat", "advert.interval", "flood.advert.interval", + "flood.max", "flood.max.advert", "flood.max.unscoped", "loop.detect", // MQTTPrefs (WiFi / MQTT / misc observer) "wifi.ssid", "wifi.pwd", "wifi.powersave", "mqtt.origin", "mqtt.iata", "mqtt.status", "mqtt.packets", "mqtt.raw", @@ -471,6 +472,12 @@ void WebConfigServer::handleConfigGet(AsyncWebServerRequest* req) { radio["txdelay"] = _prefs->tx_delay_factor; radio["cad"] = (bool)_prefs->cad_enabled; radio["rxgain"] = (bool)_prefs->rx_boosted_gain; + radio["repeat"] = !(bool)_prefs->disable_fwd; // CLI `repeat on` == disable_fwd 0 + radio["flood_max"] = _prefs->flood_max; + radio["flood_max_advert"] = _prefs->flood_max_advert; + radio["flood_max_unscoped"] = _prefs->flood_max_unscoped; + static const char* const LOOP_MODES[] = { "off", "minimal", "moderate", "strict" }; + radio["loop_detect"] = LOOP_MODES[_prefs->loop_detect <= LOOP_DETECT_STRICT ? _prefs->loop_detect : 0]; radio["name"] = (const char*)_prefs->node_name; radio["lat"] = _prefs->node_lat; radio["lon"] = _prefs->node_lon; diff --git a/webui/index.html b/webui/index.html index a9024350..4be74971 100644 --- a/webui/index.html +++ b/webui/index.html @@ -255,7 +255,8 @@ canvas{width:100%;height:56px;display:block}

Advanced

-
+
+
@@ -263,6 +264,8 @@ canvas{width:100%;height:56px;display:block}
RX boosted gainSX126x receivers only
+
RepeatForward mesh traffic. Off = listen-only (still observes and publishes). +
0 = off
@@ -270,6 +273,21 @@ canvas{width:100%;height:56px;display:block}
0 = off, else 3–168
+
+

Flooding & loops

+
Hop limits for flood traffic this node forwards. Lower values contain noisy neighbours; too low takes parts of the mesh out of reach.
+
+
+
0–64. Default 64.
+
+
0–64. Default 8.
+
+
+
Applies to packets with no region scope. 0–64.
+
+ +
Drops flood packets that already carry this node's hash. Stricter = fewer repeats tolerated before dropping.
+
@@ -284,17 +302,19 @@ canvas{width:100%;height:56px;display:block}

Publishing

-
Status messagesPeriodic node status +
This node only ever uploads to MQTT — nothing from a broker is injected into the mesh.
+
Node statusPeriodic health/stats to the status topic
-
Packet metadataDecoded packet summaries +
Publish packetsMaster switch for all packet traffic below
-
Raw packetsFull undecoded frames +
Add raw framesAlso send the full hex frame to the raw topic
-
RX from MQTTAccept downlink packets +
Received packetsReport packets heard over the air
-
-
+
+ +
Report packets this node sends.
@@ -469,6 +489,11 @@ function cfgVal(k){ // map a `set` key to its current value string, from st.cfg case"tx":return String(r.tx);case"af":return String(r.af); case"rxdelay":return String(r.rxdelay);case"txdelay":return String(r.txdelay); case"cad":return r.cad?"on":"off";case"radio.rxgain":return r.rxgain?"on":"off"; + case"repeat":return r.repeat?"on":"off"; + case"flood.max":return String(r.flood_max); + case"flood.max.advert":return String(r.flood_max_advert); + case"flood.max.unscoped":return String(r.flood_max_unscoped); + case"loop.detect":return r.loop_detect; case"advert.interval":return String(r.advert_interval); case"flood.advert.interval":return String(r.flood_advert_interval); case"wifi.ssid":return w.ssid;case"wifi.pwd":return w.pwd;case"wifi.powersave":return w.powersave; @@ -504,7 +529,7 @@ function loadConfig(){ var lab=el.closest(".f");var ch=lab&&lab.querySelector(".chip");if(ch)ch.remove(); }); $$("[data-rg]").forEach(function(el){el.value=String(c.radio[el.dataset.rg]);el.classList.remove("dirty")}); - st.orig.radio=radioCombo(); + st.orig.radio=radioCombo();afHint(); for(var i=1;i<=st.nslots;i++){refreshSlotFields($("#app-slots"),i);refreshSlotFields($("#wz-slots"),i)} updateSaveBar(); }); @@ -512,8 +537,17 @@ function loadConfig(){ function markDirty(k,v,el){ if(v===st.orig[k]){delete st.dirty[k]}else{st.dirty[k]=v} if(el)el.classList.toggle("dirty",k in st.dirty); + if(k==="af")afHint(); updateSaveBar(); } +// The CLI's `dutycycle` is not its own pref — it just writes airtime_factor as +// 100/(dc)-1. Showing the equivalent duty cycle beside `af` gives the number +// operators actually reason about without a second field fighting over one pref. +function afHint(){ + var e=$("#af-dc");if(!e)return; + var af=parseFloat($('[data-k="af"]').value); + e.textContent=isFinite(af)&&af>=0?"≈ "+(Math.round(1000/(af+1))/10)+"% duty cycle":""; +} document.addEventListener("input",function(ev){ var el=ev.target; if(el.dataset.k){ From a7c1cc631f71c4a1734f9da94cec82f57b279e3d Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 18 Jul 2026 09:34:11 -0700 Subject: [PATCH 16/84] feat(webconfig): add web configuration portal and MQTT enhancements Introduce a web configuration portal for easier node management and provisioning without serial CLI. Enhance MQTT functionality with improved IATA code validation, dynamic slot management, and background NTP synchronization. Update web UI elements for better user experience and security notes regarding open AP usage. --- MQTT_IMPLEMENTATION.md | 60 ++ WEB_CONFIG_REVIEW.md | 516 ++++++++++++++++++ examples/simple_repeater/MyMesh.h | 7 +- examples/simple_room_server/MyMesh.h | 7 +- scripts/generate_webconfig_html.py | 49 +- src/helpers/CommonCLI_Observer.cpp | 57 +- src/helpers/bridges/MQTTBridge.cpp | 74 ++- src/helpers/bridges/MQTTBridge.h | 5 + src/helpers/esp32/WebConfigServer.cpp | 129 ++++- src/helpers/esp32/WebConfigServer.h | 4 + .../HeltecTrackerV2Board.cpp | 7 + webui/index.html | 185 +++++-- 12 files changed, 983 insertions(+), 117 deletions(-) create mode 100644 WEB_CONFIG_REVIEW.md diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 5bc75004..21735931 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -474,6 +474,66 @@ These are standard MeshCore commands, not MQTT-specific, but important for obser See [MQTT_SNMP.md](MQTT_SNMP.md) for full SNMP documentation. +### Web Configuration Portal + +The observer builds include a browser-based configuration portal so a node can +be provisioned and managed without the serial CLI. It is started from the CLI +(serial or remote admin) and is never on by default on a configured node. + +#### CLI commands +- `start webconfig` — start the portal. If WiFi is already configured and + connected, it binds to the node's **LAN** IP and requires the admin password + to log in. If WiFi is **not** configured (`wifi.ssid` empty), it raises the + setup AP instead (same as first boot). +- `start webconfig ap` — force the **setup AP** even when WiFi is configured. + The MQTT bridge must be stopped first (`set bridge off`); the AP owns the + radio. Used for re-provisioning in the field. +- `stop webconfig` — stop the portal and free its resources. LAN mode runs until + this is issued; the setup AP also auto-stops after an idle timeout (default 10 + minutes with no station associated). + +#### First-boot / setup-AP behavior +On a node with no WiFi configured, the portal comes up automatically as an open +SoftAP named `MeshCore-Setup-XXXX` (last two bytes of the public key), with a +captive-portal redirect. The device display shows the AP name and portal URL +(`http://192.168.4.1/`). Walk through the wizard (WiFi → radio → MQTT → review), +then **Save & reboot**; the node reboots and joins the configured network. + +Optionally set a WPA2 password for the setup AP at build time with +`-D WEBCONFIG_AP_PASSWORD='"yourpassword"'`. + +#### Modes and authentication +- **Setup AP**: unauthenticated. Trust is based on physical proximity to the + open/PSK AP. Only the SoftAP interface serves the API — on `start webconfig + ap` the STA is explicitly disassociated so the API is **not** exposed on the + LAN the node was attached to. +- **LAN**: requires the admin password (same one used for remote CLI admin). + Sessions use a cookie with a sliding idle expiry (default 20 minutes); five + failed logins trigger a 30-second lockout. + +> **Security note:** the open setup AP transports WiFi/MQTT credentials over +> plain HTTP. Provision on a trusted, non-public frequency/location, set +> `WEBCONFIG_AP_PASSWORD` where feasible, and prefer LAN mode for ongoing +> management. The setup AP is intended for initial provisioning, not +> long-running operation. + +#### Applying changes +- **Radio** (freq/BW/SF/CR): persisted but applied only on reboot; the UI shows + a "reboot to apply" hint. +- **WiFi SSID/password**: changing these in LAN mode saves and reboots so the + node reconnects on the new network (the page will drop; find the new IP on + your router). In the setup wizard, saving always reboots. +- **MQTT publishing toggles / slot config**: applied live to the running bridge + (no reboot needed). +- **NTP server**: saved immediately; the time sync runs in the background — + verify with `get mqtt.ntp.diag`. + +#### Recovery +If provisioning fails or you're locked out of the portal, connect over USB +serial and use the CLI directly (e.g. `set wifi.ssid ...`, `set wifi.pwd ...`, +`get wifi.status`, `stop webconfig`). Serial access always works regardless of +the portal state. + ## Command Architecture The CLI commands are organized into two levels: diff --git a/WEB_CONFIG_REVIEW.md b/WEB_CONFIG_REVIEW.md new file mode 100644 index 00000000..71c7f202 --- /dev/null +++ b/WEB_CONFIG_REVIEW.md @@ -0,0 +1,516 @@ +# WebConfig Branch Review + +## Scope + +This review covers the fork-owned WebConfig and Heltec Tracker additions on the `webconfig` branch, principally commits `d7a7e1b6`, `dfee21a0`, and `639c07a4`, plus the fork-owned MQTT/CLI paths they invoke. Issues inherited unchanged from `meshcore-dev/MeshCore` are intentionally excluded. + +No implementation changes are included in this document. + +## Executive Summary + +The portal builds successfully and has a sound high-level design: HTTP handlers avoid directly running CLI/radio operations, configuration writes are marshalled to the loop task, secrets are represented by placeholders, and the UI is self-contained for offline provisioning. + +Before deployment, the most important work is: + +1. Secure setup/forced-AP reachability. +2. Correlate each save with its own result. +3. Prevent reboot after partially failed wizard saves. +4. Make persisted Wi-Fi and MQTT settings match live runtime behavior. +5. Remove cross-task preference/statistics races and long loop-task blocking. + +## Priority 1: Security and Data Integrity + +### 1. Forced AP mode exposes the unauthenticated setup API on the existing LAN + +**Severity:** High + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:125-134` +- `src/helpers/esp32/WebConfigServer.cpp:346-349` +- `src/helpers/bridges/MQTTBridge.cpp:801-866` +- `src/helpers/bridges/MQTTBridge.cpp:919-926` +- `examples/simple_repeater/MyMesh.cpp:1329-1339` +- `examples/simple_room_server/MyMesh.cpp:974-984` + +**Problem:** + +`MQTTBridge::end()` deliberately leaves the STA association connected. Forced setup then selects `WIFI_AP_STA`, starts the server on port 80, and disables authentication for all requests while `MODE_SETUP` is active. The server is therefore reachable through both the setup AP and the existing LAN connection. + +The comment that setup mode implies physical proximity is not valid in forced-AP mode. Any host on the existing LAN can read or change configuration and reboot the node without the admin password. + +**Suggested fix:** + +Choose one of these approaches: + +- Disconnect and disable STA before entering unauthenticated setup mode, leaving only the SoftAP interface active. If scanning requires STA mode, keep the interface enabled but explicitly disconnect it and prevent auto-reconnect. +- Bind the setup listener only to the SoftAP interface if the ESPAsyncWebServer/network stack supports reliable interface binding. +- Keep authentication enabled for setup-mode requests arriving through STA, while allowing unauthenticated requests only through the SoftAP interface. + +Add a hardware test that starts from an associated STA connection, stops the bridge, enters forced AP mode, and confirms that another LAN host cannot access `/api/config` or `/api/reboot` without authentication. + +### 2. Default provisioning sends credentials over an open HTTP network + +**Severity:** High + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:130-134` +- `src/helpers/esp32/WebConfigServer.cpp:346-349` +- `src/helpers/esp32/WebConfigServer.cpp:530-598` + +**Problem:** + +No reviewed observer environment defines `WEBCONFIG_AP_PASSWORD`, so setup creates an open AP. Setup requests are unauthenticated and use plain HTTP. Wi-Fi passwords, MQTT passwords, and access tokens can be captured by another nearby station. A nearby party can also provision the device before the intended operator. + +**Suggested fix:** + +- Generate a unique per-device setup password from secure random data during first boot and show it on the display or a physical label. +- Alternatively require a short-lived setup PIN displayed on-device and validated by the API before secrets can be submitted. +- Add an explicit provisioning-session expiry and invalidate the setup credential after successful setup. +- If an intentionally open AP remains supported, document the threat model prominently and avoid describing proximity as authentication. + +### 3. Save results are not correlated with the submitted batch + +**Severity:** High + +**Locations:** + +- `webui/index.html:630-680` +- `src/helpers/esp32/WebConfigServer.cpp:547-550` +- `src/helpers/esp32/WebConfigServer.cpp:601-641` + +**Problem:** + +The server keeps the last completed result readable. The frontend polls `/api/config/result` after nearly every POST failure because the POST may have reached the device even if its response was lost. There is no request or batch identity. + +A rejected, lost, or concurrent save can therefore consume a previous or different tab's result, report success, clear `st.dirty`, and overwrite unsaved values with the device configuration. + +**Suggested fix:** + +- Have the browser generate a random request ID and include it in the POST body. +- Store that ID with the batch and include it in the 202 response and every result response. +- Require the frontend to accept `pending` or `done` only when the ID matches. +- Treat definite HTTP responses such as 400, 409, and 413 as final rejection; only poll after an ambiguous network failure. +- Return the active request ID with 409 so the browser can distinguish its own retry from another client's batch. + +Add regression tests for stale `DONE`, two tabs, 409, lost 202, lost result response, and a POST that never reached the server. + +### 4. Failed wizard commands still cause partial application and reboot + +**Severity:** High + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:265-297` +- `src/helpers/esp32/WebConfigServer.cpp:624-637` +- `webui/index.html:950-957` + +**Problem:** + +Commands are persisted independently. Error replies are recorded, but `_batch_reboot` still schedules a fallback reboot, and reading the result arms the three-second reboot without checking aggregate success. The wizard can display rejected settings while the device reboots with a partially applied configuration. + +**Suggested fix:** + +- Track aggregate batch success while draining commands. +- Arm automatic reboot only if every required command succeeded. +- If partial persistence cannot be rolled back, return an explicit `partial` state listing applied and rejected keys. +- Keep the portal active after partial failure and present deliberate choices: correct and retry, or reboot with the partial configuration. +- Longer term, validate all values before executing any setter, or stage a complete preference snapshot and commit it atomically. + +## Priority 2: Runtime Correctness + +### 5. LAN Wi-Fi edits do not perform the reconnect promised by the UI + +**Severity:** High + +**Locations:** + +- `webui/index.html:339-350` +- `webui/index.html:630-641` +- `src/helpers/CommonCLI_Observer.cpp:278-285` + +**Problem:** + +The Wi-Fi tab says the connection will restart and the page will drop. Normal editor saves do not request reboot, and the underlying setters only persist the SSID/password. The active connection remains on the old network, while subsequent config reads show the new persisted values. + +**Suggested fix:** + +Use an explicit `Save and reconnect` or `Save and reboot` flow for SSID/password changes. Deliver the result first, then schedule the reconnect/reboot. Do not replace the displayed active network with the persisted network until the transition has begun successfully. + +### 6. MQTT publishing controls report success without changing live behavior + +**Severity:** High + +**Locations:** + +- `webui/index.html:303-319` +- `src/helpers/CommonCLI_Observer.cpp:215-238` +- `src/helpers/bridges/MQTTBridge.cpp:637-645` + +**Problem:** + +Status, packet, raw, RX, and TX options are copied into cached bridge fields during initialization. Their CLI setters persist preferences but do not refresh those fields or restart the bridge. The UI reports success although the running bridge continues using old values. + +**Suggested fix:** + +- Add one task-safe bridge method that reloads the publishing flags from preferences on the MQTT task, or +- coalesce these changes into one full bridge restart after the batch. + +If live application is not desirable, label these controls as restart-required and provide a reboot action instead of claiming immediate success. + +### 7. Custom MQTT endpoint edits do not reliably refresh the live slot + +**Severity:** High + +**Locations:** + +- `src/helpers/CommonCLI_Observer.cpp:385-425` +- `examples/simple_repeater/MyMesh.cpp:1354-1367` +- `examples/simple_room_server/MyMesh.cpp:999-1012` +- `src/helpers/bridges/MQTTBridge.cpp:1066-1072` +- `src/helpers/bridges/MQTTBridge.cpp:2076-2095` + +**Problem:** + +Custom server and port setters persist without requesting slot reconfiguration. Credential changes request reconfiguration, but the custom branch of `applySlotPreset()` reuses the existing slot fields instead of copying the current host, port, username, and password from preferences. + +**Suggested fix:** + +Create a single `reloadSlotFromPrefs(slot)` operation executed on the MQTT task. It should tear down the slot, copy every custom field and preset-dependent credential from `MQTTPrefs`, validate the complete endpoint, and then reconnect. Every slot-affecting CLI setter should queue that same operation rather than implementing partial restart behavior. + +### 8. Wizard review and validation mishandle intentionally cleared values + +**Severity:** Medium + +**Locations:** + +- `webui/index.html:907-929` +- `webui/index.html:931-933` + +**Problem:** + +Review calculations use `dirtyValue || originalValue`. An intentional empty string is treated as absent, so Review shows the original SSID/password/name/identity/slot while the submitted batch clears it. Final SSID validation can pass using the old SSID even though the effective new value is empty. + +**Suggested fix:** + +Resolve values by key presence, not truthiness. Add a helper such as `effectiveValue(key)` that returns `st.dirty[key]` when the key exists in `st.dirty`, including `""`, and otherwise returns `st.orig[key]`. Use it for review, validation, and reboot messaging. + +### 9. A credential equal to `********` cannot be configured + +**Severity:** Low + +**Locations:** + +- `webui/index.html:422` +- `webui/index.html:527-538` +- `src/helpers/esp32/WebConfigServer.cpp:564` + +**Problem:** + +The secret sentinel is also a valid possible password/token. The UI either considers it unchanged or the backend silently drops it. + +**Suggested fix:** + +Track secret-field edit state separately from the displayed placeholder. Prefer an empty password control with an adjacent “stored credential unchanged” indicator. If retaining the sentinel, reject that exact value with a clear validation message rather than silently ignoring it. + +## Priority 3: Concurrency, Responsiveness, and Lifecycle + +### 10. The preference mutex does not synchronize reads with writes + +**Severity:** Medium + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:454-523` +- `src/helpers/esp32/WebConfigServer.cpp:265-280` +- `src/helpers/CommonCLI.cpp:1074-1216` +- `src/helpers/CommonCLI_Observer.cpp:198-486` + +**Problem:** + +`handleConfigGet()` takes `_mux` while reading preferences, but `drainBatch()` invokes CLI setters outside `_mux`. Those setters mutate the same strings and scalar fields. The lock therefore does not protect the data from concurrent async HTTP reads. + +**Suggested fix:** + +Prefer loop-task ownership: have the HTTP handler request a configuration snapshot, let `tick()` build it on the loop task, and return the immutable snapshot. If retaining direct reads, every writer must take the same mutex, with careful review to avoid holding it through flash writes or callbacks. + +### 11. Changing `mqtt.ntp` can block the main loop for up to 30 seconds + +**Severity:** Medium + +**Locations:** + +- `src/helpers/CommonCLI_Observer.cpp:249-272` +- `src/helpers/bridges/MQTTBridge.cpp:3230-3247` +- `src/helpers/esp32/WebConfigServer.cpp:275-280` + +**Problem:** + +Web batches run from `tick()` on the Arduino loop task. The NTP setter waits for the MQTT task in a polling loop that can last 30 seconds. During that period mesh/radio processing, portal DNS, further batch work, stats, and reboot timers stop progressing. + +**Suggested fix:** + +Persist and validate hostname syntax synchronously, then queue NTP synchronization without waiting. Represent NTP validation as a separate asynchronous operation/status result. The config batch should finish immediately and the UI can poll NTP validation independently. + +### 12. MQTT status snapshots read mutable cross-core state without synchronization + +**Severity:** Medium + +**Locations:** + +- `src/helpers/bridges/MQTTBridge.cpp:295-320` +- `examples/simple_repeater/MyMesh.cpp:1403-1408` +- `examples/simple_room_server/MyMesh.cpp:1048-1053` + +**Problem:** + +The loop task reads slot state and client counters while the MQTT task and callbacks mutate them. This can yield inconsistent snapshots and formal C++ data races. + +**Suggested fix:** + +Build a plain-data `SlotStatusSnapshot` array on the MQTT task and publish it atomically or under a shared lock. The web/loop side should read only the copied snapshot and should not call client methods cross-task. + +### 13. Fixed-delay server deletion does not prove connections have ended + +**Severity:** Medium, hardware/library stress-test required + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:188-204` + +**Problem:** + +Stopping the listener and waiting two seconds does not establish that accepted slow or stalled clients have completed. Deleting the server and route lambdas while a request remains active risks use-after-free or crashes. + +**Suggested fix:** + +Use connection/request reference counting and finalize deletion only after all accepted requests disconnect, with an upper-bound recovery policy. If the library cannot expose lifecycle state safely, consider retaining one server instance for the firmware lifetime and enabling/disabling routes/listening without deleting captured handler state. + +### 14. Repeated starts permanently grow the global default-header list + +**Severity:** Medium + +**Location:** + +- `src/helpers/esp32/WebConfigServer.cpp:177-185` + +**Problem:** + +`DefaultHeaders::Instance().addHeader()` appends to a process-lifetime list on every server creation. Repeated start/stop cycles consume heap and add duplicate `Cache-Control` headers to every response. + +**Suggested fix:** + +Register the global header once through a static one-time guard, or avoid global headers and add `Cache-Control: no-store` to each WebConfig response. + +### 15. Stats polling can accumulate overlapping requests + +**Severity:** Low + +**Locations:** + +- `webui/index.html:717-747` + +**Problem:** + +A three-second `setInterval` starts a new request even if the previous fetch is still pending. Degraded Wi-Fi can accumulate requests and pressure both browser and ESP32 memory. + +**Suggested fix:** + +Schedule the next poll with `setTimeout` only after the current request settles. Add an in-flight guard and use the existing API timeout support. + +## Priority 4: Validation, Build, and Maintainability + +### 16. Malformed keys can cause out-of-bounds reads and invalid JSON errors + +**Severity:** Low + +**Locations:** + +- `src/helpers/esp32/WebConfigServer.cpp:41-59` +- `src/helpers/esp32/WebConfigServer.cpp:554-562` + +**Problem:** + +Short attacker-supplied keys are indexed at positions 4 and 5 without first establishing their length. Rejected keys are interpolated into hand-built JSON without escaping quotes or backslashes. + +**Suggested fix:** + +Check the key length before `memcmp` and indexed access. Construct error responses with ArduinoJson rather than string interpolation. + +### 17. Input lengths do not match fixed firmware buffers + +**Severity:** Low + +**Locations:** + +- `webui/index.html` configuration controls +- `src/helpers/CommonCLI.h:106-159` + +**Problem:** + +Several controls accept more text than the fixed `MQTTPrefs` fields retain. The CLI truncates values while the UI reports success. + +**Suggested fix:** + +Add `maxlength` values matching each destination buffer minus the NUL terminator, and validate lengths in the backend because client-side constraints are bypassable. Return a clear error instead of silently truncating. + +### 18. Generated HTML freshness relies only on timestamps + +**Severity:** Low + +**Location:** + +- `scripts/generate_webconfig_html.py:30-38` + +**Problem:** + +A generated header with a future timestamp can survive later source edits and embed stale UI code. + +**Suggested fix:** + +Use an explicit SCons source/target dependency or store the source hash in the generated header and regenerate whenever it differs. Deterministic gzip output can remain unchanged. + +### 19. Tracker v1.1 reports itself as Tracker V2 + +**Severity:** Medium + +**Locations:** + +- `boards/heltec_tracker_v1_1.json:19,28` +- `variants/heltec_tracker_v2/platformio.ini:61-109` +- `variants/heltec_tracker_v2/HeltecTrackerV2Board.cpp:82-84` + +**Problem:** + +The v1.1 environment reuses the V2 board implementation, whose manufacturer name is hardcoded as `Heltec Tracker V2`. WebConfig and MQTT metadata therefore report the wrong board. + +**Suggested fix:** + +Return `Heltec Tracker V1.1` when `HELTEC_TRACKER_V1_1` is defined, and retain the V2 value otherwise. Add a build-time or host-side assertion for both target identities. + +### 20. WebConfig operation and security behavior are undocumented + +**Severity:** Documentation gap + +**Locations:** + +- Commands added in `src/helpers/CommonCLI_Observer.cpp:982-994` +- `MQTT_IMPLEMENTATION.md` + +**Problem:** + +The build targets are documented, but operators cannot discover `start webconfig`, `start webconfig ap`, `stop webconfig`, first-boot AP behavior, authentication, timeout, or the security implications of setup mode. + +**Suggested fix:** + +Add an operator section to `MQTT_IMPLEMENTATION.md` covering: + +- first-boot setup behavior; +- AP name and setup credential; +- LAN versus AP modes; +- exact CLI commands; +- authentication requirements; +- idle and absolute timeout behavior; +- how Wi-Fi changes are applied; +- how to recover through serial if provisioning fails. + +### 21. Repeater and room-server integration is duplicated + +**Severity:** Optimization + +**Locations:** + +- `examples/simple_repeater/MyMesh.*` +- `examples/simple_room_server/MyMesh.*` +- Corresponding `UITask.cpp` files + +**Problem:** + +Lifecycle, restart coalescing, stats construction, and display behavior are implemented twice. Fixes can easily land in only one role. + +**Suggested fix:** + +Extract shared WebConfig callbacks, lifecycle ownership, and JSON snapshot helpers into a common observer helper. Keep role-specific radio/stat sources as injected callbacks. + +### 22. The HTML generator runs for every ESP32 build + +**Severity:** Optimization + +**Locations:** + +- `platformio.ini:57-72` + +**Problem:** + +The generator runs from `esp32_base` even when WebConfig is not compiled into the target. + +**Suggested fix:** + +Move the pre-script to MQTT observer environments or have the script inspect build flags and return immediately unless `WITH_MQTT_BRIDGE`/WebConfig is enabled. + +### 23. New observer targets enable MQTT debug logging + +**Severity:** Optimization + +**Locations:** + +- `variants/heltec_tracker_v2/platformio.ini:190,226,262,298` + +**Problem:** + +`MQTT_DEBUG=1` increases serial activity and code/logging overhead in targets that otherwise appear intended for deployment. + +**Suggested fix:** + +Remove it from production environments or create explicit debug variants. Confirm that no diagnostic output includes credentials or tokens. + +## Test Recommendations + +No WebConfig-specific automated tests were found. Add focused tests for: + +1. Save request/result correlation, including stale and concurrent batches. +2. Partial command failures and reboot gating. +3. Effective-value handling when fields are intentionally cleared. +4. Secret placeholder/edit semantics. +5. CLI parsing and backend length validation. +6. Runtime application of Wi-Fi and cached MQTT settings. +7. Complete custom-slot reconfiguration. +8. NTP updates without loop-task blocking. +9. Config and MQTT status snapshots under concurrent updates. +10. Repeated server start/stop heap behavior and duplicate headers. +11. Forced AP reachability from both SoftAP and STA networks. +12. Setup AP absolute expiry with an idle associated station. +13. Correct board identity for Tracker v1.1 and V2 targets. + +## Verification Already Performed + +The review ran the following checks successfully: + +- `git diff --check` +- Extracted JavaScript with `node --check` +- PlatformIO builds: + - `heltec_tracker_v1_1_repeater_observer_mqtt` + - `heltec_tracker_v2_repeater_observer_mqtt` + - `heltec_tracker_v1_1_room_server_observer_mqtt` + - `heltec_tracker_v2_room_server_observer_mqtt` + +Observed static usage: + +- Repeater: 76,960 bytes RAM (23.5%), approximately 47.4% flash. +- Room server: 80,656 bytes RAM (24.6%), approximately 47.3% flash. + +Successful compilation does not validate the AP/STA security boundary, async teardown, concurrency, reconnection, or partial-save behavior; those require the targeted host/hardware tests above. + +## Suggested Implementation Order + +1. Fix forced-AP LAN exposure and define secure provisioning authentication. +2. Introduce request IDs for save/result correlation. +3. Gate reboot on aggregate success and represent partial application explicitly. +4. Correct effective empty-value handling in the wizard. +5. Align Wi-Fi, MQTT flags, and custom-slot runtime behavior with the UI. +6. Remove NTP blocking and marshal config/status snapshots to their owning tasks. +7. Fix server lifecycle/global-header accumulation. +8. Add input-length/backend validation and polling safeguards. +9. Fix Tracker v1.1 identity and add operator documentation. +10. Add automated tests, then run the full hardware matrix. diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 127ad7d1..ee6b935d 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -363,8 +363,11 @@ public: bool syncMqttNtp() override { if (!bridge || !bridge->isRunning()) return false; - // Marshal onto the MQTT task (Core 0); this runs on the CLI thread (Core 1). - return bridge->requestForcedNtpSync(); + // Queue the sync onto the MQTT task (Core 0) without blocking: this runs on + // the Arduino loop task (serial CLI and the web config batch both drain + // here), and blocking up to 30 s would stall mesh/radio forwarding. Returns + // true once queued; verify with `get mqtt.ntp.diag`. + return bridge->requestForcedNtpSync(0); } bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) override { diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 466fb09d..41377965 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -338,8 +338,11 @@ public: bool syncMqttNtp() override { if (!bridge || !bridge->isRunning()) return false; - // Marshal onto the MQTT task (Core 0); this runs on the CLI thread (Core 1). - return bridge->requestForcedNtpSync(); + // Queue the sync onto the MQTT task (Core 0) without blocking: this runs on + // the Arduino loop task (serial CLI and the web config batch both drain + // here), and blocking up to 30 s would stall mesh/radio forwarding. Returns + // true once queued; verify with `get mqtt.ntp.diag`. + return bridge->requestForcedNtpSync(0); } bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) override { diff --git a/scripts/generate_webconfig_html.py b/scripts/generate_webconfig_html.py index d587fa5d..143687d3 100644 --- a/scripts/generate_webconfig_html.py +++ b/scripts/generate_webconfig_html.py @@ -3,7 +3,16 @@ # Pre-build script: gzip webui/index.html into a PROGMEM C header so the # webconfig portal can serve the page straight from flash with # Content-Encoding: gzip. The generated header is .gitignored; this script -# regenerates it whenever the source page (or this script) is newer. +# regenerates it whenever the source page (or this script) content changes. +# +# Freshness is decided by a content hash of the build inputs (not timestamps): +# a generated header written with a skewed/future mtime would otherwise mask a +# later source edit and embed stale UI. gzip output is deterministic (mtime=0), +# so an unchanged source produces an unchanged header. +# +# The generator is idempotent and cheap: when the inputs are unchanged it only +# hashes two small files and returns, so running it from esp32_base on every +# ESP32 build is negligible even for targets that don't compile the portal. # # Output: src/helpers/esp32/WebConfigHtml.h # WEBCONFIG_HTML_GZ[] - gzipped page (PROGMEM) @@ -21,21 +30,37 @@ SOURCE = os.path.join("webui", "index.html") OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h") # __file__ is not defined inside PIO/SCons-executed extra_scripts SCRIPT = os.path.join("scripts", "generate_webconfig_html.py") +HASH_MARKER = "// build-inputs-sha256: " def status(msg): sys.stderr.write("WebConfig HTML: %s\n" % msg) -def needs_rebuild(): - if not os.path.isfile(OUTPUT): - return True - out_mtime = os.path.getmtime(OUTPUT) - if os.path.getmtime(SOURCE) > out_mtime: - return True - if os.path.isfile(SCRIPT) and os.path.getmtime(SCRIPT) > out_mtime: - return True - return False +def content_hash(): + # Hash the source page and this generator so any change to either forces a + # regenerate, independent of file timestamps. + h = hashlib.sha256() + with open(SOURCE, "rb") as f: + h.update(f.read()) + if os.path.isfile(SCRIPT): + h.update(b"\0") + with open(SCRIPT, "rb") as f: + h.update(f.read()) + return h.hexdigest() + + +def stored_hash(): + try: + with open(OUTPUT, "r") as f: + for line in f: + if line.startswith(HASH_MARKER): + return line[len(HASH_MARKER):].strip() + if line.startswith("#"): # reached the C preprocessor lines + break + except OSError: + return None + return None def main(): @@ -43,7 +68,8 @@ def main(): status("ERROR: %s not found" % SOURCE) sys.exit(2) - if not needs_rebuild(): + src_hash = content_hash() + if os.path.isfile(OUTPUT) and stored_hash() == src_hash: return with open(SOURCE, "rb") as f: @@ -56,6 +82,7 @@ def main(): lines = [] lines.append("// Auto-generated by scripts/generate_webconfig_html.py from %s" % SOURCE.replace(os.sep, "/")) lines.append("// DO NOT EDIT - edit webui/index.html instead.") + lines.append("%s%s" % (HASH_MARKER, src_hash)) lines.append("#pragma once") lines.append("#include ") lines.append("#include ") diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index 75d14d48..ab724717 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -205,13 +205,35 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf savePrefs(); strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.iata ", 10) == 0) { - StrHelper::strncpy(_mqtt_prefs.mqtt_iata, &config[10], sizeof(_mqtt_prefs.mqtt_iata)); - for (int i = 0; _mqtt_prefs.mqtt_iata[i]; i++) { - _mqtt_prefs.mqtt_iata[i] = toupper(_mqtt_prefs.mqtt_iata[i]); + const char* iata = &config[10]; + size_t iata_len = strlen(iata); + if (iata_len == 0) { + // Empty clears the region code (meshcore-topic publishing stays disabled + // until one is set). This keeps the pre-existing "clear IATA" capability. + _mqtt_prefs.mqtt_iata[0] = '\0'; + savePrefs(); + _callbacks->restartBridge(); + strcpy(reply, "OK - IATA cleared"); + } else { + // A region code goes straight into MQTT topic paths, so require exactly + // three alphanumeric characters (real IATA codes are 3 letters, e.g. DEN). + bool valid = (iata_len == 3); + for (size_t i = 0; valid && i < iata_len; i++) { + char c = iata[i]; + valid = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + } + if (!valid) { + strcpy(reply, "Error: IATA code must be exactly 3 letters/digits (e.g. DEN)"); + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_iata, iata, sizeof(_mqtt_prefs.mqtt_iata)); + for (int i = 0; _mqtt_prefs.mqtt_iata[i]; i++) { + _mqtt_prefs.mqtt_iata[i] = toupper(_mqtt_prefs.mqtt_iata[i]); + } + savePrefs(); + _callbacks->restartBridge(); + strcpy(reply, "OK"); + } } - savePrefs(); - _callbacks->restartBridge(); - strcpy(reply, "OK"); } else if (memcmp(config, "mqtt.status ", 12) == 0) { _mqtt_prefs.mqtt_status_enabled = memcmp(&config[12], "on", 2) == 0; savePrefs(); @@ -260,16 +282,18 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } savePrefs(); #ifdef ESP_PLATFORM - // Validate by running an immediate sync. syncMqttNtp() marshals onto the MQTT - // task (Core 0) so no NTP I/O happens on this (Core 1) CLI thread. + // Queue a sync on the MQTT task (Core 0) but do NOT block: this handler + // runs on the Arduino loop task, shared with mesh/radio processing and the + // web config batch, so a synchronous wait of up to 30 s would stall the + // node. The sync runs in the background; verify with `get mqtt.ntp.diag`. if (WiFi.status() != WL_CONNECTED) { strcpy(reply, "OK - saved (WiFi not connected; NTP sync pending)"); } else if (!_callbacks->isMqttBridgeRunning()) { strcpy(reply, "OK - saved (MQTT bridge not running)"); } else if (_callbacks->syncMqttNtp()) { - strcpy(reply, "OK - time synced"); + strcpy(reply, "OK - saved (NTP sync started; check 'get mqtt.ntp.diag')"); } else { - strcpy(reply, "Error: NTP sync failed"); + strcpy(reply, "OK - saved (NTP sync unavailable)"); } #else strcpy(reply, "OK - saved"); @@ -385,12 +409,17 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(subcmd, "server ", 7) == 0) { StrHelper::strncpy(_mqtt_prefs.mqtt_slot_host[slot], &subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot])); savePrefs(); + // Reconfigure the slot so the new host reaches the live connection (other + // custom-slot setters do the same; without it the change only applies on + // the next reboot/bridge restart). + _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else if (memcmp(subcmd, "port ", 5) == 0) { int port = atoi(&subcmd[5]); if (port > 0 && port <= 65535) { _mqtt_prefs.mqtt_slot_port[slot] = port; savePrefs(); + _callbacks->restartBridgeSlot(slot); strcpy(reply, "OK"); } else { strcpy(reply, "Error: port must be between 1 and 65535"); @@ -460,7 +489,13 @@ bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* conf } else if (memcmp(config, "mqtt.owner ", 11) == 0) { const char* owner_key = &config[11]; int key_len = strlen(owner_key); - if (key_len == 64) { + if (key_len == 0) { + // Owner key is optional — empty clears it (previously this errored, so a + // set key could never be removed via the portal/CLI). + _mqtt_prefs.mqtt_owner_public_key[0] = '\0'; + savePrefs(); + strcpy(reply, "OK - owner key cleared"); + } else if (key_len == 64) { bool valid_key = true; for (int i = 0; i < key_len; i++) { if (!((owner_key[i] >= '0' && owner_key[i] <= '9') || diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index c7e7e66d..0bed6211 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -320,6 +320,17 @@ bool MQTTBridge::getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out) return true; } +int MQTTBridge::getMaxActiveSlots() { + // Each WSS/TLS connection needs ~40KB for mbedTLS buffers. Without PSRAM even + // 3 concurrent connections would exhaust internal heap, so cap at 2; with + // PSRAM cap at 5 (6 configurable but 5 active max). +#if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) + return psramFound() ? 5 : 2; +#else + return 2; +#endif +} + uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } @@ -605,15 +616,8 @@ void MQTTBridge::begin() { MQTT_DEBUG_PRINTLN("PSRAM: not configured for this board (no BOARD_HAS_PSRAM)"); #endif - // Limit active slots based on available memory. - // Each WSS/TLS connection needs ~40KB for mbedTLS buffers. - // Without PSRAM, even 3 concurrent connections would exhaust internal heap. - // With PSRAM, cap at 5 for safety (6 configurable but 5 active max). - #if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) - _max_active_slots = psramFound() ? 5 : 2; - #else - _max_active_slots = 2; - #endif + // Limit active slots based on available memory (see getMaxActiveSlots()). + _max_active_slots = getMaxActiveSlots(); MQTT_DEBUG_PRINTLN("Max active slots: %d", _max_active_slots); // Check if WiFi credentials are configured first @@ -634,7 +638,10 @@ void MQTTBridge::begin() { _iata[i] = toupper(_iata[i]); } - // Update enabled flags from preferences + // Initial snapshot of the publish toggles. NOTE: the publish hot paths read + // these live from _obs->mqtt_* (status/packets/raw/rx/tx) so a CLI/web `set` + // takes effect without a bridge restart; these members are kept only for + // startup logging/back-compat and are not the source of truth. _status_enabled = _obs->mqtt_status_enabled; _packets_enabled = _obs->mqtt_packets_enabled; _raw_enabled = _obs->mqtt_raw_enabled; @@ -1107,8 +1114,10 @@ void MQTTBridge::mqttTaskLoop() { refreshNTP(); } - // Publish status updates (handle millis() overflow correctly) - if (_status_enabled) { + // Publish status updates (handle millis() overflow correctly). + // Read the toggle live from prefs (like mqtt.packets/rx/tx below) so a + // CLI/web `set mqtt.status` change applies without a bridge restart. + if (_obs->mqtt_status_enabled) { bool has_destinations = _cached_has_connected_slots; // Early exit if no destinations - skip all the expensive logic below @@ -2086,10 +2095,24 @@ void MQTTBridge::applySlotPreset(int slot_index, const char* preset_name) { } if (strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0) { - slot.enabled = true; slot.preset = nullptr; - // Custom broker settings should already be set via setSlotCustomBroker - if (_initialized && customEndpointComplete(slot.host, slot.port)) { + // Re-sync every custom field from prefs (same copy begin() does at startup) + // so a CLI/web edit to the host, port, credentials, or JWT audience is + // actually picked up on reconfigure. Previously this branch reused the + // stale slot fields, so e.g. changing mqttN.server or mqttN.username had no + // effect on the live connection. Token and topic are read live from _obs in + // setupSlot()/buildTopicForSlot(), so they don't need copying here. + strncpy(slot.host, _obs->mqtt_slot_host[slot_index], sizeof(slot.host) - 1); + slot.host[sizeof(slot.host) - 1] = '\0'; + slot.port = _obs->mqtt_slot_port[slot_index]; + strncpy(slot.username, _obs->mqtt_slot_username[slot_index], sizeof(slot.username) - 1); + slot.username[sizeof(slot.username) - 1] = '\0'; + strncpy(slot.password, _obs->mqtt_slot_password[slot_index], sizeof(slot.password) - 1); + slot.password[sizeof(slot.password) - 1] = '\0'; + strncpy(slot.audience, _obs->mqtt_slot_audience[slot_index], sizeof(slot.audience) - 1); + slot.audience[sizeof(slot.audience) - 1] = '\0'; + slot.enabled = (slot.host[0] != '\0'); + if (_initialized && slot.enabled && customEndpointComplete(slot.host, slot.port)) { setupSlot(slot_index); } return; @@ -2336,8 +2359,10 @@ void MQTTBridge::loop() { refreshNTP(); } - // Publish status updates (handle millis() overflow correctly) - if (_status_enabled) { + // Publish status updates (handle millis() overflow correctly). + // Read the toggle live from prefs so a CLI/web `set mqtt.status` change + // applies without a bridge restart. + if (_obs->mqtt_status_enabled) { bool has_destinations = _cached_has_connected_slots; if (has_destinations) { @@ -2522,9 +2547,10 @@ void MQTTBridge::processPacketQueue() { queued.snr, queued.rssi); taskYIELD(); // allow higher-priority tasks to run between packet publishes - // Publish raw if enabled + // Publish raw if enabled (live from prefs so `set mqtt.raw` applies without + // a bridge restart) bool raw_published = false; - if (_raw_enabled) { + if (_obs->mqtt_raw_enabled) { raw_published = publishRaw(&queued.packet_copy); } @@ -2623,8 +2649,9 @@ void MQTTBridge::processPacketQueue() { queued.snr, queued.rssi); // No taskYIELD() on non-ESP32 platforms (non-FreeRTOS, cooperative scheduling not needed) + // Live from prefs so `set mqtt.raw` applies without a bridge restart. bool raw_published = false; - if (_raw_enabled) { + if (_obs->mqtt_raw_enabled) { raw_published = publishRaw(&queued.packet_copy); } @@ -3236,6 +3263,13 @@ bool MQTTBridge::requestForcedNtpSync(uint32_t timeout_ms) { _ntp_force_result = false; _ntp_force_requested = true; + // Fire-and-forget: callers on the Arduino loop task (web config batch, and + // the CLI which shares that task) must not block up to 30 s polling the MQTT + // task — that stalls mesh/radio forwarding, portal DNS, and reboot timers. + // The task still performs the sync; the result is observable via + // `get mqtt.ntp.diag`. Blocking callers pass a non-zero timeout. + if (timeout_ms == 0) return true; + unsigned long start = millis(); while (!_ntp_force_done) { if (millis() - start >= timeout_ms) { diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index ca065b6c..fee9845d 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -446,6 +446,11 @@ public: bool isSlotEnabledAndAttempted(int slot_index) const; const char* getSlotPresetName(int slot_index) const; static int getRuntimeSlotCount() { return RUNTIME_MQTT_SLOTS; } + /** Max slots that can be connected at once: 5 with PSRAM, 2 without (each + * WSS/TLS connection needs ~40KB for mbedTLS buffers). This is the number of + * usefully-configurable servers; RUNTIME_MQTT_SLOTS carries a spare for + * reconfiguration. Safe to call before begin(). */ + static int getMaxActiveSlots(); /** Resolved origin for MQTT JSON: node_name when mqtt_origin is empty, else mqtt_origin (with quote stripping). */ static void getEffectiveMqttOrigin(const NodePrefs* np, const MQTTPrefs* obs, char* buf, size_t buf_size); static const char* effectiveNtpPrimary(const MQTTPrefs* obs); diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 60578df1..f7ee3354 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -38,13 +38,21 @@ static const char* const ALLOWED_SLOT_KEYS[] = { "preset", "server", "port", "username", "password", "token", "topic", "audience", }; +// Shortest slot key is "mqttN.x" (mqtt + digit + '.' + 1-char field) = 7 chars. +// The prefix probe below indexes key[4..6], so it must never run on a shorter +// string — an attacker-supplied "mqtt" or "m" would otherwise read past the +// terminator. strcmp() is null-safe, so the exact-match loop needs no guard. +static bool isSlotKeyPrefix(const char* key) { + return strlen(key) >= 7 && memcmp(key, "mqtt", 4) == 0 + && key[4] >= '1' && key[4] <= ('0' + MAX_MQTT_SLOTS) && key[5] == '.'; +} + static bool isAllowedSetKey(const char* key) { for (size_t i = 0; i < sizeof(ALLOWED_SET_KEYS) / sizeof(ALLOWED_SET_KEYS[0]); i++) { if (strcmp(key, ALLOWED_SET_KEYS[i]) == 0) return true; } // mqtt<1-6>. - if (memcmp(key, "mqtt", 4) == 0 && key[4] >= '1' && key[4] <= ('0' + MAX_MQTT_SLOTS) - && key[5] == '.') { + if (isSlotKeyPrefix(key)) { for (size_t i = 0; i < sizeof(ALLOWED_SLOT_KEYS) / sizeof(ALLOWED_SLOT_KEYS[0]); i++) { if (strcmp(&key[6], ALLOWED_SLOT_KEYS[i]) == 0) return true; } @@ -54,7 +62,7 @@ static bool isAllowedSetKey(const char* key) { static bool isSecretKey(const char* key) { if (strcmp(key, "wifi.pwd") == 0) return true; - if (memcmp(key, "mqtt", 4) == 0 && key[5] == '.' + if (isSlotKeyPrefix(key) && (strcmp(&key[6], "password") == 0 || strcmp(&key[6], "token") == 0)) return true; return false; } @@ -126,6 +134,15 @@ bool WebConfigServer::startSetupMode(char reply[]) { // the AP is up. STA stays unconnected - the bridge won't touch WiFi // while wifi_ssid is empty, and `start webconfig ap` requires it stopped. WiFi.mode(WIFI_AP_STA); + // Setup mode serves an UNAUTHENTICATED API on the trust of physical AP + // proximity. `start webconfig ap` can be run with the STA still associated + // to the operator's LAN (MQTTBridge::end() leaves the STA link up and + // auto-reconnect on), which would expose that open API to every host on the + // LAN. Drop the association and disable auto-reconnect so only the SoftAP + // interface is reachable; the STA interface itself stays up (unassociated) + // purely so WiFi.scanNetworks() below can populate the SSID picker. + WiFi.setAutoReconnect(false); + WiFi.disconnect(false /*keep radio on*/, true /*erase stored AP so it can't reconnect*/); snprintf(_ap_ssid, sizeof(_ap_ssid), "MeshCore-Setup-%02X%02X", _pub_key[0], _pub_key[1]); #ifdef WEBCONFIG_AP_PASSWORD bool ap_ok = WiFi.softAP(_ap_ssid, WEBCONFIG_AP_PASSWORD); @@ -180,7 +197,16 @@ void WebConfigServer::createServer() { // device reflash behind the same IP), which poisons /api/config/result and // friends with stale responses from earlier sessions. Forbid caching on // every response; the HTML is small enough to refetch per visit. - DefaultHeaders::Instance().addHeader("Cache-Control", "no-store"); + // + // DefaultHeaders::Instance() is a process-lifetime singleton and addHeader() + // appends unconditionally, so registering here on every start/stop cycle + // would leak heap and stack duplicate "Cache-Control" headers on every + // response. Register exactly once for the firmware lifetime. + static bool s_default_headers_registered = false; + if (!s_default_headers_registered) { + DefaultHeaders::Instance().addHeader("Cache-Control", "no-store"); + s_default_headers_registered = true; + } registerRoutes(); _server->begin(); } @@ -276,8 +302,19 @@ void WebConfigServer::drainBatch(uint32_t now) { BatchEntry& e = _batch[_batch_next++]; e.reply[0] = 0; uint32_t t0 = millis(); - _cb->execCommand(e.cmd, e.reply); - if (e.reply[0] == 0) strcpy(e.reply, "OK"); + // Hold _mux across the setter: it mutates the same _prefs/_obs strings that + // handleConfigGet() serializes on the async_tcp task, so without the lock + // the "read" side is unprotected and a GET can observe a half-written value. + // One command per tick keeps the hold brief; the flash write inside stalls + // WiFi regardless, so a concurrent GET waiting on it costs nothing extra. + { + WCLock lock(_mux); + _cb->execCommand(e.cmd, e.reply); + if (e.reply[0] == 0) strcpy(e.reply, "OK"); + // Success convention across every allowlisted setter is an "OK" prefix + // (the UI relies on the same test); anything else is a rejection. + if (strncmp(e.reply, "OK", 2) != 0) _batch_all_ok = false; + } _batch_last_cmd = millis(); Serial.printf("WC: cmd %d/%d '%s' took %lums\n", (int)_batch_next, (int)_batch_count, e.key, (unsigned long)(_batch_last_cmd - t0)); @@ -286,12 +323,13 @@ void WebConfigServer::drainBatch(uint32_t now) { _cb->onConfigBatchEnd(); WCLock lock(_mux); _batch_state = BATCH_DONE; - if (_batch_reboot) { - // Fallback only: the real 3 s reboot timer is armed when the client reads - // /api/config/result (handleConfigResult), so the browser gets its - // confirmation before the AP/WiFi drops. This covers a client that - // disconnected and never polls — generous enough for a phone that got - // bounced off the AP mid-save to rejoin and fetch its confirmation. + if (_batch_reboot && _batch_all_ok) { + // Fallback only, and only when every command succeeded: rebooting into a + // partially-applied config would strand the node. The real 3 s reboot timer + // is armed when the client reads /api/config/result (handleConfigResult), so + // the browser gets its confirmation before the AP/WiFi drops. This covers a + // client that disconnected and never polls — generous enough for a phone + // that got bounced off the AP mid-save to rejoin and fetch its confirmation. _reboot_at = now + 30000; if (_reboot_at == 0) _reboot_at = 1; } @@ -397,6 +435,9 @@ void WebConfigServer::handleStatus(AsyncWebServerRequest* req) { doc["uptime_s"] = millis() / 1000; doc["runtime_slots"] = RUNTIME_MQTT_SLOTS; doc["max_slots"] = MAX_MQTT_SLOTS; + // Servers the UI should expose: only as many as can actually be active at + // once (2 without PSRAM, 5 with). Configuring more never connects. + doc["active_slots"] = MQTTBridge::getMaxActiveSlots(); AsyncResponseStream* res = req->beginResponseStream("application/json"); serializeJson(doc, *res); @@ -541,13 +582,21 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) { return; } bool reboot_after = doc["reboot"] | false; + const char* reqid = doc["reqid"] | ""; JsonObject set = doc["set"]; WCLock lock(_mux); // A DONE batch stays readable until the next POST claims the slot, so a // client that lost the result response can re-poll instead of failing. if (_batch_state == BATCH_PENDING) { - req->send(409, "application/json", "{\"error\":\"busy\"}"); + // Echo the in-flight batch's reqid so the caller can tell its own retry + // (same reqid — landed, keep polling) from another client's save. + StaticJsonDocument<96> bd; + bd["error"] = "busy"; + bd["reqid"] = (const char*)_batch_reqid; + String out; + serializeJson(bd, out); + req->send(409, "application/json", out); return; } @@ -556,9 +605,17 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) { const char* key = kv.key().c_str(); const char* val = kv.value().as(); if (!val || !isAllowedSetKey(key)) { - char err[96]; - snprintf(err, sizeof(err), "{\"error\":\"bad key\",\"key\":\"%.32s\"}", key); - req->send(400, "application/json", err); + // Build with ArduinoJson so an attacker-supplied key containing quotes or + // backslashes is escaped rather than breaking out of the JSON string. + char safe_key[33]; + strncpy(safe_key, key, sizeof(safe_key) - 1); + safe_key[sizeof(safe_key) - 1] = 0; + StaticJsonDocument<128> ed; + ed["error"] = "bad key"; + ed["key"] = safe_key; + String out; + serializeJson(ed, out); + req->send(400, "application/json", out); return; } if (isSecretKey(key) && strcmp(val, SECRET_SENTINEL) == 0) continue; // unchanged @@ -587,15 +644,22 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) { _batch_next = 0; _batch_reboot = reboot_after; _batch_reboot_armed = false; + _batch_all_ok = true; + strncpy(_batch_reqid, reqid, sizeof(_batch_reqid) - 1); + _batch_reqid[sizeof(_batch_reqid) - 1] = 0; _batch_state = BATCH_PENDING; // tick() picks it up on the loop task uint32_t du = millis() + 60000; if (du == 0) du = 1; _diag_until = du; Serial.printf("WC: config POST accepted, %d cmds, reboot=%d\n", count, (int)reboot_after); - char msg[64]; - snprintf(msg, sizeof(msg), "{\"state\":\"pending\",\"count\":%d}", count); - req->send(202, "application/json", msg); + StaticJsonDocument<96> ack; + ack["state"] = "pending"; + ack["count"] = count; + ack["reqid"] = (const char*)_batch_reqid; + String out; + serializeJson(ack, out); + req->send(202, "application/json", out); } void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) { @@ -612,14 +676,23 @@ void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) { return; } if (_batch_state == BATCH_PENDING) { - req->send(200, "application/json", "{\"state\":\"pending\"}"); + StaticJsonDocument<96> pd; + pd["state"] = "pending"; + pd["reqid"] = (const char*)_batch_reqid; + String out; + serializeJson(pd, out); + req->send(200, "application/json", out); return; } - Serial.printf("WC: result read -> done (reboot=%d armed=%d)\n", - (int)_batch_reboot, (int)_batch_reboot_armed); + Serial.printf("WC: result read -> done (reboot=%d armed=%d all_ok=%d)\n", + (int)_batch_reboot, (int)_batch_reboot_armed, (int)_batch_all_ok); DynamicJsonDocument doc(6144); doc["state"] = "done"; - doc["reboot"] = _batch_reboot; + // Only advertise a reboot when it will actually happen: a partially-failed + // batch is not rebooted (see below), so the UI must not show a reboot screen. + doc["reboot"] = _batch_reboot && _batch_all_ok; + doc["all_ok"] = _batch_all_ok; + doc["reqid"] = (const char*)_batch_reqid; JsonArray results = doc.createNestedArray("results"); for (int i = 0; i < _batch_count; i++) { JsonObject r = results.createNestedObject(); @@ -627,10 +700,12 @@ void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) { r["reply"] = (const char*)_batch[i].reply; } // State stays DONE (re-readable) until the next POST claims the slot. - if (_batch_reboot && !_batch_reboot_armed) { - // Confirmation delivered — reboot 3 s from now (replaces the 15 s - // drain-time fallback) so the UI can show its countdown first. Armed - // once; re-reads must not keep pushing the deadline out. + if (_batch_reboot && _batch_all_ok && !_batch_reboot_armed) { + // Confirmation delivered and every command succeeded — reboot 3 s from now + // (replaces the 30 s drain-time fallback) so the UI can show its countdown + // first. Armed once; re-reads must not keep pushing the deadline out. A + // partially-failed batch is deliberately left running so the operator can + // correct and retry instead of rebooting into a broken config. _batch_reboot_armed = true; _reboot_at = millis() + 3000; if (_reboot_at == 0) _reboot_at = 1; diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index 1dd5f31f..79b195c8 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -118,6 +118,10 @@ private: uint32_t _batch_last_cmd = 0; bool _batch_reboot = false; bool _batch_reboot_armed = false; + bool _batch_all_ok = true; // every drained command replied "OK" (gates reboot) + // Client-supplied batch identity, echoed in the 202/result/409 responses so a + // reused/lost/concurrent result can't be mistaken for this client's own. + char _batch_reqid[24] = {0}; BatchEntry _batch[MAX_BATCH]; // LAN-mode session (single slot; new login evicts the old session) diff --git a/variants/heltec_tracker_v2/HeltecTrackerV2Board.cpp b/variants/heltec_tracker_v2/HeltecTrackerV2Board.cpp index f182c905..245b71a7 100644 --- a/variants/heltec_tracker_v2/HeltecTrackerV2Board.cpp +++ b/variants/heltec_tracker_v2/HeltecTrackerV2Board.cpp @@ -80,7 +80,14 @@ void HeltecTrackerV2Board::begin() { } const char* HeltecTrackerV2Board::getManufacturerName() const { + // The v1.1 environment reuses this V2 board implementation (same variant + // dir), so report the correct identity per build flag — this string feeds + // WebConfig and MQTT status/board metadata. +#ifdef HELTEC_TRACKER_V1_1 + return "Heltec Tracker V1.1"; +#else return "Heltec Tracker V2"; +#endif } bool HeltecTrackerV2Board::setLoRaFemLnaEnabled(bool enable) { diff --git a/webui/index.html b/webui/index.html index 4be74971..16e8fd4c 100644 --- a/webui/index.html +++ b/webui/index.html @@ -148,11 +148,11 @@ canvas{width:100%;height:56px;display:block}

Connect this node to your WiFi network so it can reach the MQTT servers.

- +
- +
Leave blank for an open network. 2.4 GHz networks only.
@@ -171,6 +171,8 @@ canvas{width:100%;height:56px;display:block}
Maximum legal power varies by region — check local rules.
+
RepeatForward mesh traffic. Off = listen-only (still observes and publishes). +
Need settings that aren't listed? Choose Keep current settings, finish setup, then fine-tune in the Advanced editor.
@@ -183,16 +185,16 @@ canvas{width:100%;height:56px;display:block}

Step 3 · MQTT

- +
How this observer identifies itself in published status/packet messages. Informational only — not used for topics or authentication. Prefilled with the node name.
- +
Nearest airport code, e.g. DEN. Used in topic paths.
Optional — 64-char hex public key of your companion node. Included in auth JWTs so services that support it can let you claim this node.
- +
Optional — also included in auth JWTs for claiming this node on some services.

Servers

@@ -293,12 +295,12 @@ canvas{width:100%;height:56px;display:block}

Identity

-
+
How this observer identifies itself in published messages; informational only. Blank = node name.
-
+
64-char hex public key of the owner's companion node (optional).
-
+

Publishing

@@ -324,15 +326,15 @@ canvas{width:100%;height:56px;display:block}

Time & SNMP

-
+
Enter none to clear.
-
+
SNMP agentRestart required
-
+
@@ -342,10 +344,10 @@ canvas{width:100%;height:56px;display:block}
Changing WiFi restarts the connection — this page will drop and the node reappears on the new network. Find its new IP from your router or the serial console.
- +
-
+
@@ -379,7 +381,7 @@ canvas{width:100%;height:56px;display:block}
- +
@@ -420,7 +422,7 @@ canvas{width:100%;height:56px;display:block}