From 6b3f7df4d0481a1ba748596e2d028fc5abc8edd6 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Mon, 12 Oct 2026 14:57:01 -0700 Subject: [PATCH] Suspend failed ESP32 MQTT and NTP uplinks for power saving Share one bounded WiFi outage and recovery schedule across MQTT, automatic NTP, Companion station ownership and WebConfig. Preserve enabled settings and foreground access while allowing normal sleep after confirmed physical shutdown. Honor failed SDK client stops before handing off WiFi, protect clean teardown and reconfiguration, and retry transient ESP-NOW channel restoration without repeated disconnects. Add production-method, SDK failure and real sleep regressions to CI. Validation: native MQTT/NTP/clock/reconnect suites; production power and wrapper failure tests; browser, room HTTP, OTA and combined bridge regressions; Full Heltec V4 repeater, room and companion builds with capability, flash and linked-memory checks. --- .github/workflows/run-unit-tests.yml | 5 + docs/WiFi.md | 26 + examples/companion_radio/MyMesh.cpp | 5 +- examples/companion_radio/main.cpp | 200 +++- examples/simple_repeater/MyMesh.cpp | 25 +- examples/simple_room_server/MyMesh.cpp | 23 +- .../src/PsychicMqttClient.cpp | 39 +- lib/PsychicMqttClient/src/PsychicMqttClient.h | 9 +- src/helpers/WiFiNetworkPowerPolicy.h | 116 +++ src/helpers/bridges/MQTTBridge.cpp | 264 +++++- src/helpers/bridges/MQTTBridge.h | 22 + src/helpers/esp32/WebConfigServer.cpp | 70 +- src/helpers/esp32/WebConfigServer.h | 10 + src/helpers/esp32/WiFiOtaSeeder.cpp | 4 + test/fixtures/mqtt_ap_ownership.cpp | 12 +- test/fixtures/rs232_mqtt_runtime.cpp | 37 +- test/fixtures/webconfig_ap_start.cpp | 2 + test/fixtures/webconfig_ota_handoff.cpp | 14 +- test/fixtures/webconfig_recovery_auth.cpp | 5 + test/fixtures/webconfig_station_owner.cpp | 1 + test/fixtures/webconfig_station_recovery.cpp | 30 + test/test_companion_wifi_network_power.py | 474 ++++++++++ test/test_companion_wifi_ntp.py | 9 +- test/test_esp32_full_power_lifecycle.py | 58 ++ test/test_esp32_full_sleep.py | 55 +- test/test_mqtt_disconnect.py | 323 +++++++ test/test_mqtt_network_timeout.py | 13 + test/test_mqtt_transport_results.py | 21 + test/test_mqtt_wifi_network_power.py | 856 ++++++++++++++++++ test/test_room_web_http.py | 24 +- test/test_rs232_mqtt_runtime.py | 5 + test/test_tls_download_clock_gates.py | 17 +- test/test_webconfig_network_power.py | 151 +++ test/test_webconfig_recovery_auth.py | 2 +- test/test_webconfig_station_ownership.py | 4 + test/test_wifi_network_power_policy.py | 405 +++++++++ test/test_wifi_ota_start.py | 4 +- 37 files changed, 3258 insertions(+), 82 deletions(-) create mode 100644 src/helpers/WiFiNetworkPowerPolicy.h create mode 100644 test/test_companion_wifi_network_power.py create mode 100644 test/test_mqtt_disconnect.py create mode 100644 test/test_mqtt_wifi_network_power.py create mode 100644 test/test_webconfig_network_power.py create mode 100644 test/test_wifi_network_power_policy.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 82c882a28..296bfe57a 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -222,10 +222,15 @@ jobs: python3 -B test/test_mqtt_transport_results.py -v python3 -B test/test_mqtt_tls_security.py -v python3 -B test/test_mqtt_network_timeout.py -v + python3 -B test/test_mqtt_disconnect.py -v python3 -B test/test_mqtt_connection_admission.py -v python3 -B test/test_mqtt_handshake_scheduler.py -v python3 -B test/test_mqtt_diagnostics.py -v python3 -B test/test_mqtt_publish_ack.py -v + python3 -B test/test_wifi_network_power_policy.py -v + python3 -B test/test_mqtt_wifi_network_power.py -v + python3 -B test/test_companion_wifi_network_power.py -v + python3 -B test/test_webconfig_network_power.py -v - name: Verify DNS lease restoration and bounded diagnostics run: python3 -B test/test_network_dns.py -v diff --git a/docs/WiFi.md b/docs/WiFi.md index bea2b7a2a..694b375db 100644 --- a/docs/WiFi.md +++ b/docs/WiFi.md @@ -102,6 +102,32 @@ best effort rather than durable storage. Each enabled slot publishes independently, so one failed broker does not intentionally stop the other slots. +## Automatic network outage power saving + +ESP32 MQTT firmware uses one shared outage timer for MQTT and NTP. If no broker +demonstrates a working connection for five minutes, the MQTT worker stops its +clients and NTP request before suspending background WiFi. Saved WiFi and broker +settings remain enabled. Failed requests, a WiFi association, and NTP replies +alone do not restart the outage timer or keep a failed MQTT uplink awake. + +After 30 minutes, the firmware opens a recovery window lasting up to two +minutes. A broker acknowledgment or a connection that stays up for 30 seconds +returns the node to normal operation. Otherwise WiFi is suspended for another +30 minutes. An explicit management request can ask for an earlier probe. +Automatic NTP requests have a 15-second timeout; failed requests wait 30 minutes +instead of repeatedly running a blocking server search. Successful NTP retains +the daily refresh schedule. + +Active browser administration, setup/OTA access points, connected Companion +clients, and an attached WiFi OTA seeder protect their network access. An idle +WebConfig listener does not prevent the cooldown. An enabled SNMP agent keeps a +working LAN connection available. ESP-NOW keeps its radio and channel; only the +failed infrastructure connection is suspended. LoRa continues operating, and +when CPU power saving is enabled, its normal sleep eligibility returns after +background WiFi shuts down. USB console and logging sleep rules still apply +separately. This policy does not automatically turn off working services or +expire an active setup access point. + See [MQTT_IMPLEMENTATION.md](https://github.com/mikecarper/MeshCore/blob/keymindCascade/MQTT_IMPLEMENTATION.md) for the complete preset list, custom broker configuration, topic formats, authentication, diagnostics, and memory limits. diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index a24f467cb..8538dc4a6 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -4721,7 +4721,10 @@ void MyMesh::serviceMQTT(const char* wifi_ssid, const char* wifi_password) { } } - if (WiFi.status() == WL_CONNECTED && _mqtt_configured && !_mqtt_started && _mqtt_bridge + // Start the recovery scheduler even when the router is down at boot. The + // Companion still owns association; waiting for GOT_IP here would leave its + // failed station outside MQTT/NTP's shared power budget indefinitely. + if (_mqtt_configured && !_mqtt_started && _mqtt_bridge #ifdef WITH_WEBCONFIG && !isWebConfigSetupActive() #endif diff --git a/examples/companion_radio/main.cpp b/examples/companion_radio/main.cpp index 364621874..69d27e5d3 100644 --- a/examples/companion_radio/main.cpp +++ b/examples/companion_radio/main.cpp @@ -100,6 +100,7 @@ MultiSerialInterface interface_manager; #include #include #include + #include #include #include #include @@ -1508,6 +1509,12 @@ void halt() { static bool companion_wifi_active = false; static bool companion_wifi_disable_in_progress = false; static bool companion_wifi_services_stopped = false; + static bool companion_wifi_background_stopped = false; + static bool companion_wifi_espnow_restore_pending = false; + static uint32_t companion_wifi_espnow_restore_at = 0; +#ifdef WITH_WEBCONFIG + static bool companion_wifi_webconfig_start_pending = false; +#endif static volatile bool companion_wifi_setup_requested = false; static volatile bool companion_wifi_setup_stop_requested = false; static bool companion_wifi_credential_reload_pending = false; @@ -1634,6 +1641,10 @@ void halt() { } static void serviceCompanionWiFiNtp() { + if (mesh::wifi::networkPower().backgroundSuspended()) { + cancelCompanionWiFiNtp(false); + return; + } // Configured MQTT builds already perform a fresh startup NTP sync and a // daily refresh. The common Companion owner covers every non-MQTT or // runtime-unconfigured WiFi node without racing MQTT's SNTP proof. @@ -1832,6 +1843,7 @@ void halt() { void scheduleCompanionWiFiCredentialReload() { loadCompanionWiFiCredentials(); + mesh::wifi::networkPower().requestProbe(); companion_wifi_credential_reload_pending = true; companion_wifi_credential_reload_at = millis() + 1500UL; if (companion_wifi_credential_reload_at == 0) { @@ -1898,6 +1910,7 @@ void halt() { void requestCompanionWiFiSetup() { if (mesh::wireless::control().blocked(mesh::wireless::WiFi)) return; + mesh::wifi::networkPower().requestProbe(); companion_wifi_setup_stop_requested = false; companion_wifi_setup_requested = true; } @@ -1921,6 +1934,7 @@ void halt() { return false; } companion_wifi_requested = enabled; + if (enabled) mesh::wifi::networkPower().requestProbe(); if (!companion_wifi_requested && companion_wifi_active) { companion_wifi_disable_in_progress = true; } @@ -2230,6 +2244,60 @@ void halt() { #endif #if defined(ESP32) && defined(WIFI_SSID) + static bool companionWiFiBackgroundSuspended() { + // MQTT publishes this flag only after its own TLS and SNTP work is quiet. + // A foreground OTA command can acquire the radio before its next pass. + return mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive() + && !board.isOTAUpdateRunning(); + } + + static bool companionWiFiAutomaticRecoveryAllowed() { + // MQTT owns one combined outage/cooldown clock. Raising an automatic AP + // during that grace period would create a foreground owner indefinitely. + return !mesh::wifi::networkPower().managed(); + } + +#ifdef WITH_WEBCONFIG + static void startCompanionWiFiAutoWebConfig() { + companion_wifi_webconfig_start_pending = false; + if (!WebConfigServer::loadEnabled(true)) return; + if (companion_wifi_has_credentials && WiFi.status() != WL_CONNECTED) { + // The Companion owns the initial saved-SSID attempt. Do not ask the + // portal to start its separate fifteen-second AP fallback at boot. + companion_wifi_webconfig_start_pending = true; + return; + } + char web_reply[160]; + the_mesh.startWebConfig(!companion_wifi_has_credentials, web_reply); + WIFI_DEBUG_PRINTLN("%s", web_reply); + } + + static void serviceCompanionWiFiAutoWebConfig() { + if (!companion_wifi_webconfig_start_pending || !companion_wifi_requested + || !companion_wifi_active || companionWiFiBackgroundSuspended()) return; + if (!WebConfigServer::loadEnabled(true)) { + companion_wifi_webconfig_start_pending = false; + return; + } + if (WiFi.status() != WL_CONNECTED + || the_mesh.isWebConfigActiveOrStopping()) return; + startCompanionWiFiAutoWebConfig(); + } +#endif + + static void serviceCompanionWiFiNetworkPowerHold() { + bool active = wifi_interface.isConnected() + || companion_wifi_credential_reload_pending + || companion_wifi_setup_requested + || board.isOTAUpdateRunning(); +#ifdef ENABLE_OTA + active = active || ota_console_client.connected(); +#endif + mesh::wifi::networkPower().setHold( + mesh::wifi::NetworkPowerHold::CompanionClient, active); + } + static void resetCompanionWiFiRecoveryState() { wifi_reconnect_tracker = WiFiReconnectPolicy::Tracker(); wifi_setup_attempted = false; @@ -2242,6 +2310,7 @@ void halt() { || static_cast(millis() - companion_wifi_credential_reload_at) < 0) { return; } + if (companionWiFiBackgroundSuspended()) return; #ifdef WITH_WEBCONFIG if (the_mesh.isWebConfigActiveOrStopping()) return; #endif @@ -2250,6 +2319,7 @@ void halt() { companion_wifi_credential_reload_pending = false; companion_wifi_credential_reload_at = 0; + companion_wifi_espnow_restore_pending = false; loadCompanionWiFiCredentials(); resetCompanionWiFiRecoveryState(); if (!companion_wifi_requested || !companion_wifi_active @@ -2270,9 +2340,11 @@ void halt() { static void startCompanionWiFi() { if (mesh::wireless::control().blocked(mesh::wireless::WiFi)) return; if (companion_wifi_active) return; + if (companionWiFiBackgroundSuspended()) return; if (!mesh::wifi::stationMutationAllowed()) return; board.setInhibitSleep(true); + companion_wifi_espnow_restore_pending = false; mesh::wifi::setStationAutoReconnect(true); resetCompanionWiFiRecoveryState(); @@ -2290,11 +2362,7 @@ void halt() { } #endif #ifdef WITH_WEBCONFIG - if (WebConfigServer::loadEnabled(true)) { - char web_reply[160]; - the_mesh.startWebConfig(!companion_wifi_has_credentials, web_reply); - WIFI_DEBUG_PRINTLN("%s", web_reply); - } + startCompanionWiFiAutoWebConfig(); #endif applyCompanionWiFiPowerSave(); @@ -2305,6 +2373,7 @@ void halt() { #endif companion_wifi_active = true; companion_wifi_services_stopped = false; + companion_wifi_background_stopped = false; WIFI_DEBUG_PRINTLN("WiFi enabled by BOOT/GPIO 0 control"); } @@ -2329,17 +2398,80 @@ void halt() { static void stopCompanionInfrastructureWiFi() { WiFi.setAutoReconnect(false); -#if defined(MESH_PRIMARY_ESPNOW) && MESH_PRIMARY_ESPNOW - // ESP-NOW is the mesh radio on this target. Drop the station/AP services - // without stopping the driver, then restore its fixed LR mesh channel. - WiFi.disconnect(false, false); - WiFi.mode(WIFI_STA); - mesh::wifi::applyProtocolMask(WIFI_IF_STA); - mesh::wifi::restoreEspNowChannel(); -#else - WiFi.disconnect(true, false); - WiFi.mode(WIFI_OFF); + if (mesh::wifi::espNowChannelConstrained()) { + // Primary ESP-NOW and a live optional bridge both share this driver. + // Drop only station association and keep their fixed mesh channel. + WiFi.disconnect(false, false); + WiFi.mode(WIFI_STA); + mesh::wifi::applyProtocolMask(WIFI_IF_STA); + companion_wifi_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + companion_wifi_espnow_restore_at = millis() + 1000UL; + } else { + companion_wifi_espnow_restore_pending = false; + WiFi.disconnect(true, false); + WiFi.mode(WIFI_OFF); + } + } + + static void serviceCompanionEspNowChannelRestore() { + if (!companion_wifi_espnow_restore_pending) return; + if (!mesh::wifi::espNowChannelConstrained()) { + companion_wifi_espnow_restore_pending = false; + return; + } + if (board.isOTAUpdateRunning() + || mesh::wifi::networkPower().foregroundActive() + || !mesh::wifi::stationMutationAllowed()) return; + const uint32_t now = millis(); + if (static_cast(now - companion_wifi_espnow_restore_at) < 0) return; + // esp_wifi_set_channel may fail until an asynchronous disconnect/scan + // completes. Retry the channel only; repeated disconnects would prevent + // the shared ESP-NOW receiver from recovering during a long cooldown. + companion_wifi_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + companion_wifi_espnow_restore_at = now + 1000UL; + } + + static void serviceCompanionWiFiBackgroundPower() { + serviceCompanionEspNowChannelRestore(); + if (!companion_wifi_active || !companion_wifi_requested) return; + if (companionWiFiBackgroundSuspended()) { + if (companion_wifi_background_stopped) return; + // Settings and MQTT's scheduling task stay enabled. These idle socket + // listeners and the common NTP request own no useful foreground work. + cancelCompanionWiFiNtp(false); + wifi_interface.end(); +#ifdef ENABLE_OTA + ota_console_stop(); #endif + stopCompanionInfrastructureWiFi(); + board.setInhibitSleep(mesh::wifi::espNowChannelConstrained()); + companion_wifi_background_stopped = true; + WIFI_DEBUG_PRINTLN("Network unavailable; WiFi suspended until the next recovery probe"); + return; + } + if (!companion_wifi_background_stopped) return; + if (!mesh::wifi::stationMutationAllowed()) return; + + board.setInhibitSleep(true); + companion_wifi_espnow_restore_pending = false; + resetCompanionWiFiRecoveryState(); + if (companion_wifi_has_credentials) { + WiFi.mode(WIFI_STA); + mesh::wifi::applyProtocolMask(WIFI_IF_STA); + mesh::wifi::setStationAutoReconnect(true); + wifi_reconnect_tracker.noteDisconnected(millis()); + mesh::wifi::beginStation(configured_wifi_ssid, configured_wifi_password); + } + applyCompanionWiFiPowerSave(); + wifi_interface.begin(TCP_PORT); + wifi_interface.enable(); +#ifdef ENABLE_OTA + ota_console_start(); +#endif + companion_wifi_background_stopped = false; + WIFI_DEBUG_PRINTLN("WiFi resumed for a bounded network recovery probe"); } static bool finishStoppingCompanionWiFi() { @@ -2368,6 +2500,10 @@ void halt() { board.setInhibitSleep(false); resetCompanionWiFiRecoveryState(); companion_wifi_active = false; + companion_wifi_background_stopped = false; +#ifdef WITH_WEBCONFIG + companion_wifi_webconfig_start_pending = false; +#endif #if defined(MESH_PRIMARY_ESPNOW) && MESH_PRIMARY_ESPNOW WIFI_DEBUG_PRINTLN( "Companion WiFi services disabled; ESP-NOW mesh radio remains active"); @@ -2388,6 +2524,10 @@ void halt() { if (companion_wifi_requested && !companion_wifi_active) { startCompanionWiFi(); } + serviceCompanionWiFiBackgroundPower(); +#ifdef WITH_WEBCONFIG + serviceCompanionWiFiAutoWebConfig(); +#endif } #endif @@ -3014,7 +3154,25 @@ static bool companionNativeUsbLightSleepReady() { // Full compiles every transport, but a compiled driver is not an active // receiver. Preserve live services and pending replies while permitting // the same short sleep slice when those services have been turned off. - if (companion_wireless.enabled() != 0 || interface_manager.hasPendingIO() + uint8_t enabled_services = companion_wireless.enabled(); +#if defined(ESP32) && defined(WIFI_SSID) + // The user's saved WiFi preference and an idle WebConfig listener remain + // enabled during automatic suspension. Only a confirmed physical power-off + // releases their sleep blocker; foreground work and shared ESP-NOW retain it. + if (companion_wifi_background_stopped + && mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive() + && !mesh::wifi::espNowChannelConstrained()) { + wifi_mode_t parked_mode = WIFI_MODE_NULL; + const esp_err_t parked_result = esp_wifi_get_mode(&parked_mode); + if ((parked_result == ESP_OK && parked_mode == WIFI_MODE_NULL) + || parked_result == ESP_ERR_WIFI_NOT_INIT + || parked_result == ESP_ERR_WIFI_NOT_STARTED) { + enabled_services &= static_cast(~mesh::wireless::WiFi); + } + } +#endif + if (enabled_services != 0 || interface_manager.hasPendingIO() || board.isOTAUpdateRunning() || board.isRadioTestActive() || sensors.gpsUsesSerialUart(1)) return false; #if defined(BLE_PIN_CODE) && !CONFIG_IDF_TARGET_ESP32C6 @@ -3624,6 +3782,9 @@ void loop() { } #endif mesh::wireless::control().service(millis()); +#if defined(ESP32) && defined(WIFI_SSID) + serviceCompanionWiFiNetworkPowerHold(); +#endif #if defined(NRF52_PLATFORM) board.feedWatchdog(); #endif @@ -3800,8 +3961,11 @@ void loop() { } #endif serviceCompanionWiFiCredentialReload(); + serviceCompanionWiFiNetworkPowerHold(); serviceCompanionWiFiState(); - if (companion_wifi_requested && companion_wifi_active) { + if (companion_wifi_requested && companion_wifi_active + && !companionWiFiBackgroundSuspended() + && !companion_wifi_background_stopped) { #ifdef ENABLE_OTA ota_console_loop(); // service the local text console (port 5002) #endif @@ -3847,7 +4011,7 @@ void loop() { } } else if (configured_wifi_ssid[0]) { wifi_reconnect_tracker.noteDisconnected(wifi_now); - if (!wifi_setup_recovery_mode + if (!wifi_setup_recovery_mode && companionWiFiAutomaticRecoveryAllowed() #ifdef WITH_WEBCONFIG && !the_mesh.isWebConfigSetupActive() #else diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index a346e6348..6f742ec36 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1,6 +1,7 @@ #include "MyMesh.h" #if defined(ESP32_PLATFORM) #include +#include #if defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ || defined(LIGHTWEIGHT_WIFI_OTA) || (defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA)) #include @@ -11125,6 +11126,9 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup return true; } if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + if (!automatic_setup && !_webconfig->isStopping()) { + _webconfig->requestNetworkAccess(); + } strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); return true; @@ -11163,6 +11167,7 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup } } + if (!automatic_setup) _webconfig->requestNetworkAccess(); const bool setup_was_suspended = _unconfigured_setup_espnow_suspended; #ifdef WITH_ESPNOW_BRIDGE const bool espnow_was_running = isEspNowBridgeRunning(); @@ -13819,11 +13824,15 @@ void MyMesh::serviceIdleWiFi() { && (!defined(MESH_PRIMARY_ESPNOW) || !MESH_PRIMARY_ESPNOW) \ && (!defined(MESH_ESPNOW_RADIO) || !MESH_ESPNOW_RADIO) if (_cli.getBoard()->isOTAUpdateRunning()) return; + if (mesh::wifi::networkPower().foregroundActive()) return; #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return; #endif #ifdef WITH_MQTT_BRIDGE if (_ota_update_at) return; + // The MQTT worker owns physical shutdown and resume, including cooldown. + // Never race its Core 0 Arduino/SDK interface transition from this loop. if (mqtt_bridge && (mqtt_bridge->isRunning() || mqtt_bridge->isStopping())) return; #endif #ifdef WITH_ESPNOW_BRIDGE @@ -13852,8 +13861,12 @@ void MyMesh::serviceIdleWiFi() { bool MyMesh::hasPendingWork() const { if (isDualRadioActive()) return true; if (hasPendingOtaApply()) return true; +#if defined(ESP32_PLATFORM) + if (mesh::wifi::networkPower().foregroundActive()) return true; +#endif #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return true; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return true; #endif #if defined(ESP32_PLATFORM) \ && (defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ @@ -13871,7 +13884,13 @@ bool MyMesh::hasPendingWork() const { if (deferred_cli_command.pending || pending_self_advert || _cli.hasActiveUserGpioTimer()) return true; #if defined(WITH_BRIDGE) const AbstractBridge* active_bridge = activeBridge(); - if (active_bridge && active_bridge->isRunning()) return true; + if (active_bridge && active_bridge->isRunning()) { +#ifdef WITH_MQTT_BRIDGE + if (active_bridge != mqtt_bridge || !mqtt_bridge->isNetworkSuspended()) return true; +#else + return true; +#endif + } #ifdef WITH_MQTT_BRIDGE if (mqtt_bridge && mqtt_bridge->isStopping()) return true; #endif diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 5b4d1dc32..75a384848 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1,6 +1,7 @@ #include "MyMesh.h" #if defined(ESP32_PLATFORM) #include +#include #if defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ || defined(LIGHTWEIGHT_WIFI_OTA) || (defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA)) #include @@ -3119,6 +3120,9 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup return true; } if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + if (!automatic_setup && !_webconfig->isStopping()) { + _webconfig->requestNetworkAccess(); + } strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); return true; @@ -3154,6 +3158,7 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup } } + if (!automatic_setup) _webconfig->requestNetworkAccess(); const bool setup_was_suspended = _unconfigured_setup_espnow_suspended; #ifdef WITH_ESPNOW_BRIDGE const bool espnow_was_running = isEspNowBridgeRunning(); @@ -4198,11 +4203,15 @@ void MyMesh::serviceIdleWiFi() { && (!defined(MESH_PRIMARY_ESPNOW) || !MESH_PRIMARY_ESPNOW) \ && (!defined(MESH_ESPNOW_RADIO) || !MESH_ESPNOW_RADIO) if (_cli.getBoard()->isOTAUpdateRunning()) return; + if (mesh::wifi::networkPower().foregroundActive()) return; #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return; #endif #ifdef WITH_MQTT_BRIDGE if (_ota_update_at) return; + // The MQTT worker also owns its suspended station: Core 1 must not race + // the worker's shutdown or the next bounded probe on Core 0. if (bridge && (bridge->isRunning() || bridge->isStopping())) return; #endif #ifdef WITH_ESPNOW_BRIDGE @@ -4226,8 +4235,12 @@ void MyMesh::serviceIdleWiFi() { bool MyMesh::hasPendingWork() const { if (isDualRadioActive()) return true; if (hasPendingOtaApply()) return true; +#if defined(ESP32_PLATFORM) + if (mesh::wifi::networkPower().foregroundActive()) return true; +#endif #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return true; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return true; #endif #if defined(ESP32_PLATFORM) \ && (defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ @@ -4243,7 +4256,11 @@ bool MyMesh::hasPendingWork() const { if (_local_cli_output.busy()) return true; #endif #if defined(WITH_BRIDGE) - if (bridge && bridge->isRunning()) return true; // bridge needs WiFi radio, can't sleep +#ifdef WITH_MQTT_BRIDGE + if (bridge && bridge->isRunning() && !bridge->isNetworkSuspended()) return true; +#else + if (bridge && bridge->isRunning()) return true; +#endif #ifdef WITH_MQTT_BRIDGE if (bridge && bridge->isStopping()) return true; #endif diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index 6f0f16f36..54dc75b49 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -471,7 +471,7 @@ esp_err_t PsychicMqttClient::reconnect() void PsychicMqttClient::disconnect() { - if (_client == nullptr) + if (_client == nullptr || !_started) { ESP_LOGW(TAG, "MQTT client not started."); return; @@ -481,16 +481,41 @@ void PsychicMqttClient::disconnect() { ESP_LOGI(TAG, "Disconnecting MQTT client."); _stopMqttClient = false; - esp_mqtt_client_disconnect(_client); - - // Wait for all disconnect events to be processed - while (!_stopMqttClient) + const esp_err_t disconnect_result = esp_mqtt_client_disconnect(_client); + if (disconnect_result == ESP_OK) { - vTaskDelay(10 / portTICK_PERIOD_MS); + // The SDK can lose its disconnect notification during a failed + // transport. Give callbacks one second, then use SDK stop to close + // the transport instead of waiting forever on that notification. + const uint32_t started_at = static_cast(millis()); + const uint32_t poll_ticks = 10 / portTICK_PERIOD_MS; + while (!_stopMqttClient + && static_cast(millis() - started_at) < 1000U) + { + vTaskDelay(poll_ticks ? poll_ticks : 1); + } + if (!_stopMqttClient) + { + ESP_LOGW(TAG, "Disconnect notification timed out; stopping MQTT client."); + } + } + else + { + ESP_LOGW(TAG, "MQTT disconnect request failed: %s; stopping client.", + esp_err_to_name(disconnect_result)); } } - esp_mqtt_client_stop(_client); + const esp_err_t stop_result = esp_mqtt_client_stop(_client); + if (stop_result != ESP_OK) + { + // A failed stop is not permission for the station owner to disable + // WiFi or reclaim transport buffers. isStarted() keeps that handoff + // blocked until an owning-task retry actually stops the SDK client. + ESP_LOGW(TAG, "MQTT client stop failed: %s", esp_err_to_name(stop_result)); + return; + } + _connected = false; _started = false; ESP_LOGI(TAG, "MQTT client stopped."); } diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index f5b29f31b..e1bced0fa 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -366,7 +366,8 @@ public: /** * @brief Disconnects the MQTT client from the server. - * This call might be blocking until the client is stopped cleanly + * Disconnect notification waits at most one second before SDK stop. + * A failed SDK stop leaves isStarted() true so callers retain ownership. */ void disconnect(); @@ -494,10 +495,10 @@ private: esp_mqtt_client_handle_t _client = nullptr; esp_mqtt_client_config_t _mqtt_cfg; esp_mqtt_error_codes_t _lastError; - bool _connected = false; - bool _stopMqttClient = false; + std::atomic _connected{false}; + std::atomic _stopMqttClient{false}; bool _config_dirty = true; - bool _started = false; + std::atomic _started{false}; // Runtime cap on the esp-mqtt outbox for QoS 0 async publishes (bytes). // 0 = disabled. Enforced in publish(); not an esp-mqtt config field. diff --git a/src/helpers/WiFiNetworkPowerPolicy.h b/src/helpers/WiFiNetworkPowerPolicy.h new file mode 100644 index 000000000..0a1538ced --- /dev/null +++ b/src/helpers/WiFiNetworkPowerPolicy.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include + +namespace mesh { +namespace wifi { + +static constexpr uint32_t kNetworkFailureGraceMs = 5UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkRecoveryProbeMs = 2UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkRecoveryIntervalMs = 30UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkHealthyMs = 30000UL; + +// MQTT and its NTP bootstrap share one outage clock. A successful WiFi +// association or time reply alone is not proof that an MQTT destination works. +// Failed attempts must never extend either the grace period or the cooldown. +class NetworkPowerPolicy { +public: + enum class Action { Run, Suspend, Resume }; + + Action update(uint32_t now, bool useful_network, bool foreground, + bool requested_probe = false) { + if (!_started) { + _started = true; + _since = now; + } + if (useful_network) { + const bool was_suspended = _suspended; + _suspended = false; + _probing = false; + _since = now; + return was_suspended ? Action::Resume : Action::Run; + } + if (_suspended) { + if (foreground || requested_probe + || now - _since >= kNetworkRecoveryIntervalMs) { + _suspended = false; + _probing = true; + _since = now; + return Action::Resume; + } + return Action::Suspend; + } + if (foreground) return Action::Run; + const uint32_t budget = _probing + ? kNetworkRecoveryProbeMs : kNetworkFailureGraceMs; + if (now - _since >= budget) { + _suspended = true; + _probing = false; + _since = now; + return Action::Suspend; + } + return Action::Run; + } + + bool suspended() const { return _suspended; } + +private: + bool _started = false; + bool _suspended = false; + bool _probing = false; + uint32_t _since = 0; +}; + +enum class NetworkPowerHold : uint8_t { + CompanionClient = 1, + WebConfig = 2, + OtaSeeder = 4 +}; + +// One program-wide handoff between the MQTT owner task and the station/UI +// owners. Only the worker publishes suspension, after stopping TLS and SNTP. +// Foreground owners publish holds; they never stop the worker's transports. +class NetworkPowerControl { +public: + bool managed() const { return _managed.load(std::memory_order_acquire); } + void setManaged(bool value) { + _managed.store(value, std::memory_order_release); + if (!value) { + _suspended.store(false, std::memory_order_release); + _probe.store(false, std::memory_order_release); + } + } + bool backgroundSuspended() const { + return managed() && _suspended.load(std::memory_order_acquire); + } + void setSuspended(bool value) { + _suspended.store(value, std::memory_order_release); + } + void setHold(NetworkPowerHold owner, bool value) { + const uint8_t mask = static_cast(owner); + if (value) _holds.fetch_or(mask, std::memory_order_acq_rel); + else _holds.fetch_and(static_cast(~mask), std::memory_order_acq_rel); + } + bool foregroundActive() const { + return _holds.load(std::memory_order_acquire) != 0; + } + void requestProbe() { _probe.store(true, std::memory_order_release); } + bool consumeProbeRequest() { + return _probe.exchange(false, std::memory_order_acq_rel); + } + +private: + std::atomic _managed{false}; + std::atomic _suspended{false}; + std::atomic _probe{false}; + std::atomic _holds{0}; +}; + +inline NetworkPowerControl& networkPower() { + static NetworkPowerControl control; + return control; +} + +} // namespace wifi +} // namespace mesh diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 564d670e6..21754a061 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -89,6 +89,15 @@ static constexpr uint32_t MQTT_NTP_NON_ESP_SETTLE_MS = 5000UL; #endif static constexpr unsigned long MQTT_NTP_MIN_VALID_EPOCH = 1767225600UL; +static uint32_t mqttNtpFailureRetryAt(uint32_t now) { +#ifdef ESP_PLATFORM + const uint32_t deadline = now + mesh::wifi::kNetworkRecoveryIntervalMs; + return deadline == 0 ? 1 : deadline; +#else + return MQTTConnectionPolicy::ntpRetryAt(now); +#endif +} + static bool mqttNtpRefreshDue(uint32_t now, uint32_t last_sync, uint32_t retry_at) { return MQTTConnectionPolicy::ntpRefreshDue(now, last_sync, retry_at); @@ -1346,6 +1355,10 @@ void MQTTBridge::finishStopped() { // Timezone is inline class storage (_timezone_storage) - nothing to delete. // The shared JSON document's pools were freed by releaseRuntimeBuffers() above. _initialized = false; +#ifdef ESP_PLATFORM + _network_suspended.store(false, std::memory_order_release); + mesh::wifi::networkPower().setManaged(false); +#endif // A clean task stop already cancels this on its owning core. The second // cancellation is intentionally idempotent and also releases the lease // after the lifecycle's bounded forced-stop fallback. @@ -1449,6 +1462,16 @@ void MQTTBridge::LifecycleOps::onStopComplete(bool clean) { // --------------------------------------------------------------------------- // FreeRTOS task entry point // --------------------------------------------------------------------------- +void MQTTBridge::applyTimezoneRules() { + TimeChangeRule dst_rule, std_rule; + if (!timezoneRulesFromString(_obs->timezone_string, dst_rule, std_rule)) { + TimeChangeRule utc = {"UTC", Last, Sun, Mar, 0, 0}; + dst_rule = utc; + std_rule = utc; + } + _timezone_storage.setRules(dst_rule, std_rule); +} + #ifdef ESP_PLATFORM void MQTTBridge::mqttTask(void* parameter) { MQTTBridge* bridge = static_cast(parameter); @@ -1559,10 +1582,168 @@ bool MQTTBridge::waitUnlessStopping(uint32_t delay_ms) { return false; } +bool MQTTBridge::serviceNetworkPower(uint32_t now) { + auto& control = mesh::wifi::networkPower(); + bool useful_network = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + const uint32_t acknowledgments = _slots[i].client + ? _slots[i].client->getPublishAck() : 0; + const bool broker_replied = acknowledgments != 0 + && acknowledgments != _network_publish_acks[i]; + _network_publish_acks[i] = acknowledgments; + // A flapping CONNACK must not buy another full outage window forever. + // A PUBACK proves delivery immediately; a quiet connection must stay up + // for 30 seconds, leaving time to prove recovery within the probe window. + if (_slots[i].enabled && _slots[i].connected + && (broker_replied || (now - static_cast(_slots[i].connected_at_ms) + >= mesh::wifi::kNetworkHealthyMs))) { + useful_network = true; + } + } + wifi_mode_t mode = WIFI_MODE_NULL; + const bool ap_active = esp_wifi_get_mode(&mode) == ESP_OK && (mode & WIFI_MODE_AP); + bool foreground = control.foregroundActive() || ap_active + || (_board && _board->isOTAUpdateRunning()) + || _ntp_force_requested || _ntp_diag_requested; +#ifdef WITH_SNMP + foreground = foreground || (_snmp_agent && _snmp_agent->isRunning() + && WiFi.status() == WL_CONNECTED); +#endif + const auto action = _network_power_policy.update( + now, useful_network, foreground, control.consumeProbeRequest()); + if (action == mesh::wifi::NetworkPowerPolicy::Action::Suspend) { + const bool newly_suspended = !_network_suspended.load(std::memory_order_acquire); + if (newly_suspended) { + cancelNtpRefresh(); + _ntp_sync_pending = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + _slot_attempt_pending[i] = false; + if (_slots[i].client && _slots[i].client->isStarted()) { + _slots[i].client->disconnect(); + // The wrapper only clears isStarted after SDK stop succeeds. Never + // hand station ownership off while any transport still runs. + if (_slots[i].client->isStarted()) return false; + } + _slots[i].connected = false; + _slots[i].connected_at_ms = 0; + _status_publish_pending[i] = false; + } + updateCachedConnectionStatus(); + _network_suspended.store(true, std::memory_order_release); + control.setSuspended(true); // TLS/SNTP teardown precedes the handoff. + MQTT_DEBUG_PRINTLN("Network unavailable; suspending background WiFi for 30 minutes"); + } + if (_manage_wifi && !control.foregroundActive() + && !(_board && _board->isOTAUpdateRunning()) + && mqttStationWiFiMutationAllowed() + && (newly_suspended || (!mesh::wifi::espNowChannelConstrained() + && WiFi.getMode() != WIFI_OFF))) { + WiFi.setAutoReconnect(false); + WiFi.disconnect(false, false); + if (mesh::wifi::espNowChannelConstrained()) { + // ESP-NOW is an independent radio owner. Stop failed STA attempts, + // keeping its driver, protocol and fixed channel available. + _network_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + _network_espnow_restore_at = static_cast(millis()) + 1000U; + } else if (WiFi.getMode() != WIFI_OFF) { + WiFi.mode(WIFI_OFF); + } + } + serviceEspNowChannelRestore(static_cast(millis())); + return false; + } + _network_espnow_restore_pending = false; + if (_network_suspended.exchange(false, std::memory_order_acq_rel)) { + // Publish before restarting: the Companion/role loop must not reclaim + // WiFi underneath a new association or TLS handshake. + control.setSuspended(false); + _last_wifi_check = now - 10001UL; + _wifi_status_initialized = false; + _last_slot_reconnect_ms = now - MQTTConnectionPolicy::kReconnectGuardMs; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + _slots[i].circuit_breaker_tripped = false; + _slots[i].last_reconnect_attempt = now - MQTTConnectionPolicy::reconnectDelayMs( + _slots[i].reconnect_backoff, static_cast(i)); + } + _ntp_refresh_retry_at = 0; + _ntp_sync_pending = !_fresh_ntp_this_boot.load(std::memory_order_acquire); + if (_manage_wifi) initializeWiFiInTask(); + MQTT_DEBUG_PRINTLN("Starting bounded network recovery probe"); + } + return true; +} + +void MQTTBridge::serviceEspNowChannelRestore(uint32_t now) { + if (!_network_espnow_restore_pending) return; + if (!_manage_wifi || !_network_suspended.load(std::memory_order_acquire) + || !mesh::wifi::espNowChannelConstrained()) { + _network_espnow_restore_pending = false; + return; + } + // Disconnect and scan completion are asynchronous. Retry only a failed + // channel write, once per second, without another station disconnect or + // driver mode change. Foreground owners retain their channel throughout. + if (mesh::wifi::networkPower().foregroundActive() + || (_board && _board->isOTAUpdateRunning()) + || !mqttStationWiFiMutationAllowed() + || static_cast(now - _network_espnow_restore_at) < 0) return; + _network_espnow_restore_pending = mesh::wifi::restoreEspNowChannel() != ESP_OK; + _network_espnow_restore_at = static_cast(millis()) + 1000U; +} + +void MQTTBridge::serviceBackgroundNtp(uint32_t now) { + if (WiFi.status() != WL_CONNECTED) { + pollNtpRefresh(now); + return; + } + // A retained/admin-set clock can start MQTT without a blocking multi-server + // NTP walk. It is not credited as a fresh internet reply or useful network. + if (!_ntp_synced.load(std::memory_order_acquire)) { + uint32_t system_time = 0; + MQTTConnectionPolicy::checkedRtcEpoch(static_cast(time(nullptr)), + MQTT_NTP_MIN_VALID_EPOCH, system_time); + const uint32_t rtc_time = _rtc ? _rtc->getCurrentTime() : 0; + const auto source = MQTTConnectionPolicy::chooseFallbackClock( + false, system_time, rtc_time, MQTT_NTP_MIN_VALID_EPOCH); + if (source != MQTTConnectionPolicy::ClockSource::None) { + const uint32_t epoch = source == MQTTConnectionPolicy::ClockSource::Rtc + ? rtc_time : system_time; + struct timeval accepted = {static_cast(epoch), 0}; + settimeofday(&accepted, nullptr); + if (_rtc) { + _rtc->setCurrentTime(epoch); + if (epoch < rtc_time) { + _backward_clock_reset_epoch.store(epoch, std::memory_order_release); + } + } + _ntp_synced.store(true, std::memory_order_release); + } + } + pollNtpRefresh(now); + if (_ntp_refresh_pending) return; + const bool bootstrap = _ntp_sync_pending + || !_fresh_ntp_this_boot.load(std::memory_order_acquire); + const bool retry_due = _ntp_refresh_retry_at == 0 + || mqttNtpRefreshDue(now, _last_ntp_sync, _ntp_refresh_retry_at); + if ((bootstrap && retry_due) + || mqttNtpRefreshDue(now, _last_ntp_sync, _ntp_refresh_retry_at)) { + _ntp_sync_pending = false; + refreshNTP(); + } +} + // --------------------------------------------------------------------------- // mqttTaskLoop() - main loop running on Core 0 // --------------------------------------------------------------------------- void MQTTBridge::mqttTaskLoop() { + _network_power_policy = mesh::wifi::NetworkPowerPolicy(); + _network_suspended.store(false, std::memory_order_release); + _network_espnow_restore_pending = false; + mesh::wifi::networkPower().setManaged(true); + // Timezone configuration is independent of whether an NTP server answers. + // The old blocking bootstrap applied these rules as a side effect. + applyTimezoneRules(); // An active OTA/setup AP can outlive MQTT's start request. Stay stoppable // while waiting, and leave slot/TLS work until station initialization ran. while (!_stop_requested && !initializeWiFiInTask()) { @@ -1586,10 +1767,21 @@ void MQTTBridge::mqttTaskLoop() { if (_stop_requested) { MQTT_DEBUG_PRINTLN("MQTT task: cooperative stop - tearing down clients on Core 0"); cancelNtpRefresh(); + bool transports_stopped = true; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { teardownSlot(i); + if (_slots[i].client && _slots[i].client->isStarted()) transports_stopped = false; + } + if (!transports_stopped) { + // Retain clients, tokens and the stop acknowledgment until SDK stop + // actually succeeds. end() still owns the explicitly dirty timeout + // fallback; this worker must never label a failed stop as clean. + vTaskDelay(pdMS_TO_TICKS(100)); + continue; } destroySlotClients(); + _network_suspended.store(false, std::memory_order_release); + mesh::wifi::networkPower().setManaged(false); _stop_acked.store(true, std::memory_order_release); // teardown precedes Core 1 release return; } @@ -1613,6 +1805,15 @@ void MQTTBridge::mqttTaskLoop() { unsigned long now = millis(); + // The MQTT and NTP schedulers share this single WiFi outage budget. + // Quiesce on the owner core before an external Companion station owner + // sees the suspension flag; failed timers cannot wake each other up. + if (!serviceNetworkPower(static_cast(now))) { + processPacketQueue(); + vTaskDelay(pdMS_TO_TICKS(250)); + continue; + } + // Periodic heap + publish-health snapshot. Gated behind MQTT_MEMORY_DEBUG (a // dedicated diagnostics flag, NOT enabled on production or plain MQTT_DEBUG builds) // so it stays off by default -- the same data is available on demand via the @@ -1644,21 +1845,7 @@ void MQTTBridge::mqttTaskLoop() { } } - // Check for pending NTP sync (triggered from WiFi event handler) - if (_ntp_sync_pending && WiFi.status() == WL_CONNECTED) { - _ntp_sync_pending = false; - syncTimeWithNTP(); - } - - // Retry NTP every 30s if initial sync failed (slots can't start without valid time) - if (!_ntp_synced.load(std::memory_order_acquire) - && WiFi.status() == WL_CONNECTED) { - static unsigned long last_ntp_retry = 0; - if (now - last_ntp_retry >= 30000) { - last_ntp_retry = now; - syncTimeWithNTP(); - } - } + serviceBackgroundNtp(static_cast(now)); // Process a CLI-requested forced NTP sync (queued from Core 1). Running it here // keeps all NTP I/O on Core 0; requestForcedNtpSync() blocks the CLI thread until @@ -1793,16 +1980,8 @@ void MQTTBridge::mqttTaskLoop() { // Periodic configuration check (throttled to avoid spam) checkConfigurationMismatch(); - // Periodic NTP refresh (once per day) - lightweight, non-blocking. - // Uses async SNTP instead of the heavy syncTimeWithNTP() which blocks Core 0 - // for up to 20+ seconds with DNS lookups, UDP sockets, and retry loops. - pollNtpRefresh(now); - if (WiFi.status() == WL_CONNECTED - && !_ntp_refresh_pending - && mqttNtpRefreshDue(now, _last_ntp_sync, - _ntp_refresh_retry_at)) { - refreshNTP(); - } + // Automatic boot/reconnect/daily NTP is serviced above as one bounded + // asynchronous operation. Explicit administrator diagnostics remain separate. // Publish status updates (handle millis() overflow correctly). // Read the toggle live from prefs (like mqtt.packets/rx/tx below) so a @@ -2142,10 +2321,14 @@ bool MQTTBridge::setupSlot(int index) { // transport handles are reused; setCredentials / setServer overwrite the config // fields in place before connect() restarts the ESP-IDF client. if (slot.initial_connect_done) { - if (slot.client->connected()) { + if (slot.client->isStarted()) { _slot_attempt_pending[index] = false; slot.client->disconnect(); } + if (slot.client->isStarted()) { + slot.last_reconnect_attempt = millis(); + return false; + } // Clear TLS verification fields so a stale CA-bundle attach or cert // pointer from a prior preset doesn't override the new one. esp_mqtt_client_config_t* cfg = slot.client->getMqttConfig(); @@ -2375,6 +2558,7 @@ void MQTTBridge::teardownSlot(int index, bool force) { delay(50); #endif } + if (slot.client && !force && slot.client->isStarted()) return; // Invalidate the token but keep the buffer: the client survives teardown and still // holds this pointer in its config (see MQTTSlot::auth_token). @@ -3062,6 +3246,7 @@ void MQTTBridge::setSlotPreset(int slot_index, const char* preset_name) { #ifdef ESP_PLATFORM if (_mqtt_task_handle != nullptr) { _slot_reconfigure_pending[slot_index] = true; + mesh::wifi::networkPower().requestProbe(); MQTT_DEBUG_PRINTLN("MQTT%d reconfigure queued (preset: %s)", slot_index + 1, preset_name); return; } @@ -3077,6 +3262,12 @@ void MQTTBridge::applySlotPreset(int slot_index, const char* preset_name) { MQTTSlot& slot = _slots[slot_index]; teardownSlot(slot_index); + if (slot.client && slot.client->isStarted()) { + // Keep the requested preset queued and the running SDK's current config + // untouched until its owning task can complete the failed stop. + _slot_reconfigure_pending[slot_index] = true; + return; + } if (strcmp(preset_name, MQTT_PRESET_NONE) == 0 || preset_name[0] == '\0') { slot.enabled = false; @@ -4460,7 +4651,21 @@ void MQTTBridge::refreshNTP() { #endif _ntp_refresh_previous_epoch = _rtc ? (unsigned long)_rtc->getCurrentTime() : 0; +#ifdef ESP_PLATFORM + const char* servers[kMaxNtpServers]; + int server_count = 0; + fillNtpServerList(_obs, servers, server_count); + const int first = _background_ntp_server % server_count; + // Let SNTP try three servers within one 15-second request, rotating through + // the configured list across probes instead of blocking on every server. + configTime(0, 0, servers[first], servers[(first + 1) % server_count], + servers[(first + 2) % server_count]); + // A blackholed leading server can consume the whole SDK receive timeout. + // Advance by one so every server eventually leads, even with six entries. + _background_ntp_server = static_cast((first + 1) % server_count); +#else configTime(0, 0, effectiveNtpPrimary(_obs)); +#endif _ntp_refresh_retry_at = 0; _ntp_refresh_started_at = millis(); _ntp_refresh_pending = true; @@ -4473,7 +4678,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { const uint32_t elapsed = now - _ntp_refresh_started_at; if (WiFi.status() != WL_CONNECTED) { cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN("NTP refresh aborted; WiFi disconnected"); return; } @@ -4514,7 +4719,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { // The ESP status is edge-like and may reset after it is read, so retry the // whole operation instead of waiting on a completion that was consumed. cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN( "NTP refresh returned implausible time %lld; retry scheduled", (long long)raw_system_time); @@ -4523,7 +4728,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { if (elapsed >= MQTT_NTP_REFRESH_TIMEOUT_MS) { cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN("NTP refresh timed out; retry scheduled"); } } @@ -4845,6 +5050,7 @@ void MQTTBridge::runNtpEstimateProbe() { bool MQTTBridge::requestNtpTimeEstimate() { if (!isRunning() || _stop_requested.load(std::memory_order_acquire) + || isNetworkSuspended() || WiFi.status() != WL_CONNECTED || _ntp_estimate_requested) return false; _ntp_estimate_done = false; _ntp_estimate_ok = false; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 783d69cab..daa672239 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -25,6 +25,7 @@ class MeshSNMPAgent; // Forward declaration #ifdef ESP_PLATFORM #include "helpers/esp32/SntpOperationCoordinator.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #include #include #include @@ -447,6 +448,17 @@ private: unsigned long _last_wifi_reconnect_attempt; uint8_t _wifi_reconnect_backoff_attempt; // 0..5 -> 15s, 30s, 60s, 120s, 300s; reset on connect unsigned long _last_slot_reconnect_ms; // guards against concurrent TLS handshakes (15 s inter-slot gap) +#ifdef ESP_PLATFORM + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + uint8_t _background_ntp_server = 0; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void serviceBackgroundNtp(uint32_t now); +#endif AlertFaultPolicy::OutageSnapshot wifiOutage() const { return AlertFaultPolicy::unpackOutageSnapshot( @@ -550,6 +562,7 @@ private: void getClientVersion(char* buffer, size_t buffer_size) const; void logMemoryStatus(); void refreshOriginFromPrefs(); + void applyTimezoneRules(); // begin()/end()-scoped PSRAM buffers. Each allocation is independent so a // transient heap shortage degrades to the existing stack fallback instead // of making the bridge unusable. @@ -606,6 +619,15 @@ public: void requestStop(); bool isStopping() const { return _lifecycle.isStopInProgress(); } bool isRunning() const override { return _initialized; } + // Configured and still scheduling recovery, but transports are quiesced. + // Do not conflate automatic suspension with a persisted bridge disable. + bool isNetworkSuspended() const { +#ifdef ESP_PLATFORM + return _network_suspended.load(std::memory_order_acquire); +#else + return false; +#endif + } void loop() override; void onPacketReceived(mesh::Packet *packet) override; void sendPacket(mesh::Packet *packet) override; diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 686c12f5d..694b0c865 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -368,6 +368,7 @@ WebConfigServer::WebConfigServer(Callbacks* callbacks, void* mqtt_prefs, bool ow WebConfigServer::~WebConfigServer() { detachRoutes(); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); closeTerminal(); delete _room_mailbox; if (_mux) vSemaphoreDelete(_mux); @@ -653,6 +654,14 @@ bool WebConfigServer::formatWiFiStatus( appendOtaSeederStatus(reply, reply_len); return true; } + if (mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive()) { + snprintf(reply, reply_len, + "> suspended; recovery every 30 minutes, SSID: %s", + configured ? ssid : "(not configured)"); + appendOtaSeederStatus(reply, reply_len); + return true; + } if (active && active->_mode == MODE_CONNECTING) { snprintf(reply, reply_len, "> connecting, SSID: %s", configured ? ssid : "(not configured)"); @@ -754,6 +763,35 @@ bool WebConfigServer::getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t return true; } +void WebConfigServer::noteNetworkActivity() { + // This is separate from _last_activity, which also tracks boot and captive + // portal events. Those events must not renew an idle LAN listener's lease. + _network_activity_until = WebConfigBatch::scheduleAt(millis(), 120000UL); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); +} + +void WebConfigServer::requestNetworkAccess() { + noteNetworkActivity(); + mesh::wifi::networkPower().requestProbe(); +} + +bool WebConfigServer::isNetworkForegroundActive(uint32_t now) const { + if (_mode == MODE_OFF && !_stopping) return false; + if (_mode == MODE_SETUP || _stopping || handlerRefCount() != 0) return true; + WCLock lock(_mux); + if (_batch_state == BATCH_PENDING || _setup_wifi_handoff_pending) return true; + if (_terminal && (_terminal->running || _terminal->queue.pending)) return true; + if (_room_mailbox && (_room_mailbox->state() == mesh::RoomWebMailbox::State::Pending + || _room_mailbox->state() == mesh::RoomWebMailbox::State::Running)) return true; + return _network_activity_until + && !WebConfigBatch::deadlineReached(now, _network_activity_until); +} + +void WebConfigServer::refreshNetworkPowerHold(uint32_t now) { + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, + isNetworkForegroundActive(now)); +} + // --------------------------------------------------------------------------- // Lifecycle // --------------------------------------------------------------------------- @@ -817,6 +855,9 @@ bool WebConfigServer::startSetupMode(char reply[]) { strcpy(reply, "Err: webconfig busy"); return false; } + // Publish the foreground owner before changing interfaces. The background + // worker must not power down the station underneath this AP startup. + requestNetworkAccess(); _retry_saved_wifi_in_setup = false; _setup_reconnect_in_progress = false; _setup_reconnect_deadline = 0; @@ -888,6 +929,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { } if (!ap_ok) { if (!promote_lan) stopOwnedWiFiRadio(); + refreshNetworkPowerHold(millis()); strcpy(reply, "Err: failed to start AP"); return false; } @@ -905,6 +947,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { _initial_setup = _wifi_ssid[0] == 0 && node.admin_password[0] != 0; _setup_started_at = millis(); _last_activity = _setup_started_at; + refreshNetworkPowerHold(_setup_started_at); // Keep the pre-AP scan results for the picker. Only an explicit browser // rescan needs to leave the AP's home channel after clients have joined. @@ -942,6 +985,8 @@ bool WebConfigServer::startLanMode(char reply[]) { } bool WebConfigServer::beginSavedStation(uint32_t now) { + if (mesh::wifi::networkPower().backgroundSuspended() + && !isNetworkForegroundActive(now)) return false; WiFi.mode(WIFI_STA); if (mesh::wifi::applyProtocolMask(WIFI_IF_STA) != ESP_OK) { return false; @@ -1020,6 +1065,7 @@ void WebConfigServer::requestStop() { _stopping = true; _stop_warn_at = WebConfigBatch::scheduleAt(millis(), STOP_WARN_MS); _stop_warned = false; + refreshNetworkPowerHold(millis()); } bool WebConfigServer::stopForOTA(char reply[]) { @@ -1095,6 +1141,8 @@ void WebConfigServer::finalizeTeardown() { _stats_json[0] = 0; _board_cmds = 0; _board_cmds_probed = false; + _network_activity_until = 0; + refreshNetworkPowerHold(millis()); if (_cb) _cb->onWebConfigStopped(); } @@ -1113,6 +1161,7 @@ void WebConfigServer::probeBoardCommands() { } void WebConfigServer::tick(uint32_t now) { + refreshNetworkPowerHold(now); if (_stopping) { uint32_t refs = handlerRefCount(); switch (WebConfigBatch::stopStep(refs, _stop_warned, _stop_warn_at, now)) { @@ -1137,6 +1186,12 @@ void WebConfigServer::tick(uint32_t now) { serviceTerminal(now); serviceRoom(now); + // The MQTT/NTP owner has stopped its transports before publishing this + // state. An idle portal must not start an independent reconnect loop while + // those consumers are in their shared thirty-minute recovery cooldown. + if (mesh::wifi::networkPower().backgroundSuspended() + && !isNetworkForegroundActive(now)) return; + // A primary ESP-NOW radio and its WiFi station cannot remain on different // channels. Reject a router-driven channel move (for example a CSA) and put // the mesh radio back on its boot channel before any reconnect work runs. @@ -1614,6 +1669,8 @@ void WebConfigServer::dispatchRequest(AsyncWebServerRequest* req, return; } + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + (target->*handler)(req); portENTER_CRITICAL(&s_wc_route_mux); @@ -1729,6 +1786,7 @@ static bool roomRequestIdentity(AsyncWebServerRequest* req, uint8_t (&token)[32] void WebConfigServer::handleRoomPage(AsyncWebServerRequest* req) { if (_mode == MODE_OFF || !_cb || !_cb->supportsRoomService()) { req->send(404); return; } _last_activity = millis(); + noteNetworkActivity(); auto* response = new (std::nothrow) WebConfigPacedProgmemResponse( "text/html; charset=utf-8", reinterpret_cast(ROOM_WEB_HTML), sizeof(ROOM_WEB_HTML) - 1); if (!response) { req->send(503); return; } @@ -1765,6 +1823,7 @@ void WebConfigServer::handleRoomPost(AsyncWebServerRequest* req) { } if (result == mesh::RoomWebMailbox::Result::Accepted) { _last_activity = millis(); + noteNetworkActivity(); req->send(202, "application/json", "{\"accepted\":true}"); } else if (result == mesh::RoomWebMailbox::Result::Busy) { req->send(429, "application/json", "{\"error\":\"room service busy\"}"); @@ -1814,7 +1873,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { // saved WiFi network must not grant that same password-reset privilege. if (!req || !req->client() || req->client()->localIP() != WiFi.softAPIP()) return false; - if (_initial_setup || _wifi_ssid[0] == 0) return true; + if (_initial_setup || _wifi_ssid[0] == 0) { noteNetworkActivity(); return true; } } else if (_mode != MODE_LAN) { return false; } @@ -1823,7 +1882,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { WCLock lock(_mux); _cb->getNodeSnapshot(node); } - if (node.admin_password[0] == 0) return true; + if (node.admin_password[0] == 0) { noteNetworkActivity(); return true; } if (_session_token[0] == 0) return false; if (!req->hasHeader("Cookie")) return false; const String& cookies = req->getHeader("Cookie")->value(); @@ -1834,6 +1893,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { if ((uint32_t)(now - _session_last_seen) > WEBCONFIG_SESSION_TTL_MS) return false; if (!fixedTimeEquals(token.c_str(), _session_token, 32)) return false; _session_last_seen = now; // sliding expiry + noteNetworkActivity(); return true; } @@ -1857,6 +1917,7 @@ static void webConfigRejectEncoding(AsyncWebServerRequest* req) { void WebConfigServer::handleRoot(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); if (!webConfigClientAcceptsGzip(req)) { webConfigRejectEncoding(req); return; } if (req->hasHeader("If-None-Match") && req->getHeader("If-None-Match")->value() == WEBCONFIG_HTML_ETAG) { @@ -1876,6 +1937,7 @@ void WebConfigServer::handleRoot(AsyncWebServerRequest* req) { void WebConfigServer::handleUi(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); if (!req->hasParam("v") || req->getParam("v")->value() != WEBCONFIG_HTML_VERSION) { // Part URLs are immutable. Never serve a new layout under an old loader's @@ -1988,6 +2050,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { return; } if (_initial_setup || _wifi_ssid[0] == 0) { + noteNetworkActivity(); req->send(200, "application/json", "{\"ok\":true}"); return; } @@ -1998,6 +2061,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { _cb->getNodeSnapshot(node); } if (node.admin_password[0] == 0) { + noteNetworkActivity(); req->send(200, "application/json", "{\"ok\":true}"); return; } @@ -2026,6 +2090,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { _login_lock_until = 0; for (int i = 0; i < 4; i++) sprintf(&_session_token[i * 8], "%08lx", (unsigned long)esp_random()); _session_last_seen = now; + noteNetworkActivity(); AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); char cookie[80]; @@ -2951,6 +3016,7 @@ void WebConfigServer::handleScan(AsyncWebServerRequest* req) { void WebConfigServer::handlePresets(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); #ifdef WITH_MQTT_BRIDGE if (_mqtt_prefs == NULL) { req->send(200, "application/json", "{\"presets\":[]}"); diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index aa67aa0f5..4974c7439 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -36,6 +36,7 @@ #include #include #include +#include #include class AsyncWebServer; @@ -252,6 +253,12 @@ public: bool stopForOTA(char reply[]); void tick(uint32_t now); // call every loop iteration + // A listening LAN portal is not itself a reason to keep a failed background + // network awake. Explicit starts and real browser activity get a bounded + // window; AP setup and in-flight work retain the radio until they complete. + void requestNetworkAccess(); + bool isNetworkForegroundActive(uint32_t now) const; + Mode mode() const { return _mode; } bool isRunning() const { return _mode != MODE_OFF; } bool isStopping() const { return _stopping; } @@ -367,6 +374,7 @@ private: uint32_t _diag_last = 0; volatile uint32_t _last_activity = 0; + volatile uint32_t _network_activity_until = 0; uint32_t _reboot_at = 0; // 0 = none scheduled uint32_t _stop_warn_at = 0; bool _stop_warned = false; @@ -378,6 +386,8 @@ private: bool _board_cmds_probed = false; void createServer(); + void noteNetworkActivity(); + void refreshNetworkPowerHold(uint32_t now); bool beginSavedStation(uint32_t now); void registerRoutes(); typedef void (WebConfigServer::*RequestHandler)(AsyncWebServerRequest*); diff --git a/src/helpers/esp32/WiFiOtaSeeder.cpp b/src/helpers/esp32/WiFiOtaSeeder.cpp index 1924e1b09..c4b18eef6 100644 --- a/src/helpers/esp32/WiFiOtaSeeder.cpp +++ b/src/helpers/esp32/WiFiOtaSeeder.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,7 @@ void detachTcpFolder(bool disconnected) { void stopClient(bool disconnected = false) { detachTcpFolder(disconnected); if (seeder_client) seeder_client.stop(); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); } } // namespace @@ -89,6 +91,7 @@ void WiFiOtaSeeder::loop() { // A different host transport may already own the replacement context. if (active && !active->folder_active) active->clear_folder_dest(); tcp_folder_attached = false; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); } if (seeder_client && seeder_client.connected()) return; @@ -125,6 +128,7 @@ void WiFiOtaSeeder::loop() { } tcp_folder_attached = true; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, true); char link_info[24]; snprintf(link_info, sizeof(link_info), "tcp %s", seeder_client.remoteIP().toString().c_str()); diff --git a/test/fixtures/mqtt_ap_ownership.cpp b/test/fixtures/mqtt_ap_ownership.cpp index bb4aed59b..5f0a40020 100644 --- a/test/fixtures/mqtt_ap_ownership.cpp +++ b/test/fixtures/mqtt_ap_ownership.cpp @@ -7,6 +7,7 @@ #include "helpers/AlertFaultPolicy.h" #include "helpers/MQTTConnectionPolicy.h" #include "helpers/WiFiPowerSave.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #define MQTT_DEBUG_PRINTLN(...) ((void)0) #define pdMS_TO_TICKS(value) (value) using TickType_t=uint32_t; @@ -69,6 +70,9 @@ namespace mesh{namespace wifi{ int s_wifi_disconnect_reason=0;uint32_t s_wifi_disconnect_time=0; unsigned long s_wifi_connected_at=0; struct MQTTBridge{ + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic_network_suspended{false}; + bool _network_espnow_restore_pending=false; bool _wifi_event_registered=false,_manage_wifi=true,_ntp_sync_pending=false; int _wifi_reconnect_backoff_attempt=0; std::atomic_ntp_synced{true},_stop_requested{false},_stop_acked{false}; @@ -78,13 +82,14 @@ struct MQTTBridge{ void setWifiOutage(AlertFaultPolicy::OutageSnapshot value){outage=value;} char _wifi_ssid[8]="mesh",_wifi_password[8]="secret"; void beginWiFiStation(); + void applyTimezoneRules() {} bool _wifi_status_initialized=true; int _last_wifi_status=WL_DISCONNECTED; unsigned long _last_wifi_check=0,_last_wifi_reconnect_attempt=0; uint8_t _wifi_power_save=1; struct{bool canonical_wifi=true;}_node_info; struct{uint8_t wifi_power_save=1;}prefs,*_obs=&prefs; - struct Client{void disconnect(){}}; + struct Client{void disconnect(){}bool isStarted()const{return false;}}; struct Slot{Client* client=nullptr;bool connected=false;}_slots[RUNTIME_MQTT_SLOTS]; bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS]={}; std::vector cleanup; @@ -96,8 +101,11 @@ struct MQTTBridge{ bool handleWiFiConnection(unsigned long now); void mqttTaskLoop(){ @TASK_START@ + while(true){ @TASK_STOP@ - ++slot_work; + ++slot_work; + break; + } } }; @INITIALIZE@ diff --git a/test/fixtures/rs232_mqtt_runtime.cpp b/test/fixtures/rs232_mqtt_runtime.cpp index af380185b..2904ec461 100644 --- a/test/fixtures/rs232_mqtt_runtime.cpp +++ b/test/fixtures/rs232_mqtt_runtime.cpp @@ -113,6 +113,7 @@ struct MQTTNodeInfo { class MQTTBridge : public AbstractBridge { bool running = false; bool stopping = false; + bool network_suspended = false; public: inline static unsigned live = 0, fail_starts = 0; unsigned begins = 0, ends = 0, loops = 0, sent = 0, received = 0; @@ -127,12 +128,21 @@ public: assert(filters); ++begins; if (fail_starts) { --fail_starts; return; } running = true; + network_suspended = false; } - void end() override { ++ends; running = stopping = false; } + void end() override { ++ends; running = stopping = network_suspended = false; } void requestStop() { stopping = running; } bool isStopping() const { return stopping; } bool isRunning() const override { return running; } - void loop() override { ++loops; if (stopping) { ++ends; running = stopping = false; } } + bool isNetworkSuspended() const { return network_suspended; } + void setNetworkSuspended(bool paused) { + assert(running && !stopping); + network_suspended = paused; + } + void loop() override { + ++loops; + if (stopping) { ++ends; running = stopping = network_suspended = false; } + } void sendPacket(mesh::Packet* p) override { if (allow(p)) ++sent; } void onPacketReceived(mesh::Packet* p) { if (allow(p)) ++received; } }; @@ -270,6 +280,29 @@ int main() { expect_get(cli, "bridge.type", "> mqtt+rs232+espnow"); expect_get(cli, "rs232.enabled", "> off"); expect_get(cli, "rs232.running", "> off"); + fresh.mqtt_bridge->setNetworkSuspended(true); + assert(fresh.isMqttBridgeRunning() && fresh._prefs.bridge_enabled); + assert(fresh.bridgesPreventSleep()); // ESP-NOW retains its radio owner. + assert(fresh.setEspNowBridgeState(false)); + assert(!fresh.bridgesPreventSleep()); // Paused MQTT is still configured. + assert(fresh.setRs232BridgeState(true)); + assert(fresh.bridgesPreventSleep()); // An active UART remains independent. + assert(fresh.setRs232BridgeState(false)); + assert(!fresh.bridgesPreventSleep()); + fresh.mqtt_bridge->setNetworkSuspended(false); + assert(fresh.bridgesPreventSleep()); // The recovery probe owns the radio. + char pause_reply[256] = {}; + cli.set("espnow.enabled off", pause_reply); + assert(strcmp(pause_reply, "OK") == 0); + fresh.mqtt_bridge->setNetworkSuspended(true); + assert(fresh.requestMqttBridgeStop()); + assert(fresh.isMqttBridgeStopping() && fresh.bridgesPreventSleep()); + fresh.serviceBridges(); + assert(!fresh.isMqttBridgeRunning() && !fresh.bridgesPreventSleep()); + assert(fresh.setBridgeState(true)); + assert(fresh.isMqttBridgeRunning() && fresh.bridgesPreventSleep()); + cli.set("espnow.enabled on", pause_reply); + assert(strcmp(pause_reply, "OK") == 0 && fresh.isEspNowBridgeRunning()); } MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh); char reply[256] = {}; diff --git a/test/fixtures/webconfig_ap_start.cpp b/test/fixtures/webconfig_ap_start.cpp index 9ab27d962..65f3a294e 100644 --- a/test/fixtures/webconfig_ap_start.cpp +++ b/test/fixtures/webconfig_ap_start.cpp @@ -337,6 +337,8 @@ struct WebConfigServer { bool auth_allowed = true; ~WebConfigServer() { delete _dns; } void createServer() { ++servers; } + void requestNetworkAccess() {} + void refreshNetworkPowerHold(uint32_t) {} bool startSetupMode(char reply[]); bool checkAuth(AsyncWebServerRequest*) const { return auth_allowed; } void handleScan(AsyncWebServerRequest* req); diff --git a/test/fixtures/webconfig_ota_handoff.cpp b/test/fixtures/webconfig_ota_handoff.cpp index 25929c8cd..1d9bdd0f1 100644 --- a/test/fixtures/webconfig_ota_handoff.cpp +++ b/test/fixtures/webconfig_ota_handoff.cpp @@ -5,11 +5,14 @@ #include #include #include +#include static uint32_t millis() { return 123; } struct WebConfigBatch { static uint32_t scheduleAt(uint32_t now, uint32_t delay) { return now + delay; } + static bool deadlineReached(uint32_t now, uint32_t at) { return int32_t(now - at) >= 0; } }; +struct WCLock { explicit WCLock(void*) {} }; constexpr int WIFI_STA = 1; constexpr int WIFI_AP = 2; struct WiFiFake { @@ -29,7 +32,7 @@ struct DNS { void stop() {} }; struct Callbacks { unsigned stops = 0; void onWebConfigStopped() { ++stops; } }; struct WebConfigServer { enum Mode { MODE_OFF, MODE_LAN, MODE_SETUP }; - enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_CONFIG = 0 }; + enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_PENDING = 1, BATCH_CONFIG = 0 }; Server server; Server* _server = &server; DNS* _dns = nullptr; @@ -46,11 +49,18 @@ struct WebConfigServer { uint32_t _setup_reconnect_deadline = 0, _reboot_at = 0, _batch_state = 0; uint32_t _batch_next = 0, _setup_wifi_handoff_deadline = 0, _batch_kind = 0; uint32_t _board_cmds = 0; + uint32_t _network_activity_until = 0; + void* _mux = nullptr; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; char _wifi_ssid[32] = {}, _setup_wifi_handoff_ip[16] = {}; char _session_token[16] = {}, _stats_json[16] = {}; bool routes = true; void detachRoutes() { routes = false; } void closeTerminal() {} + uint32_t handlerRefCount() const { return 0; } + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); bool isRunning() const { return _mode != MODE_OFF; } bool isStopping() const { return _stopping; } void requestStop(); @@ -75,6 +85,7 @@ int main() { assert(portal.stopForOTA(reply)); assert(strcmp(reply, "WebConfig stopped") == 0); assert(!portal.routes && !portal.server.listening && portal.isStopping()); + assert(mesh::wifi::networkPower().foregroundActive()); assert(WiFi.sta && !WiFi.ap && WiFi.stops == 0); assert(WiFi.connected == !setup); // OTA can now bind port 80 and create its own AP, while HTTP request @@ -86,6 +97,7 @@ int main() { assert(WiFi.ap && WiFi.sta && WiFi.stops == 0); assert(WiFi.ap_stops == unsigned(setup)); assert(portal.callbacks.stops == 1 && !portal._keep_wifi_on_stop); + assert(!mesh::wifi::networkPower().foregroundActive()); } } WebConfigServer ordinary; diff --git a/test/fixtures/webconfig_recovery_auth.cpp b/test/fixtures/webconfig_recovery_auth.cpp index 8833f2941..a23229e16 100644 --- a/test/fixtures/webconfig_recovery_auth.cpp +++ b/test/fixtures/webconfig_recovery_auth.cpp @@ -47,6 +47,8 @@ struct WebConfigServer { uint32_t _last_activity=0, _session_last_seen=0, _login_lock_until=0; int _login_fails=0; Callback callback; Callback* _cb=&callback; WebConfigServer() { strcpy(callback.node.admin_password, "existing-secret"); } + unsigned network_activity = 0; + void noteNetworkActivity() { ++network_activity; } bool checkAuth(AsyncWebServerRequest*); void handleLogin(AsyncWebServerRequest*); void handleLogout(AsyncWebServerRequest*); @@ -58,12 +60,15 @@ int main() { attacker.body="{\"set\":{\"password\":\"attacker\",\"wifi.ssid\":\"EvilNet\"}}"; attacker._tempObject=attacker.body.data(); assert(!portal.checkAuth(&attacker)); + assert(portal.network_activity==0); portal.checkConfigPost(&attacker); assert(attacker.status==401 && !portal.config_accepted); attacker.password("wrong"); portal.handleLogin(&attacker); assert(attacker.status==401 && portal._session_token[0]==0); + assert(portal.network_activity==0); AsyncWebServerRequest admin; admin.password("existing-secret"); portal.handleLogin(&admin); assert(admin.status==200); + assert(portal.network_activity>0); assert(strlen(portal._session_token)==32); assert(admin.response.headers["Set-Cookie"].find("HttpOnly; SameSite=Lax")!=std::string::npos); admin.header.cookies=std::string("wcs=")+portal._session_token; diff --git a/test/fixtures/webconfig_station_owner.cpp b/test/fixtures/webconfig_station_owner.cpp index bd96c7fdf..0888787e1 100644 --- a/test/fixtures/webconfig_station_owner.cpp +++ b/test/fixtures/webconfig_station_owner.cpp @@ -23,6 +23,7 @@ struct WebConfigServer { bool isRunning() const { return false; } bool isStopping() const { return false; } void updateWiFiOwnership(bool owner) { owns = owner; } + void requestNetworkAccess() {} bool startAutoMode(char* reply) { strcpy(reply, "starting"); return true; } bool startSetupMode(char* reply) { strcpy(reply, "setup"); return true; } }; diff --git a/test/fixtures/webconfig_station_recovery.cpp b/test/fixtures/webconfig_station_recovery.cpp index f0f5bbdaa..6857ac024 100644 --- a/test/fixtures/webconfig_station_recovery.cpp +++ b/test/fixtures/webconfig_station_recovery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include static uint32_t clock_ms = 1000; uint32_t millis() { return clock_ms; } void delay(unsigned) {} @@ -96,6 +97,7 @@ public: bool _stopping = false, _retry_saved_wifi_in_setup = false; bool _setup_reconnect_in_progress = false, _board_cmds_probed = false; bool _was_setup_ap = false, _initial_setup = false; + bool foreground = false; uint32_t _setup_reconnect_deadline = 0, _connect_deadline = 0; uint32_t _setup_started_at = 0, _last_activity = 0; WiFiReconnectPolicy::Tracker _wifi_reconnect_tracker; @@ -134,6 +136,7 @@ public: void probeBoardCommands() { _board_cmds_probed = true; } void serviceTerminal(uint32_t) {} void serviceRoom(uint32_t) {} + bool isNetworkForegroundActive(uint32_t) const { return foreground || _mode == MODE_SETUP; } bool bluetoothWiFiCoexistenceRequired() { return false; } bool acceptWiFi(Entry& e, const char* value) { @CREDENTIAL_BATCH@ @@ -197,6 +200,33 @@ int main() { companion.tick(16000); assert(companion.setup_starts == 1 && companion._retry_saved_wifi_in_setup); assert(!companion._runtime_wifi_ownership_managed); +#elif defined(TEST_BACKGROUND_SUSPEND) + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + portal._mode = WebConfigServer::MODE_LAN; + clock_ms = 301000; + portal.tick(clock_ms); + assert(WiFi.begins == 0 && portal.setup_starts == 0); + // A connecting listener must not reopen an AP or acquire STA while the + // background owner has deliberately stopped MQTT and NTP for the cooldown. + portal._mode = WebConfigServer::MODE_CONNECTING; + portal._connect_deadline = 16000; + portal.tick(clock_ms); + assert(WiFi.begins == 0 && portal.setup_starts == 0); + portal._connect_deadline = 0; + portal.foreground = true; + portal.tick(++clock_ms); + assert(WiFi.begins == 1 && portal._connect_deadline == clock_ms + 15000); + portal.foreground = false; + clock_ms += 16000; + portal.tick(clock_ms); + assert(portal.setup_starts == 0 && WiFi.begins == 1); + // Resuming the worker permits ordinary ownership/reconnect logic again. + power.setSuspended(false); + portal._mode = WebConfigServer::MODE_LAN; + clock_ms += 301000; + portal.tick(clock_ms); + assert(WiFi.begins == 2); #elif defined(TEST_EXTERNAL_OWNER) portal.updateWiFiOwnership(false); assert(portal._runtime_wifi_ownership_managed); diff --git a/test/test_companion_wifi_network_power.py b/test/test_companion_wifi_network_power.py new file mode 100644 index 000000000..85ee389c4 --- /dev/null +++ b/test/test_companion_wifi_network_power.py @@ -0,0 +1,474 @@ +"""Execute Companion's production outage transitions without hardware.""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from test_replay_reset_integration import extract_braced + + +ROOT = Path(__file__).resolve().parents[1] +MAIN = ROOT / "examples/companion_radio/main.cpp" + + +HARNESS = r''' +#include +#include +#include +#include + +namespace mesh { namespace wifi { +bool espnow = false, station_allowed = true; +int protocols = 0, channels = 0, associations = 0, restore_result = 0; +bool espNowChannelConstrained() { return espnow; } +bool stationMutationAllowed() { return station_allowed; } +void applyProtocolMask(int) { ++protocols; } +int restoreEspNowChannel() { ++channels; return restore_result; } +void setStationAutoReconnect(bool); +void beginStation(const char*, const char*) { ++associations; } +}} + +constexpr int WIFI_STA = 1, WIFI_OFF = 0, WIFI_IF_STA = 0, TCP_PORT = 5000; +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6; +constexpr int ESP_OK = 0; +uint32_t test_now = 10000; +uint32_t millis() { return test_now; } +struct Board { + bool ota = false, inhibit = true; + bool isOTAUpdateRunning() const { return ota; } + void setInhibitSleep(bool value) { inhibit = value; } +} board; +struct Interface { + bool connected = false, enabled = true; + int ends = 0, begins = 0; + bool isConnected() const { return connected; } + void end() { enabled = false; ++ends; } + void begin(int) { ++begins; } + void enable() { enabled = true; } +} wifi_interface; +struct ConsoleClient { + bool active = false; + bool connected() const { return active; } +} ota_console_client; +struct Wifi { + int current_mode = WIFI_STA; + int current_status = WL_DISCONNECTED; + bool reconnect = true, last_disconnect_stopped = false; + int disconnects = 0; + void setAutoReconnect(bool value) { reconnect = value; } + void disconnect(bool stop, bool) { + last_disconnect_stopped = stop; + ++disconnects; + } + void mode(int value) { current_mode = value; } + int status() const { return current_status; } +} WiFi; +void mesh::wifi::setStationAutoReconnect(bool value) { WiFi.setAutoReconnect(value); } + +struct Tracker { void noteDisconnected(uint32_t) {} } wifi_reconnect_tracker; +bool companion_wifi_active = true, companion_wifi_requested = true; +bool companion_wifi_background_stopped = false; +bool companion_wifi_espnow_restore_pending = false; +uint32_t companion_wifi_espnow_restore_at = 0; +bool companion_wifi_webconfig_start_pending = false; +bool companion_wifi_credential_reload_pending = false; +bool companion_wifi_setup_requested = false; +bool companion_wifi_has_credentials = true; +char configured_wifi_ssid[] = "saved-network"; +char configured_wifi_password[] = "saved-password"; +int ntp_cancels = 0, resets = 0, ps_applied = 0, console_starts = 0, console_stops = 0; +void cancelCompanionWiFiNtp(bool retry) { assert(!retry); ++ntp_cancels; } +void resetCompanionWiFiRecoveryState() { ++resets; } +void applyCompanionWiFiPowerSave() { ++ps_applied; } +void ota_console_start() { ++console_starts; } +void ota_console_stop() { ++console_stops; } +#define WIFI_DEBUG_PRINTLN(...) do {} while (0) +struct WebConfigServer { + static bool enabled; + static bool loadEnabled(bool) { return enabled; } +}; +bool WebConfigServer::enabled = true; +struct Mesh { + bool web_active = false, last_force_ap = false; + int web_starts = 0; + bool isWebConfigActiveOrStopping() const { return web_active; } + bool startWebConfig(bool force_ap, char*) { + ++web_starts; last_force_ap = force_ap; web_active = true; return true; + } +} the_mesh; + +@METHODS@ + +void reset() { + auto& power = mesh::wifi::networkPower(); + power.setManaged(false); + power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + power.setManaged(true); + power.setSuspended(false); + mesh::wifi::espnow = false; + mesh::wifi::station_allowed = true; + mesh::wifi::protocols = mesh::wifi::channels = mesh::wifi::associations = 0; + mesh::wifi::restore_result = 0; + test_now = 10000; + board = {}; wifi_interface = {}; ota_console_client = {}; WiFi = {}; + companion_wifi_active = companion_wifi_requested = true; + companion_wifi_background_stopped = false; + companion_wifi_espnow_restore_pending = false; + companion_wifi_espnow_restore_at = 0; + companion_wifi_webconfig_start_pending = false; + companion_wifi_credential_reload_pending = companion_wifi_setup_requested = false; + companion_wifi_has_credentials = true; + ntp_cancels = resets = ps_applied = console_starts = console_stops = 0; + the_mesh = {}; WebConfigServer::enabled = true; +} + +int main() { + reset(); + // Merely listening for a client does not hold the radio on. + serviceCompanionWiFiNetworkPowerHold(); + assert(!mesh::wifi::networkPower().foregroundActive()); + wifi_interface.connected = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + wifi_interface.connected = false; +#ifdef ENABLE_OTA + ota_console_client.active = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + ota_console_client.active = false; +#endif + companion_wifi_credential_reload_pending = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + companion_wifi_credential_reload_pending = false; + companion_wifi_setup_requested = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + companion_wifi_setup_requested = false; + board.ota = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + + reset(); + mesh::wifi::networkPower().setSuspended(true); + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped); + assert(companion_wifi_active && companion_wifi_requested); + assert(!board.inhibit && !WiFi.reconnect); + assert(WiFi.current_mode == WIFI_OFF && WiFi.last_disconnect_stopped); + assert(wifi_interface.ends == 1 && ntp_cancels == 1); +#ifdef ENABLE_OTA + assert(console_stops == 1); +#endif + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(wifi_interface.ends == 1 && ntp_cancels == 1); + assert(mesh::wifi::associations == 0); + assert(strcmp(configured_wifi_ssid, "saved-network") == 0); + assert(strcmp(configured_wifi_password, "saved-password") == 0); + + // The worker owns the cooldown. Once it opens a recovery window, the + // externally owned Companion station starts exactly one saved association. + mesh::wifi::networkPower().setSuspended(false); + mesh::wifi::station_allowed = false; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped && mesh::wifi::associations == 0); + mesh::wifi::station_allowed = true; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && board.inhibit); + assert(wifi_interface.enabled && wifi_interface.begins == 1); + assert(mesh::wifi::associations == 1 && WiFi.reconnect); + assert(WiFi.current_mode == WIFI_STA && ps_applied == 1); +#ifdef ENABLE_OTA + assert(console_starts == 1); +#endif + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::associations == 1 && wifi_interface.begins == 1); + + // An active updater remains powered even if it starts between the worker's + // last ownership check and publication of the suspension flag. + reset(); + mesh::wifi::networkPower().setSuspended(true); + board.ota = true; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && board.inhibit); + assert(wifi_interface.ends == 0 && WiFi.disconnects == 0); + + // Primary and dynamically enabled ESP-NOW both need their driver/channel. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped && !WiFi.reconnect); + assert(board.inhibit); + assert(WiFi.current_mode == WIFI_STA && !WiFi.last_disconnect_stopped); + assert(mesh::wifi::protocols == 1 && mesh::wifi::channels == 1); + + // Disconnect is asynchronous. The first restoration can fail until the + // station/scan ends; keep retrying the channel, never the disconnect. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_espnow_restore_pending && WiFi.disconnects == 1); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1 && WiFi.disconnects == 1); + test_now += 999; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1); + ++test_now; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && companion_wifi_espnow_restore_pending); + assert(WiFi.disconnects == 1 && mesh::wifi::protocols == 1); + test_now += 1000; + mesh::wifi::station_allowed = false; // a foreground AP owns the channel + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2); + mesh::wifi::station_allowed = true; + board.ota = true; + serviceCompanionEspNowChannelRestore(); + assert(mesh::wifi::channels == 2); + board.ota = false; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, true); + serviceCompanionEspNowChannelRestore(); + assert(mesh::wifi::channels == 2); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 3 && !companion_wifi_espnow_restore_pending); + test_now += 10000; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 3 && WiFi.disconnects == 1); + + // Retry deadlines remain safe when millis wraps during the disconnect. + reset(); + test_now = UINT32_MAX - 500; + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + test_now += 999; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1); + ++test_now; + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending); + + // The same deferred repair runs after an intentional WiFi disable, without + // restarting that station or its listeners. A removed ESP-NOW owner cancels it. + reset(); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + stopCompanionInfrastructureWiFi(); + companion_wifi_requested = companion_wifi_active = false; + test_now += 1000; + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending); + assert(WiFi.disconnects == 1 && mesh::wifi::associations == 0); + mesh::wifi::restore_result = 1; + stopCompanionInfrastructureWiFi(); + mesh::wifi::espnow = false; + test_now += 1000; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_espnow_restore_pending && mesh::wifi::channels == 3); + + // Beginning a recovery association cancels a delayed channel write. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + mesh::wifi::networkPower().setSuspended(false); + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_espnow_restore_pending && mesh::wifi::associations == 1); + + // A client can be accepted immediately before a stale worker suspension + // flag becomes visible. Its fresh foreground hold prevents teardown. + reset(); + mesh::wifi::networkPower().setSuspended(true); + wifi_interface.connected = true; + serviceCompanionWiFiNetworkPowerHold(); + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && WiFi.disconnects == 0); + assert(wifi_interface.ends == 0 && board.inhibit); + + // User-disabled or not-yet-active WiFi is never restarted by recovery. + reset(); + companion_wifi_requested = false; + mesh::wifi::networkPower().setSuspended(true); + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + companion_wifi_requested = true; + companion_wifi_active = false; + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + + // A plain WiFi Companion with no managed MQTT worker keeps its existing + // connection and standalone NTP policy; a stale flag cannot suspend it. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::networkPower().setManaged(false); + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + + // A saved-SSID boot cannot start the portal's independent fifteen-second + // AP fallback before the MQTT worker acquires its network policy. + reset(); + mesh::wifi::networkPower().setManaged(false); + startCompanionWiFiAutoWebConfig(); + assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig(); + assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0); + mesh::wifi::networkPower().setManaged(true); + assert(!companionWiFiAutomaticRecoveryAllowed()); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 0); + WiFi.current_status = WL_CONNECTED; + serviceCompanionWiFiAutoWebConfig(); + assert(!companion_wifi_webconfig_start_pending && the_mesh.web_starts == 1); + assert(!the_mesh.last_force_ap); + serviceCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 1); + + // Unconfigured devices still open setup, and unmanaged standalone WiFi + // retains its ordinary recovery AP. Explicit manual setup is independent. + reset(); + companion_wifi_has_credentials = false; + startCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 1 && the_mesh.last_force_ap); + assert(!companion_wifi_webconfig_start_pending); + mesh::wifi::networkPower().setManaged(false); + assert(companionWiFiAutomaticRecoveryAllowed()); +} +''' + + +class CompanionWiFiNetworkPowerTests(unittest.TestCase): + def test_router_down_boot_starts_the_configured_mqtt_scheduler_once(self): + source = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text(encoding="ascii") + actual = extract_braced(source, "static void copyMqttString(") + actual += extract_braced(source, "void MyMesh::serviceMQTT(") + fixture = r''' +#define WITH_WEBCONFIG 1 +#include +#include +#include +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6; +struct { int status() const { return WL_DISCONNECTED; } } WiFi; +namespace mesh { struct Log { void println(const char*) {} }; +inline Log& usbLoggingPort() { static Log log; return log; } } +struct Bridge { + int starts = 0; bool running = false; + void begin() { ++starts; running = true; mesh::wifi::networkPower().setManaged(true); } + bool isRunning() const { return running; } +}; +struct MyMesh { + bool _mqtt_enabled = true, _wc_batch_active = false; + bool _mqtt_started = false, _mqtt_configured = true, setup = false; + int stops = 0; + struct { char wifi_ssid[33] = {}, wifi_password[65] = {}; } _mqtt_prefs; + Bridge* _mqtt_bridge; + void stopMQTT() { ++stops; } + bool isWebConfigSetupActive() const { return setup; } + void serviceMQTT(const char*, const char*); +}; +''' + scenarios = r''' +int main() { + Bridge bridge; MyMesh node; node._mqtt_bridge = &bridge; + assert(WiFi.status() == WL_DISCONNECTED); + node._wc_batch_active = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node._wc_batch_active = false; node.setup = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node.setup = false; node._mqtt_configured = false; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node._mqtt_configured = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 1 && node._mqtt_started); + assert(mesh::wifi::networkPower().managed()); + assert(strcmp(node._mqtt_prefs.wifi_ssid, "saved-network") == 0); + assert(strcmp(node._mqtt_prefs.wifi_password, "saved-password") == 0); + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 1); + node._mqtt_enabled = false; + node.serviceMQTT("saved-network", "saved-password"); + assert(node.stops == 1); +} +''' + with tempfile.TemporaryDirectory(prefix="companion-router-outage-") as directory: + path = Path(directory) + cpp, binary = path / "fixture.cpp", path / "fixture" + cpp.write_text(fixture + actual + scenarios, encoding="ascii") + result = subprocess.run(["g++", "-std=c++11", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_production_transitions_with_and_without_ota_console(self): + compiler = os.environ.get("CXX") or shutil.which("g++") + self.assertIsNotNone(compiler) + source = MAIN.read_text(encoding="ascii") + methods = "\n".join(extract_braced(source, signature) for signature in ( + "static bool companionWiFiBackgroundSuspended(", + "static bool companionWiFiAutomaticRecoveryAllowed(", + "static void startCompanionWiFiAutoWebConfig(", + "static void serviceCompanionWiFiAutoWebConfig(", + "static void serviceCompanionWiFiNetworkPowerHold(", + "static void stopCompanionInfrastructureWiFi(", + "static void serviceCompanionEspNowChannelRestore(", + "static void serviceCompanionWiFiBackgroundPower(", + )) + sanitizer = (["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] if sys.platform.startswith("linux") else []) + with tempfile.TemporaryDirectory(prefix="companion-network-power-") as directory: + path = Path(directory) + cpp = path / "fixture.cpp" + cpp.write_text(HARNESS.replace("@METHODS@", methods), encoding="ascii") + for ota in (False, True): + with self.subTest(ota=ota): + binary = path / ("with-ota" if ota else "no-ota") + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-Werror", *sanitizer, *(["-DENABLE_OTA=1"] if ota else []), + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + + def test_background_paths_and_manual_recovery_use_the_shared_guard(self): + source = MAIN.read_text(encoding="ascii") + for signature in ("static void serviceCompanionWiFiCredentialReload(", + "static void startCompanionWiFi("): + body = extract_braced(source, signature) + self.assertLess(body.index("companionWiFiBackgroundSuspended()"), + body.index("WiFi.mode(WIFI_STA)")) + ntp = extract_braced(source, "static void serviceCompanionWiFiNtp(") + self.assertLess(ntp.index("networkPower().backgroundSuspended()"), + ntp.index("companion_wifi_ntp_operation.tryAcquire()")) + for signature in ("void requestCompanionWiFiSetup(", + "void scheduleCompanionWiFiCredentialReload(", + "bool setCompanionWiFiEnabled("): + self.assertIn("networkPower().requestProbe()", extract_braced(source, signature)) + self.assertIn("&& !companionWiFiBackgroundSuspended()\n" + " && !companion_wifi_background_stopped", source) + self.assertIn("if (!wifi_setup_recovery_mode && companionWiFiAutomaticRecoveryAllowed()", source) + start = extract_braced(source, "static void startCompanionWiFi(") + self.assertIn("startCompanionWiFiAutoWebConfig();", start) + self.assertNotIn("startWebConfig(", start) + self.assertNotIn("companion_wifi_webconfig_start_pending = false", start) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_companion_wifi_ntp.py b/test/test_companion_wifi_ntp.py index 837ee59ce..7ebe40280 100644 --- a/test/test_companion_wifi_ntp.py +++ b/test/test_companion_wifi_ntp.py @@ -119,7 +119,14 @@ class CompanionWiFiNtpIntegrationTests(unittest.TestCase): self.assertIn("_ntp_sync_pending = true", initialize) self.assertIn("_ntp_reconnect_latch.consumeIfConnected", task_loop) self.assertIn("_ntp_sync_pending = true", task_loop) - self.assertIn("syncTimeWithNTP();", task_loop) + self.assertIn("serviceBackgroundNtp(static_cast(now));", task_loop) + self.assertNotIn("syncTimeWithNTP();", task_loop) + background = mqtt[ + mqtt.index("void MQTTBridge::serviceBackgroundNtp(") : + mqtt.index("void MQTTBridge::mqttTaskLoop()") + ] + self.assertIn("pollNtpRefresh(now);", background) + self.assertIn("refreshNTP();", background) self.assertIn("_last_ntp_sync = millis();", sync) self.assertIn( "fresh_ntp ? 0 : mqttNtpRetryAt(_last_ntp_sync)", sync diff --git a/test/test_esp32_full_power_lifecycle.py b/test/test_esp32_full_power_lifecycle.py index 2c1f58f52..ff337c89d 100644 --- a/test/test_esp32_full_power_lifecycle.py +++ b/test/test_esp32_full_power_lifecycle.py @@ -24,6 +24,7 @@ FIXTURE = r''' #include #include #include +#include #include uint32_t now = 0; uint32_t millis() { return now; } @@ -143,6 +144,8 @@ struct MQTTNodeInfo { uint8_t* sf; uint8_t* cr; uint8_t* repeat_flag; bool repeat_when_nonzero; bool canonical_wifi = false; }; struct MQTTBridge : AbstractBridge { + bool network_suspended = false; + bool isNetworkSuspended() const { return network_suspended; } template explicit MQTTBridge(Args...) {} void setDeviceID(const char*) {} void setFirmwareVersion(const char*) {} void setBoardModel(const char*) {} void setBuildDate(const char*) {} @@ -156,6 +159,8 @@ struct WebConfigServer { uint32_t _setup_started_at = 0; int _mode = 0; static constexpr int MODE_SETUP = 1; + static constexpr int MODE_LAN = 2; + uint32_t foreground_until = 0; char _wifi_ssid[33] = {}; template WebConfigServer(void*, void* prefs, bool owner, Args...) : owns(owner) { if (prefs) strcpy(_wifi_ssid, static_cast(prefs)->wifi_ssid); @@ -165,6 +170,14 @@ struct WebConfigServer { bool isRunning() const { return running; } bool isStopping() const { return stopping; } void updateWiFiOwnership(bool owner) { owns = owner; } + void requestNetworkAccess() { + foreground_until = millis() + 120000; + mesh::wifi::networkPower().requestProbe(); + } + bool isNetworkForegroundActive(uint32_t tick) const { + return _mode == MODE_SETUP || stopping + || (foreground_until && int32_t(tick - foreground_until) < 0); + } bool startSetupMode(char* reply) { if (!allow_start) { if (owns) stopOwnedWiFiRadio(); @@ -406,6 +419,51 @@ class ESP32FullPowerLifecycleTest(unittest.TestCase): with self.subTest(role=role): self.compile_and_run(self.fixture(role), *FLAGS) + def test_suspended_mqtt_idle_portal_and_foreground_ownership(self): + checks = r''' +int main() { + MyMesh node; char reply[160] = {}; + strcpy(node._cli.prefs.wifi_ssid, "working-network"); + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); + assert(node.setMqttBridgeState(true)); + assert(node.startWebConfig(false, reply)); + node._webconfig->_mode = WebConfigServer::MODE_LAN; + now += 120001; // Explicit-start lease has elapsed; listener is idle. + power.consumeProbeRequest(); + node.@WORKER@->network_suspended = true; + power.setSuspended(true); + unsigned off_before = WiFi.off_modes; + node.serviceIdleWiFi(); + assert(WiFi.off_modes == off_before && sdk_started); + assert(node.hasPendingWork()); // Cannot sleep before worker shutdown completes. + WiFi.mode(WIFI_OFF); // Only the Core 0 network owner performs this shutdown. + node.serviceIdleWiFi(); + assert(!sdk_initialized && !node.hasPendingWork()); + assert(node.getPowerSaveSleepSeconds(30) == 30); + // An operator can wake the existing listener without creating a second + // server, taking station ownership, or deinitializing the worker's driver. + assert(node.startWebConfig(false, reply)); + assert(power.consumeProbeRequest()); + assert(node._webconfig->isNetworkForegroundActive(now)); + assert(node.hasPendingWork()); + now += 120001; + assert(!node.hasPendingWork()); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, true); + assert(node.hasPendingWork()); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); + assert(!node.hasPendingWork()); + power.setSuspended(false); + node.@WORKER@->network_suspended = false; + node.serviceIdleWiFi(); + assert(node.hasPendingWork()); // Resume is owned by the MQTT worker too. + power.setManaged(false); +} +''' + for role in ("simple_repeater", "simple_room_server"): + with self.subTest(role=role): + self.compile_and_run(self.fixture(role, checks), *FLAGS) + def test_primary_radios_and_wifi_disabled_images_do_not_reclaim_wifi(self): for role in ("simple_repeater", "simple_room_server"): source = (ROOT / "examples" / role / "MyMesh.cpp").read_text() diff --git a/test/test_esp32_full_sleep.py b/test/test_esp32_full_sleep.py index 02711b304..67c991bf8 100644 --- a/test/test_esp32_full_sleep.py +++ b/test/test_esp32_full_sleep.py @@ -98,6 +98,7 @@ int main() { #include #include #include +#include static uint32_t now = 1000; uint32_t millis() { return now; } using esp_err_t = int; @@ -135,15 +136,22 @@ struct Interfaces { bool pending = false; bool hasPendingIO() const { return pen struct Sensors { bool gps_uart = false; bool gpsUsesSerialUart(uint8_t uart) const { return uart == 1 && gps_uart; } } sensors; struct Button { bool polling = false; bool needsPolling() const { return polling; } } user_btn; struct Wireless { unsigned services = 0; unsigned enabled() const { return services; } } companion_wireless; +static bool companion_wifi_background_stopped = false; namespace mesh { bool logging = false, watchdog = false; bool isUsbLoggingEnabled() { return logging; } bool isUsbLoggingWatchdogArmed() { return watchdog; } +namespace wifi { +bool espnow_constrained = false; +bool espNowChannelConstrained() { return espnow_constrained; } +} namespace wireless { +constexpr unsigned WiFi = 1, Bluetooth = 2, EspNow = 4; struct Control { bool transition = false; bool pending() const { return transition; } } controller; Control& control() { return controller; } } } +@SUSPENDED@ @READY@ void idle() { @BRANCH@ } template void blocks(T& flag) { @@ -170,6 +178,49 @@ int main() { blocks(mesh::watchdog); blocks(mesh::wireless::controller.transition); prefs.powersaving_enabled = false; before = sleeps; idle(); assert(sleeps == before); prefs.powersaving_enabled = true; idle(); assert(sleeps == before + 1); + // A requested WiFi service and idle WebConfig listener remain logically + // enabled during MQTT cooldown. Sleep needs transport quiescence and the + // station owner's physical stop, rather than a disabled saved preference. + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + companion_wireless.services = mesh::wireless::WiFi; + companion_wifi_background_stopped = true; + sdk_result = ESP_ERR_WIFI_NOT_INIT; + before = sleeps; idle(); assert(sleeps == before + 1); + for (int stopped_result : {ESP_ERR_WIFI_NOT_STARTED, ESP_OK}) { + sdk_result = stopped_result; sdk_mode = WIFI_MODE_NULL; + before = sleeps; idle(); assert(sleeps == before + 1); + } + sdk_result = ESP_ERR_WIFI_NOT_INIT; + blocks(mesh::wifi::espnow_constrained); + for (wifi_mode_t mode : {WIFI_MODE_STA, WIFI_MODE_AP, WIFI_MODE_APSTA}) { + sdk_mode = mode; sdk_result = ESP_OK; + before = sleeps; idle(); assert(sleeps == before); + } + sdk_mode = WIFI_MODE_NULL; sdk_result = 99; + before = sleeps; idle(); assert(sleeps == before); // Failed physical cleanup/read. + sdk_result = ESP_ERR_WIFI_NOT_INIT; + companion_wifi_background_stopped = false; + before = sleeps; idle(); assert(sleeps == before); // Owner has not stopped STA yet. + companion_wifi_background_stopped = true; + for (unsigned service : {mesh::wireless::Bluetooth, mesh::wireless::EspNow}) { + companion_wireless.services = mesh::wireless::WiFi | service; + before = sleeps; idle(); assert(sleeps == before); + } + companion_wireless.services = mesh::wireless::WiFi; + for (auto hold : {mesh::wifi::NetworkPowerHold::WebConfig, + mesh::wifi::NetworkPowerHold::CompanionClient, + mesh::wifi::NetworkPowerHold::OtaSeeder}) { + power.setHold(hold, true); + before = sleeps; idle(); assert(sleeps == before); + power.setHold(hold, false); + } + blocks(board.ota); blocks(bt_controller_active); blocks(bt_host_active); + power.setSuspended(false); + before = sleeps; idle(); assert(sleeps == before); // Worker has begun its probe. + power.setManaged(false); + companion_wifi_background_stopped = false; + companion_wireless.services = 0; sdk_mode = WIFI_MODE_NULL; sdk_result = ESP_OK; // The real board policy must observe USB even while pending work/logging // inhibit this branch. A rebooting host cannot disappear for one SOF gap // and cause Full to shut down its console immediately. @@ -179,6 +230,8 @@ int main() { now += 120000; idle(); assert(sleeps == before + 1); } '''.replace("@READY@", ready).replace("@BRANCH@", branch).replace("@USB_HELD@", usb_held) + source = source.replace("@SUSPENDED@", function(main, + "static bool companionWiFiBackgroundSuspended()")) source = "#include \n" + source flags = ("ESP32_PLATFORM", "ESP32", "ENABLE_USB_INTERFACE", "ARDUINO_USB_CDC_ON_BOOT=1", 'WIFI_SSID=""', "BLE_PIN_CODE=123456", "MESH_USB_LOGGING_AVAILABLE=1", @@ -189,7 +242,7 @@ int main() { with self.subTest(pm=pm): result = run_cpp(source, (*flags, f"COMPANION_IDF_PM_AVAILABLE={pm}")) self.assertEqual(result.returncode, 0, result.stderr) - broken = run_cpp(source.replace("companion_wireless.enabled() != 0", "false"), + broken = run_cpp(source.replace("enabled_services != 0", "false"), (*flags, "COMPANION_IDF_PM_AVAILABLE=0")) self.assertNotEqual(broken.returncode, 0) diff --git a/test/test_mqtt_disconnect.py b/test/test_mqtt_disconnect.py new file mode 100644 index 000000000..10fbbc7c3 --- /dev/null +++ b/test/test_mqtt_disconnect.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Run the real MQTT wrapper stop and outage handoff under SDK failures.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from test_mqtt_publish_ack import ARDUINO_HEADER, MQTT_HEADER +from test_mqtt_transport_results import method +from test_replay_reset_integration import extract_braced + + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER = "lib/PsychicMqttClient/src/PsychicMqttClient.cpp" +BRIDGE = "src/helpers/bridges/MQTTBridge.cpp" + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define private public +#include "PsychicMqttClient.h" +#undef private + +template void log_stub(Args&&...) {} +static const char* TAG = "test"; +#define ESP_LOGI(...) log_stub(__VA_ARGS__) +#define ESP_LOGW(...) log_stub(__VA_ARGS__) +#define ESP_ERROR_CHECK_WITHOUT_ABORT(value) ((void)(value)) +#define MQTT_DEBUG_PRINTLN(...) ((void)0) +#define pdMS_TO_TICKS(ms) ((ms) / portTICK_PERIOD_MS) +static uint32_t now_ms = 0, elapsed_ms = 0, tick_ms = 1; +static uint32_t callback_after_ms = UINT32_MAX; +static int disconnect_result = ESP_OK, stop_result = ESP_OK; +static int disconnect_calls = 0, stop_calls = 0, sleeps = 0; +static PsychicMqttClient* active_client = nullptr; +static std::function on_delay; +unsigned long millis() { return now_ms; } +const char* esp_err_to_name(int) { return "mock"; } + +static void deliverDisconnect() { + if (!active_client || elapsed_ms < callback_after_ms) return; + active_client->_connected = false; + active_client->_stopMqttClient = true; + callback_after_ms = UINT32_MAX; +} +void vTaskDelay(unsigned ticks) { + assert(ticks != 0); + ++sleeps; + now_ms += ticks * tick_ms; + elapsed_ms += ticks * tick_ms; + deliverDisconnect(); + if (on_delay) on_delay(); + assert(elapsed_ms <= 2000U + tick_ms); +} +int esp_mqtt_client_disconnect(void*) { + ++disconnect_calls; + if (disconnect_result == ESP_OK) deliverDisconnect(); + return disconnect_result; +} +int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; } +// Destruction is outside this test's stop/handoff seam; failed SDK resources +// remain owned by the surrounding bridge lifecycle in the production code. +PsychicMqttClient::~PsychicMqttClient() {} + +@ADAPTER_METHODS@ + +constexpr int RUNTIME_MQTT_SLOTS = 1, WIFI_MODE_NULL = 0; +constexpr int WIFI_MODE_AP = 2, WIFI_STA = 1, WIFI_OFF = 0; +using wifi_mode_t = int; +static int wifi_mode = WIFI_STA, wifi_stops = 0; +int esp_wifi_get_mode(int* mode) { *mode = wifi_mode; return ESP_OK; } +bool mqttStationWiFiMutationAllowed() { return !(wifi_mode & WIFI_MODE_AP); } +namespace mesh { namespace wifi { +bool espNowChannelConstrained() { return false; } +int restoreEspNowChannel() { assert(false); return ESP_ERR_INVALID_STATE; } +} } +struct { + void setAutoReconnect(bool enabled) { assert(!enabled); } + void disconnect(bool stop, bool erase) { assert(!stop && !erase); } + int getMode() const { return wifi_mode; } + void mode(int mode) { + assert(mode == WIFI_OFF && mesh::wifi::networkPower().backgroundSuspended()); + wifi_mode = mode; + ++wifi_stops; + } +} WiFi; + +struct MQTTBridge { + struct Slot { + PsychicMqttClient* client = nullptr; + bool enabled = true, connected = false, circuit_breaker_tripped = false; + uint32_t connected_at_ms = 0, last_reconnect_attempt = 0; + uint8_t reconnect_backoff = 0; + bool initial_connect_done = true; + char* auth_token = nullptr; + char broker_uri[32] = "mqtt://original"; + uint32_t token_expires_at = 1, last_token_renewal = 1; + uint32_t last_log_time = 1, last_deferred_log_ms = 1; + uint8_t max_backoff_failures = 0; + } _slots[RUNTIME_MQTT_SLOTS]; + using MQTTSlot = Slot; + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}, _fresh_ntp_this_boot{false}; + bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; + bool _status_publish_pending[RUNTIME_MQTT_SLOTS] = {}; + bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {}; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + bool _ntp_force_requested = false, _ntp_diag_requested = false; + bool _ntp_sync_pending = false, _manage_wifi = true, _wifi_status_initialized = true; + uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0; + uint32_t _ntp_refresh_retry_at = 0; + std::atomic _stop_requested{false}, _stop_acked{false}; + int destroyed = 0, queues = 0, ntp_starts = 0, slot_starts = 0; + struct Board { bool isOTAUpdateRunning() const { return false; } }; + Board* _board = nullptr; + void cancelNtpRefresh() {} + void updateCachedConnectionStatus() {} + void initializeWiFiInTask() {} + void processPacketQueue() { ++queues; } + void teardownSlot(int index, bool force = false); + void destroySlotClients() { + assert(!_stop_acked); + for (auto& slot : _slots) { + assert(!slot.client || !slot.client->isStarted()); + slot.client = nullptr; + } + ++destroyed; + } + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void workerStop() { + while (true) { +@TASK_STOP@ + } + } + void workerOutageOnce() { + for (unsigned iteration = 0; iteration < 1; ++iteration) { + const uint32_t now = now_ms; +@OUTAGE_GUARD@ + ++ntp_starts; + ++slot_starts; + } + } +}; + +@BRIDGE_METHOD@ + +int main(int argc, char** argv) { + assert(argc == 4); + const int scenario = std::atoi(argv[1]); + tick_ms = static_cast(std::atoi(argv[2])); + if (std::atoi(argv[3])) now_ms = UINT32_MAX - 511U; + PsychicMqttClient client; + active_client = &client; + client._client = reinterpret_cast(1); + client._started = true; + client._connected = true; + if (scenario == 0) { + client.disconnect(); // Lost event: SDK stop is sufficient proof. + assert(elapsed_ms >= 1000U && elapsed_ms < 1000U + tick_ms); + assert(stop_calls == 1 && !client.isStarted() && !client.connected()); + const int stopped = stop_calls; + client.disconnect(); + assert(stop_calls == stopped); // already stopped is idempotent + } else if (scenario == 1) { + stop_result = ESP_ERR_INVALID_STATE; + auto& control = mesh::wifi::networkPower(); + control.setManaged(true); + MQTTBridge bridge; + bridge._slots[0].client = &client; + const uint32_t start = now_ms; + assert(bridge.serviceNetworkPower(start)); + now_ms = start + 300000U; + assert(!bridge.serviceNetworkPower(now_ms)); + assert(elapsed_ms >= 1000U && stop_calls == 1 && client.isStarted()); + assert(!bridge._network_suspended && !control.backgroundSuspended()); + assert(wifi_stops == 0 && wifi_mode == WIFI_STA); + // An owning-task retry can now complete teardown without another wait if + // the disconnect request itself fails. Only the successful SDK stop grants + // the bridge permission to turn the station off. + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_OK; + assert(!bridge.serviceNetworkPower(now_ms)); + assert(!client.isStarted() && control.backgroundSuspended() && wifi_stops == 1); + } else if (scenario == 2) { + disconnect_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(sleeps == 0 && disconnect_calls == 1 && stop_calls == 1); + assert(!client.isStarted() && !client.connected()); + } else if (scenario == 3) { + callback_after_ms = 0; + stop_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(sleeps == 0 && stop_calls == 1 && client.isStarted()); + assert(!client.connected()); // Actual event state survives failed stop. + } else if (scenario == 4) { + callback_after_ms = 50U; + client.disconnect(); + assert(elapsed_ms >= 50U && elapsed_ms < 50U + tick_ms); + assert(stop_calls == 1 && !client.isStarted() && !client.connected()); + } else if (scenario == 5) { + client._started = false; + client.disconnect(); + client._client = nullptr; + client.disconnect(); + assert(disconnect_calls == 0 && stop_calls == 0 && sleeps == 0); + } else if (scenario == 6) { + client._connected = false; + stop_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(disconnect_calls == 0 && stop_calls == 1 && sleeps == 0); + assert(client.isStarted()); + } else if (scenario == 7) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + MQTTBridge bridge; + bridge._slots[0].client = &client; + bridge._stop_requested = true; + on_delay = [&]() { + assert(!bridge._stop_acked && bridge.destroyed == 0); + assert(bridge._slots[0].client == &client); + if (elapsed_ms >= 150U) stop_result = ESP_OK; + }; + bridge.workerStop(); + assert(stop_calls == 2 && !client.isStarted()); + assert(bridge._stop_acked && bridge.destroyed == 1); + assert(bridge._slots[0].client == nullptr); + on_delay = nullptr; + } else if (scenario == 8) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + MQTTBridge bridge; + char token[] = "protected-token"; + bridge._slots[0].client = &client; + bridge._slots[0].auth_token = token; + bridge.teardownSlot(0); + assert(client.isStarted() && strcmp(token, "protected-token") == 0); + assert(bridge._slots[0].initial_connect_done); + assert(strcmp(bridge._slots[0].broker_uri, "mqtt://original") == 0); + } else if (scenario == 9) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + auto& control = mesh::wifi::networkPower(); + control.setManaged(true); + MQTTBridge bridge; + bridge._slots[0].client = &client; + const uint32_t start = now_ms; + assert(bridge.serviceNetworkPower(start)); + now_ms = start + 300000U; + bridge.workerOutageOnce(); + assert(client.isStarted() && !bridge._network_suspended); + assert(!control.backgroundSuspended() && wifi_stops == 0); + assert(bridge.queues == 1 && bridge.ntp_starts == 0 && bridge.slot_starts == 0); + } else assert(false); + printf("PASS: %d\n", scenario); +} +''' + + +class MqttDisconnectTests(unittest.TestCase): + def test_real_wrapper_and_wifi_handoff_across_sdk_failures_ticks_and_wrap(self): + compiler = os.environ.get("CXX") or shutil.which("g++") + self.assertIsNotNone(compiler) + adapter = "\n".join(method(ADAPTER, signature) for signature in ( + "PsychicMqttClient::PsychicMqttClient()", + "bool PsychicMqttClient::connected()", + "void PsychicMqttClient::disconnect()", + "void PsychicMqttClient::forceStop()", + "unsigned long PsychicMqttClient::getPublishAck()", + )) + task = method(BRIDGE, "void MQTTBridge::mqttTaskLoop()") + source = (HARNESS.replace("@ADAPTER_METHODS@", adapter) + .replace("@TASK_STOP@", extract_braced(task, " if (_stop_requested)")) + .replace("@OUTAGE_GUARD@", extract_braced(task, " if (!serviceNetworkPower(")) + .replace("@BRIDGE_METHOD@", "\n".join(method(BRIDGE, signature) for signature in ( + "bool MQTTBridge::serviceNetworkPower(", + "void MQTTBridge::serviceEspNowChannelRestore(", + "void MQTTBridge::teardownSlot(", + )))) + with tempfile.TemporaryDirectory(prefix="meshcore-mqtt-disconnect-") as temporary: + directory = Path(temporary) + (directory / "Arduino.h").write_text(ARDUINO_HEADER, encoding="ascii") + (directory / "mqtt_client.h").write_text(MQTT_HEADER, encoding="ascii") + (directory / "esp_crt_bundle.h").write_text("#pragma once\n", encoding="ascii") + cpp = directory / "disconnect.cpp" + cpp.write_text(source, encoding="ascii") + for idf in (4, 5): + for tick in (1, 10, 50): + with self.subTest(idf=idf, tick=tick): + binary = directory / f"disconnect-{idf}-{tick}" + arduino = ARDUINO_HEADER.replace( + "constexpr unsigned portTICK_PERIOD_MS = 1;", + f"constexpr unsigned portTICK_PERIOD_MS = {tick};") + (directory / "Arduino.h").write_text(arduino, encoding="ascii") + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-Werror", "-DARDUINO_ARCH_ESP32=1", "-DESP_PLATFORM=1", + f"-DESP_IDF_VERSION_MAJOR={idf}", + "-I", str(directory), "-I", str(ROOT / "lib/PsychicMqttClient/src"), + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + for scenario in range(10): + for wrap in (0, 1): + with self.subTest(scenario=scenario, wrap=wrap): + checked = subprocess.run([str(binary), str(scenario), str(tick), str(wrap)], + capture_output=True, text=True, timeout=5) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_mqtt_network_timeout.py b/test/test_mqtt_network_timeout.py index 0b24217c8..0586b7252 100644 --- a/test/test_mqtt_network_timeout.py +++ b/test/test_mqtt_network_timeout.py @@ -150,6 +150,7 @@ struct MQTTBridge { } _slots[RUNTIME_MQTT_SLOTS]; std::atomic _stop_requested{false}, _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {}; + bool _slot_reconfigure_pending[RUNTIME_MQTT_SLOTS] = {}; bool _initialized = true; int _max_active_slots = 2; MQTTPrefs prefs; @@ -189,6 +190,14 @@ int main() { assert(slot.client->_buffer_capacity == 896); char token[8] = "old-jwt"; slot.auth_token = token; + stop_result = ESP_ERR_INVALID_STATE; + bridge.applySlotPreset(0, "jwt"); + assert(slot.client->isStarted() && slot.initial_connect_done); + assert(slot.preset == &plain_preset && strcmp(token, "old-jwt") == 0); + assert(bridge._slot_reconfigure_pending[0]); + assert(init_calls == 1 && destroy_calls == 0); + stop_result = ESP_OK; + bridge._slot_reconfigure_pending[0] = false; // worker consumes the retry flag bridge.applySlotPreset(0, "jwt"); assert(token[0] == '\0' && slot.preset == &jwt_preset); assert(slot.client->_buffer_capacity == 896); @@ -287,6 +296,10 @@ class MqttNetworkTimeoutTest(unittest.TestCase): preamble = preamble.replace( 'void vTaskDelay(unsigned) { assert(false && "stub disconnect must deliver its event"); }', 'void vTaskDelay(unsigned ticks) { assert(ticks == 50); }') + preamble = preamble.replace( + 'int esp_mqtt_client_stop(void*) { ++stop_calls; return ESP_OK; }', + 'static int stop_result = ESP_OK;\n' + 'int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; }') source = "#define ESP_PLATFORM 1\n" + preamble + RECONFIGURE_PREAMBLE source += "\n".join(method(adapter, signature) for signature in ( "PsychicMqttClient::PsychicMqttClient()", diff --git a/test/test_mqtt_transport_results.py b/test/test_mqtt_transport_results.py index 100993887..2068783eb 100644 --- a/test/test_mqtt_transport_results.py +++ b/test/test_mqtt_transport_results.py @@ -191,6 +191,7 @@ TASK_PREAMBLE = r''' #include #include #include +#include "helpers/WiFiNetworkPowerPolicy.h" #define PORTABLE_MQTT_OBSERVER 1 #define MQTT_DEBUG_PRINTLN(...) ((void)0) using TickType_t = uint32_t; @@ -215,7 +216,12 @@ struct MQTTPrefs { bool mqtt_status_enabled = true; }; struct MQTTBridge { + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + bool _network_espnow_restore_pending = false; struct Slot { + struct Client { bool isStarted() const { return false; } }; + Client* client = nullptr; bool enabled = true, initial_connect_done = false, connected = false; unsigned long last_reconnect_attempt = 0; } _slots[RUNTIME_MQTT_SLOTS]; @@ -244,6 +250,14 @@ struct MQTTBridge { assert(!_stop_acked); } bool initializeWiFiInTask() { return true; } + bool serviceNetworkPower(uint32_t) { work(); return true; } + void serviceBackgroundNtp(uint32_t) { + work(); + if (scenario == 7) { + _ntp_synced = true; + _stop_requested = true; + } + } bool handleWiFiConnection(unsigned long) { work(); return false; } bool syncTimeWithNTP(bool = false, bool = false) { work(); @@ -296,6 +310,7 @@ struct MQTTBridge { } bool waitUnlessStopping(uint32_t delay_ms); void mqttTaskLoop(); + void applyTimezoneRules() {} }; void vTaskDelay(TickType_t ticks) { assert(ticks > 0); @@ -375,6 +390,7 @@ ASYNC_STOP_PREAMBLE = r''' #include #include "helpers/MQTTLifecycle.h" #include "helpers/MQTTConnectionPolicy.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #define ESP_PLATFORM 1 #define MQTT_DEBUG_PRINTLN(...) ((void)0) #define pdMS_TO_TICKS(ms) (ms) @@ -406,6 +422,7 @@ struct MQTTBridge { } ops{this}; MQTTLifecycle::Coordinator _lifecycle{ops, 1}; std::atomic _stop_requested{false}, _stop_acked{false}; + std::atomic _network_suspended{true}; bool _initialized = true, resources_owned = true; bool _slots_setup_done = true, _staged_raw_valid = true; bool _ntp_estimate_requested = true, _ntp_estimate_done = true, _ntp_estimate_ok = true; @@ -416,6 +433,8 @@ struct MQTTBridge { assert(_lifecycle.requestStart()); assert(_lifecycle.onTaskStarted()); s_mqtt_bridge_instance = this; + mesh::wifi::networkPower().setManaged(true); + mesh::wifi::networkPower().setSuspended(true); } void cancelNtpRefresh() { ++cancels; } void requestStop(); @@ -452,6 +471,7 @@ static void assert_finished(MQTTBridge& bridge, bool clean) { assert(!bridge._ntp_estimate_requested && !bridge._ntp_estimate_done); assert(!bridge._ntp_estimate_ok && bridge._ntp_estimate_epoch == 0); assert(bridge.cancels == 1); + assert(!bridge._network_suspended && !mesh::wifi::networkPower().managed()); } ''' @@ -547,6 +567,7 @@ class MqttTransportResultsTests(unittest.TestCase): path.write_text(source, encoding="utf-8") exe = Path(temp) / ("worker.exe" if os.name == "nt" else "worker") result = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-I", str(ROOT / "src"), str(path), "-o", str(exe)], text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) for scenario in range(12): diff --git a/test/test_mqtt_wifi_network_power.py b/test/test_mqtt_wifi_network_power.py new file mode 100644 index 000000000..1a1672897 --- /dev/null +++ b/test/test_mqtt_wifi_network_power.py @@ -0,0 +1,856 @@ +#!/usr/bin/env python3 +"""Execute MQTT's production WiFi/SNTP outage handoff with controlled SDK calls. + +The actual bridge methods and shared policies run without hardware, network, +Arduino, or PlatformIO. SDK calls observe suspension publication ordering and +can fail independently of MQTT and NTP state. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from test_mqtt_transport_results import method + + +ROOT = Path(__file__).resolve().parents[1] +BRIDGE = "src/helpers/bridges/MQTTBridge.cpp" + +PREAMBLE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ESP_PLATFORM 1 +#define WITH_SNMP 1 +#define MQTT_DEBUG_PRINTLN(...) ((void)0) +using wifi_mode_t = int; +constexpr int ESP_OK = 0, ESP_FAIL = -1, WIFI_MODE_NULL = 0; +constexpr int WIFI_STA = 1, WIFI_MODE_AP = 2, WIFI_OFF = 0; +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 0, RUNTIME_MQTT_SLOTS = 3; +constexpr int SNTP_SYNC_STATUS_RESET = 0, SNTP_SYNC_STATUS_COMPLETED = 1; +static constexpr uint32_t MQTT_NTP_REFRESH_TIMEOUT_MS = 15000U; +static constexpr uint32_t MQTT_NTP_NON_ESP_SETTLE_MS = 5000U; +static constexpr uint32_t MQTT_NTP_MIN_VALID_EPOCH = 1767225600U; +static uint32_t clock_ms = 100U; +static int64_t system_epoch = 0; +static int sdk_mode = WIFI_STA, sdk_mode_result = ESP_OK; +static bool station_mutation_allowed = true, espnow_constrained = false; +static int ntp_status = SNTP_SYNC_STATUS_RESET, ntp_configs = 0; +static int ntp_stops = 0, channel_restores = 0; +static int channel_restore_result = ESP_OK; +static bool require_quiescent_wifi_stop = false; +static std::vector events; +static std::vector ntp_primaries; +static std::function on_client_stop; + +static void require(bool value, const char* reason) { + if (!value) throw std::runtime_error(reason); +} +uint32_t millis() { return clock_ms; } +time_t fake_time(void*) { return static_cast(system_epoch); } +int fake_settimeofday(const timeval* value, void*) { + system_epoch = value->tv_sec; + return 0; +} +#define time fake_time +#define settimeofday fake_settimeofday + +int esp_wifi_get_mode(wifi_mode_t* mode) { + *mode = sdk_mode; + return sdk_mode_result; +} +bool mqttStationWiFiMutationAllowed() { + return station_mutation_allowed && sdk_mode_result == ESP_OK + && !(sdk_mode & WIFI_MODE_AP); +} +namespace mesh { namespace wifi { +bool espNowChannelConstrained() { return espnow_constrained; } +int restoreEspNowChannel() { + ++channel_restores; + events.push_back("espnow"); + return channel_restore_result; +} +} } + +struct WiFiModel { + int state = WL_CONNECTED; + bool reconnect = true; + int disconnects = 0, offs = 0; + int status() const { return state; } + int getMode() const { return sdk_mode; } + void setAutoReconnect(bool value) { + reconnect = value; + if (!value && require_quiescent_wifi_stop) { + require(mesh::wifi::networkPower().backgroundSuspended(), + "WiFi stopped before worker published transport quiescence"); + } + } + void disconnect(bool stop, bool erase) { + require(!stop && !erase, "suspension erased credentials or ESP-NOW driver"); + ++disconnects; + state = WL_DISCONNECTED; + events.push_back("wifi-disconnect"); + } + void mode(int value) { + require(value == WIFI_OFF, "unexpected station mode change"); + if (require_quiescent_wifi_stop) { + require(mesh::wifi::networkPower().backgroundSuspended(), + "WiFi driver stopped before the safe handoff"); + } + sdk_mode = value; + ++offs; + events.push_back("wifi-off"); + } +} WiFi; + +void sntp_set_sync_status(int value) { ntp_status = value; } +int sntp_get_sync_status() { return ntp_status; } +void esp_sntp_stop() { + if (require_quiescent_wifi_stop) { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "SNTP was stopped after publishing transport quiescence"); + } + ++ntp_stops; + events.push_back("sntp-stop"); +} +void configTime(int, int, const char* primary, const char*, const char*) { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "SNTP restarted during suspension"); + ++ntp_configs; + ntp_primaries.push_back(primary); + events.push_back("sntp-start"); +} +void configTime(int a, int b, const char* primary) { + configTime(a, b, primary, nullptr, nullptr); +} + +struct Client { + bool started = true, fail_stop = false; + int stops = 0; + uint32_t publish_acks = 0; + uint32_t getPublishAck() const { return publish_acks; } + bool isStarted() const { return started; } + void disconnect() { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "TLS stopped after handing station off"); + ++stops; + events.push_back("tls-stop"); + if (!fail_stop) started = false; + if (on_client_stop) on_client_stop(); + } +}; +struct Board { + bool ota = false; + bool isOTAUpdateRunning() const { return ota; } +}; +struct Snmp { + bool running = false; + bool isRunning() const { return running; } +}; +struct Rtc { + uint32_t epoch = 0; + uint32_t getCurrentTime() const { return epoch; } + void setCurrentTime(uint32_t value) { epoch = value; } +}; +struct Prefs {}; +const char* effectiveNtpPrimary(const Prefs*) { return "custom"; } +#ifdef ESP_PLATFORM +static void fillNtpServerList(const Prefs*, const char* servers[], int& count) { + static const char* names[] = {"custom", "pool", "google", "cloudflare", "aws", "nist"}; + count = 6; + for (int index = 0; index < count; ++index) servers[index] = names[index]; +} +#endif + +@HELPERS@ + +struct MQTTBridge { + static constexpr int kMaxNtpServers = 6; + struct Slot { + bool enabled = true, connected = false, circuit_breaker_tripped = true; + uint32_t connected_at_ms = 0, last_reconnect_attempt = 200U; + uint8_t reconnect_backoff = 5; + Client* client = nullptr; + } _slots[RUNTIME_MQTT_SLOTS]; + Client clients[RUNTIME_MQTT_SLOTS]; + std::atomic _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; + std::atomic _status_publish_pending[RUNTIME_MQTT_SLOTS] = {}; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + std::atomic _ntp_synced{false}, _fresh_ntp_this_boot{false}; + std::atomic _backward_clock_reset_epoch{0}; + bool _manage_wifi = true, _wifi_status_initialized = true; + bool _ntp_force_requested = false, _ntp_diag_requested = false; + bool _ntp_estimate_requested = false, _ntp_sync_pending = true; + bool _ntp_estimate_done = false, _ntp_estimate_ok = false; + uint32_t _ntp_estimate_epoch = 0; + std::atomic _stop_requested{false}; + bool _ntp_refresh_pending = false; + uint32_t _ntp_refresh_started_at = 0, _ntp_refresh_previous_epoch = 0; + uint32_t _ntp_refresh_retry_at = 0, _last_ntp_sync = 0; + uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0; + uint8_t _background_ntp_server = 0; + Board board; + Board* _board = &board; + Snmp snmp; + Snmp* _snmp_agent = &snmp; + Rtc rtc; + Rtc* _rtc = &rtc; + Prefs prefs; + Prefs* _obs = &prefs; + int initializes = 0, cache_updates = 0; + mesh::sntp_coord::OperationLease _ntp_refresh_operation{ + mesh::sntp_coord::processWideCoordinator(), stopMqttSntpOperation}; + MQTTBridge() { + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + _slots[index].client = &clients[index]; + _slot_attempt_pending[index] = true; + _status_publish_pending[index] = true; + } + } + bool initializeWiFiInTask() { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "station resumed before clearing suspension handoff"); + ++initializes; + sdk_mode = WIFI_STA; + WiFi.reconnect = true; + return true; + } + void updateCachedConnectionStatus() { ++cache_updates; } + bool isRunning() const { return true; } + bool isNetworkSuspended() const { return _network_suspended.load(); } + bool requestNtpTimeEstimate(); + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void serviceBackgroundNtp(uint32_t now); + void cancelNtpRefresh(); + void refreshNTP(); + void pollNtpRefresh(uint32_t now); +}; + +@METHODS@ + +static void resetEnvironment() { + auto& power = mesh::wifi::networkPower(); + power.setManaged(false); + power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); + power.consumeProbeRequest(); + power.setManaged(true); + clock_ms = 100U; + system_epoch = 0; + sdk_mode = WIFI_STA; + sdk_mode_result = ESP_OK; + station_mutation_allowed = true; + espnow_constrained = false; + ntp_status = SNTP_SYNC_STATUS_RESET; + ntp_configs = ntp_stops = channel_restores = 0; + channel_restore_result = ESP_OK; + require_quiescent_wifi_stop = false; + events.clear(); + ntp_primaries.clear(); + on_client_stop = nullptr; + WiFi = {}; +} + +static constexpr uint32_t grace = 300000U, interval = 1800000U, probe = 120000U; + +static void failureStopsBothServicesBeforeWifi() { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(clock_ms), "cold bridge was suspended"); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "initial NTP was not asynchronous"); + clock_ms += 15000U; + bridge.serviceBackgroundNtp(clock_ms); + require(!bridge._ntp_refresh_pending && ntp_stops == 1, + "timed-out NTP continued running"); + require(bridge._ntp_refresh_retry_at == clock_ms + interval, + "failed NTP was not deferred 30 minutes"); + // GOT_IP/NTP retry timers and failing SDK handshakes do not renew the clock. + for (uint32_t offset = 16000U; offset < grace; offset += 1000U) { + clock_ms = 100U + offset; + WiFi.state = WL_CONNECTED; + require(bridge.serviceNetworkPower(clock_ms), "outage grace ended early"); + bridge.serviceBackgroundNtp(clock_ms); + } + require(ntp_configs == 1, "failed SNTP retried during its cooldown"); + // Keep one live SNTP lease at the cutoff to prove teardown publication order. + bridge._ntp_refresh_retry_at = 0; + bridge.refreshNTP(); + require(bridge._ntp_refresh_operation.owns(), "SNTP request lacks ownership"); + events.clear(); + require_quiescent_wifi_stop = true; + clock_ms = 100U + grace; + require(!bridge.serviceNetworkPower(clock_ms), "failed services kept WiFi on"); + require(bridge._network_suspended && mesh::wifi::networkPower().backgroundSuspended(), + "worker did not hand off its transport suspension"); + require(events == std::vector({"sntp-stop", "tls-stop", "tls-stop", + "tls-stop", "wifi-disconnect", "wifi-off"}), + "WiFi was disabled before SNTP/TLS teardown"); + require(!bridge._ntp_refresh_pending && !bridge._ntp_refresh_operation.owns() + && !bridge._ntp_sync_pending && !WiFi.reconnect && sdk_mode == WIFI_OFF, + "a failed background service retained radio activity"); + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + require(!bridge.clients[index].started && !bridge._slots[index].connected + && !bridge._slot_attempt_pending[index] && !bridge._status_publish_pending[index], + "suspension retained a pending transport or publish"); + } +} + +static void failedSdkStopWithholdsHandoff() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge.clients[1].fail_stop = true; + require(!bridge.serviceNetworkPower(100U + grace), + "failed SDK stop permitted more background TLS or NTP work"); + require(!bridge._network_suspended && !mesh::wifi::networkPower().backgroundSuspended() + && sdk_mode == WIFI_STA && WiFi.disconnects == 0 && WiFi.offs == 0, + "station owner was allowed to stop a still-running TLS client"); + require(!bridge.clients[0].started && bridge.clients[1].started + && bridge.clients[2].started, "teardown did not stop at failed client"); + bridge.clients[1].fail_stop = false; + require(!bridge.serviceNetworkPower(101U + grace), + "retry after successful SDK stop failed to suspend"); + require(!bridge.clients[1].started && !bridge.clients[2].started && WiFi.offs == 1, + "successful retry leaked a transport or radio"); +} + +static void recoveryHasOneBoundedWindow() { + for (uint32_t start : {100U, UINT32_MAX - 10000U}) { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(start), "initial network did not run"); + const uint32_t suspended = start + grace; + require(!bridge.serviceNetworkPower(suspended), "network did not suspend"); + for (uint32_t offset = 1000U; offset < interval; offset += 1000U) { + require(!bridge.serviceNetworkPower(suspended + offset), + "a failed timer woke WiFi during the cooldown"); + } + require(bridge.initializes == 0, "cooldown restarted station"); + const uint32_t resumed = suspended + interval; + require(bridge.serviceNetworkPower(resumed), "30-minute recovery was not started"); + require(bridge.initializes == 1 && !bridge._network_suspended + && !mesh::wifi::networkPower().backgroundSuspended() + && sdk_mode == WIFI_STA, "recovery did not release station ownership"); + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + require(!bridge._slots[index].circuit_breaker_tripped + && MQTTConnectionPolicy::reconnectDue(resumed, + bridge._slots[index].last_reconnect_attempt, + bridge._slots[index].reconnect_backoff, static_cast(index)), + "recovery left brokers waiting behind the old breaker"); + } + require(bridge._ntp_sync_pending && bridge._ntp_refresh_retry_at == 0, + "recovery did not admit one new NTP attempt"); + for (uint32_t offset = 1000U; offset < probe; offset += 1000U) { + require(bridge.serviceNetworkPower(resumed + offset), "probe ended early"); + } + require(bridge.initializes == 1, "probe restarted station repeatedly"); + require(!bridge.serviceNetworkPower(resumed + probe), "failed probe was unbounded"); + require(!bridge.serviceNetworkPower(resumed + probe + interval - 1U), + "failed probe did not start a fresh 30-minute cooldown"); + } +} + +static void foregroundAndEspNowArePreserved() { + for (int owner = 0; owner < 6; ++owner) { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + if (owner == 0) mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + if (owner == 1) bridge.board.ota = true; + if (owner == 2) sdk_mode = WIFI_STA | WIFI_MODE_AP; + if (owner == 3) bridge._ntp_force_requested = true; + if (owner == 4) bridge._ntp_diag_requested = true; + if (owner == 5) bridge.snmp.running = true; + require(bridge.serviceNetworkPower(100U + grace), "foreground owner lost radio"); + require(WiFi.disconnects == 0 && WiFi.offs == 0, + "background outage disrupted a foreground owner"); + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + require(!bridge.serviceNetworkPower(100U + grace), "failed STA did not suspend"); + require(!WiFi.reconnect && WiFi.disconnects == 1 && WiFi.offs == 0 + && sdk_mode == WIFI_STA && channel_restores == 1, + "suspending failed internet disabled independent ESP-NOW radio"); + } + // An AP can take ownership during the SDK's yielding client stop calls. + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + on_client_stop = [] { sdk_mode = WIFI_STA | WIFI_MODE_AP; }; + require(!bridge.serviceNetworkPower(100U + grace), "worker did not suspend transports"); + require(WiFi.disconnects == 0 && WiFi.offs == 0 && (sdk_mode & WIFI_MODE_AP), + "foreground AP race lost the shared WiFi driver"); + } +} + +static void automaticClockEstimateCannotWakeSuspendedNetwork() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._ntp_estimate_requested = true; + require(!bridge.serviceNetworkPower(100U + grace), + "automatic clock estimate bypassed combined outage grace"); + require(!bridge.serviceNetworkPower(100U + grace + 1U), + "automatic clock estimate woke failed network immediately"); + require(!bridge.serviceNetworkPower(100U + grace + interval - 1U), + "automatic clock estimate bypassed the 30-minute cooldown"); + require(bridge.initializes == 0, "background clock request restarted station"); + bridge._ntp_estimate_requested = false; + WiFi.state = WL_CONNECTED; // Even a stale connected facade cannot bypass it. + require(!bridge.requestNtpTimeEstimate() && !bridge._ntp_estimate_requested, + "automatic clock bootstrap queued new work during suspension"); + require(bridge.serviceNetworkPower(100U + grace + interval), + "scheduled recovery did not admit clock work"); + require(bridge.requestNtpTimeEstimate() && bridge._ntp_estimate_requested, + "scheduled recovery still blocked automatic clock bootstrap"); +} + +static void deferredEspNowChannelRepair() { + for (uint32_t start : {100U, UINT32_MAX - 500U, UINT32_MAX - 999U}) { + resetEnvironment(); + MQTTBridge bridge; + espnow_constrained = true; + clock_ms = start - grace; + bridge.serviceNetworkPower(clock_ms); + clock_ms = start; + channel_restore_result = ESP_FAIL; + require(!bridge.serviceNetworkPower(clock_ms), "failed STA did not suspend"); + require(bridge._network_espnow_restore_pending && channel_restores == 1 + && WiFi.disconnects == 1 && WiFi.offs == 0, + "failed channel write was not retained independently of WiFi stop"); + for (uint32_t offset : {100U, 250U, 500U, 999U}) { + clock_ms = start + offset; + require(!bridge.serviceNetworkPower(clock_ms), "channel repair resumed failed internet"); + require(channel_restores == 1 && WiFi.disconnects == 1, + "channel repair was unthrottled or repeated disconnect"); + } + clock_ms = start + 1000U; + channel_restore_result = ESP_OK; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 2 && !bridge._network_espnow_restore_pending + && WiFi.disconnects == 1 && WiFi.offs == 0, + "transient channel failure did not recover without toggling WiFi"); + for (uint32_t offset : {1100U, 2000U, 5000U}) { + clock_ms = start + offset; + require(!bridge.serviceNetworkPower(clock_ms), "repair canceled the outage cooldown"); + require(channel_restores == 2 && WiFi.disconnects == 1, + "successful channel restoration retained repeated work"); + } + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + clock_ms = 100U + grace; + channel_restore_result = ESP_FAIL; + bridge.serviceNetworkPower(clock_ms); + clock_ms += 1000U; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + bridge.serviceEspNowChannelRestore(clock_ms); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + sdk_mode = WIFI_STA | WIFI_MODE_AP; + bridge.serviceEspNowChannelRestore(clock_ms); + sdk_mode = WIFI_STA; + bridge.board.ota = true; + bridge.serviceEspNowChannelRestore(clock_ms); + bridge.board.ota = false; + sdk_mode_result = ESP_FAIL; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 1 && bridge._network_espnow_restore_pending, + "deferred restoration changed a foreground or unknown driver channel"); + sdk_mode_result = ESP_OK; + channel_restore_result = ESP_OK; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 2 && !bridge._network_espnow_restore_pending, + "released foreground owner stranded channel restoration"); + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + clock_ms = 100U + grace; + channel_restore_result = ESP_FAIL; + bridge.serviceNetworkPower(clock_ms); + espnow_constrained = false; + clock_ms += 1000U; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 1 && !bridge._network_espnow_restore_pending, + "stopped ESP-NOW owner retained a channel mutation"); + } +} + +static void stableBrokerCountsButFlappingDoesNot() { + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 101U; + require(bridge.serviceNetworkPower(30101U), "30-second quiet broker was not useful"); + require(bridge.serviceNetworkPower(100U + 10U * grace), "working broker lost WiFi"); + require(WiFi.offs == 0, "NTP failure overrode working MQTT"); + const uint32_t failed_at = 100U + 10U * grace; + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(failed_at + grace - 1U), + "real broker success did not renew outage grace"); + require(!bridge.serviceNetworkPower(failed_at + grace), + "disconnected formerly healthy broker kept WiFi indefinitely"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + for (uint32_t offset = 1000U; offset < grace; offset += 1000U) { + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 100U + offset - 1U; + require(bridge.serviceNetworkPower(100U + offset), "grace ended early"); + } + bridge._slots[0].connected_at_ms = 100U + grace - 1U; + require(!bridge.serviceNetworkPower(100U + grace), + "flapping broker repeatedly renewed the background outage"); + } +} + +static void publishAcknowledgmentProvesRecoveryImmediately() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + require(!bridge.serviceNetworkPower(100U + grace), "initial outage did not suspend"); + const uint32_t resumed = 100U + grace + interval; + require(bridge.serviceNetworkPower(resumed), "recovery probe did not begin"); + // A broker reply during the final second proves useful service immediately, + // even though it has not met the quiet 30-second connection threshold. + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = resumed + probe - 1U; + bridge.clients[0].publish_acks = 1U; + require(bridge.serviceNetworkPower(resumed + probe - 1U), + "PUBACK failed to prove useful network"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(resumed + probe), + "successful recovery retained its old short probe deadline"); + require(bridge.serviceNetworkPower(resumed + probe - 1U + grace - 1U), + "PUBACK failed to renew a normal outage window"); + require(!bridge.serviceNetworkPower(resumed + probe - 1U + grace), + "an old PUBACK repeatedly renewed a disconnected broker"); +} + +static void quietConnectionHasExactThirtySecondHealthBoundary() { + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 200U; + require(bridge.serviceNetworkPower(30199U), "initial grace ended early"); + bridge._slots[0].connected = false; + require(!bridge.serviceNetworkPower(100U + grace), + "29.999-second connection falsely renewed outage grace"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 200U; + require(bridge.serviceNetworkPower(30200U), "30-second connection was not admitted"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(30200U + grace - 1U), + "quiet 30-second broker did not renew useful-service grace"); + require(!bridge.serviceNetworkPower(30200U + grace), + "formerly quiet broker kept failed service awake indefinitely"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(0U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 0U; + require(bridge.serviceNetworkPower(30000U), "zero-time connection was not admitted"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(30000U + grace - 1U), + "valid zero-time broker connection was mistaken for unset state"); + require(!bridge.serviceNetworkPower(30000U + grace), + "disconnected zero-time broker retained WiFi indefinitely"); + } +} + +static void everyNtpServerGetsTheLeadingTimeoutSlot() { + resetEnvironment(); + MQTTBridge bridge; + const char* expected[] = {"custom", "pool", "google", "cloudflare", "aws", "nist", "custom"}; + for (const char* primary : expected) { + bridge.refreshNTP(); + require(ntp_primaries.back() == primary, + "a blackholed leading server starved configured NTP fallback servers"); + clock_ms += 15000U; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending, + "failed rotating NTP request exceeded its bounded timeout"); + clock_ms = bridge._ntp_refresh_retry_at; + } + require(ntp_configs == 7, "NTP rotation skipped or repeated a timeout slot"); +} + +static void sntpFailureAndBusyLeaseRetries() { + for (uint32_t start : {100U, UINT32_MAX - 4000U, UINT32_MAX - interval + 1U}) { + resetEnvironment(); + MQTTBridge bridge; + clock_ms = start; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, "boot NTP did not start"); + clock_ms += 15000U; + bridge.serviceBackgroundNtp(clock_ms); + const uint32_t failed_at = clock_ms; + const uint32_t retry = mqttNtpFailureRetryAt(failed_at); + require(bridge._ntp_refresh_retry_at == retry && !bridge._ntp_refresh_pending, + "failed SNTP kept running or lost its bounded retry"); + clock_ms = failed_at + interval - 1U; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1, "failed SNTP retried before 30 minutes"); + clock_ms = retry; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 2 && bridge._ntp_refresh_pending, "SNTP retry never became due"); + } + resetEnvironment(); + { + MQTTBridge bridge; + mesh::sntp_coord::OperationLease foreground_ntp(mesh::sntp_coord::processWideCoordinator()); + require(foreground_ntp.tryAcquire(), "foreground SNTP owner did not acquire"); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 0 && !bridge._ntp_refresh_pending + && bridge._ntp_refresh_retry_at == clock_ms + 5000U, + "SNTP contention was treated as failed internet or busy-waited"); + clock_ms += 4999U; + foreground_ntp.release(); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 0, "SNTP busy-owner retry was not bounded"); + clock_ms += 1U; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1, "released SNTP owner never admitted retry"); + } +} + +static void retainedClockAndFreshNtpDoNotCreditFailedMqtt() { + resetEnvironment(); + MQTTBridge bridge; + bridge.rtc.epoch = MQTT_NTP_MIN_VALID_EPOCH + 100U; + require(bridge.serviceNetworkPower(100U), "initial policy did not run"); + bridge.serviceBackgroundNtp(100U); + require(bridge._ntp_synced && !bridge._fresh_ntp_this_boot, + "retained RTC became fake proof of fresh internet"); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "retained clock suppressed fresh NTP request"); + system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U; + ntp_status = SNTP_SYNC_STATUS_COMPLETED; + clock_ms = 101U; + bridge.serviceBackgroundNtp(clock_ms); + require(bridge._fresh_ntp_this_boot && bridge._last_ntp_sync == clock_ms + && !bridge._ntp_refresh_pending && ntp_stops == 1, + "fresh NTP did not update the RTC and release SNTP"); + require(bridge.serviceNetworkPower(100U + grace - 1U), "outage grace ended early"); + require(!bridge.serviceNetworkPower(100U + grace), + "successful NTP kept failed MQTT's WiFi on indefinitely"); +} + +int main(int argc, char** argv) { + require(argc == 2, "one case is required"); + const std::string name = argv[1]; + try { + if (name == "handoff") failureStopsBothServicesBeforeWifi(); + else if (name == "failed-stop") failedSdkStopWithholdsHandoff(); + else if (name == "recovery") recoveryHasOneBoundedWindow(); + else if (name == "owners") foregroundAndEspNowArePreserved(); + else if (name == "background-estimate") automaticClockEstimateCannotWakeSuspendedNetwork(); + else if (name == "channel-repair") deferredEspNowChannelRepair(); + else if (name == "healthy") stableBrokerCountsButFlappingDoesNot(); + else if (name == "puback") publishAcknowledgmentProvesRecoveryImmediately(); + else if (name == "quiet-boundary") quietConnectionHasExactThirtySecondHealthBoundary(); + else if (name == "ntp-retry") sntpFailureAndBusyLeaseRetries(); + else if (name == "ntp-rotation") everyNtpServerGetsTheLeadingTimeoutSlot(); + else if (name == "ntp-proof") retainedClockAndFreshNtpDoNotCreditFailedMqtt(); + else return 2; + } catch (const std::exception& error) { + std::cerr << name << ": " << error.what() << '\n'; + return 1; + } + std::cout << "PASS: " << name << '\n'; + return 0; +} +''' + +PORTABLE_MAIN = r''' +int main() { + try { + resetEnvironment(); + MQTTBridge bridge; + clock_ms = UINT32_MAX - 1000U; + bridge.refreshNTP(); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "portable async SNTP request did not start"); + require(!bridge._ntp_refresh_operation.owns(), + "portable stack unexpectedly acquired ESP32 SNTP lease"); + clock_ms += 4999U; + bridge.pollNtpRefresh(clock_ms); + require(bridge._ntp_refresh_pending && !bridge._ntp_synced, + "portable stack accepted clock before five-second settlement"); + system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U; + clock_ms += 1U; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending && bridge._ntp_synced + && bridge._fresh_ntp_this_boot && bridge.rtc.epoch == system_epoch + && bridge._last_ntp_sync == clock_ms, + "portable stack lost its existing settled-clock behavior"); + bridge.refreshNTP(); + WiFi.state = WL_DISCONNECTED; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending + && bridge._ntp_refresh_retry_at == clock_ms + 5000U, + "ESP32 outage policy changed portable WiFi-failure retry"); + require(ntp_stops == 0, "portable SNTP called ESP32 SDK cleanup"); + require(mqttNtpFailureRetryAt(UINT32_MAX - 4999U) == 1U, + "portable retry lost its wrap-safe zero-sentinel handling"); + require(mqttNtpRetryAt(clock_ms) == clock_ms + 5000U, + "portable busy-owner retry unexpectedly changed"); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + std::cout << "PASS: portable\n"; + return 0; +} +''' + + +class MqttWiFiNetworkPowerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + raise RuntimeError("a host C++ compiler is required") + helpers = "\n".join(method(BRIDGE, signature) for signature in ( + "static uint32_t mqttNtpFailureRetryAt(", + "static bool mqttNtpRefreshDue(", + "static uint32_t mqttNtpRetryAt(", + "static void stopMqttSntpOperation(", + )) + methods = "\n".join(method(BRIDGE, signature) for signature in ( + "bool MQTTBridge::serviceNetworkPower(", + "void MQTTBridge::serviceEspNowChannelRestore(", + "void MQTTBridge::serviceBackgroundNtp(", + "void MQTTBridge::cancelNtpRefresh(", + "void MQTTBridge::refreshNTP(", + "void MQTTBridge::pollNtpRefresh(", + "bool MQTTBridge::requestNtpTimeEstimate(", + )) + cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-mqtt-network-power-") + cls.addClassCleanup(cls.temporary.cleanup) + directory = Path(cls.temporary.name) + cpp = directory / "power.cpp" + source = PREAMBLE.replace("@HELPERS@", helpers).replace("@METHODS@", methods) + cpp.write_text(source, encoding="ascii") + cls.binary = directory / "power" + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(cpp), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + # The portable stack keeps its previous five-second settle and retry; + # the new 30-minute failed-network policy is ESP32-only. + portable_methods = "\n".join(method(BRIDGE, signature) for signature in ( + "void MQTTBridge::cancelNtpRefresh(", + "void MQTTBridge::refreshNTP(", + "void MQTTBridge::pollNtpRefresh(", + )) + portable_source = PREAMBLE[:PREAMBLE.index("static constexpr uint32_t grace")] + portable_source = (portable_source.replace("#define ESP_PLATFORM 1", "") + .replace("@HELPERS@", helpers) + .replace("@METHODS@", portable_methods) + PORTABLE_MAIN) + portable_cpp = directory / "portable.cpp" + portable_cpp.write_text(portable_source, encoding="ascii") + cls.portable_binary = directory / "portable" + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(portable_cpp), + "-o", str(cls.portable_binary)], + capture_output=True, text=True, timeout=60) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + + def run_case(self, name): + checked = subprocess.run([str(self.binary), name], capture_output=True, + text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("PASS: " + name, checked.stdout) + + def test_failed_ntp_and_mqtt_quiesce_before_wifi_shutdown_publication(self): + self.run_case("handoff") + + def test_failed_sdk_transport_stop_keeps_station_owned_and_awake(self): + self.run_case("failed-stop") + + def test_recovery_restarts_once_and_stops_after_bounded_probe_across_wrap(self): + self.run_case("recovery") + + def test_foreground_and_espnow_owners_preserve_driver_even_during_stop_race(self): + self.run_case("owners") + + def test_automatic_clock_estimate_cannot_bypass_background_outage_cooldown(self): + self.run_case("background-estimate") + + def test_failed_espnow_channel_restore_retries_without_wifi_toggles_or_owner_conflicts(self): + self.run_case("channel-repair") + + def test_stable_mqtt_counts_as_useful_and_flapping_mqtt_cannot_renew(self): + self.run_case("healthy") + + def test_publish_ack_proves_recovery_immediately_and_is_consumed_once(self): + self.run_case("puback") + + def test_quiet_broker_health_requires_exactly_thirty_seconds(self): + self.run_case("quiet-boundary") + + def test_sntp_failure_and_busy_lease_have_different_bounded_retries(self): + self.run_case("ntp-retry") + + def test_all_six_ntp_servers_lead_a_bounded_request_before_rotation_repeats(self): + self.run_case("ntp-rotation") + + def test_retained_clock_or_fresh_ntp_cannot_keep_failed_mqtt_wifi_active(self): + self.run_case("ntp-proof") + + def test_portable_non_esp_stack_preserves_settle_and_short_retry_policy(self): + checked = subprocess.run([str(self.portable_binary)], capture_output=True, + text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("PASS: portable", checked.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_room_web_http.py b/test/test_room_web_http.py index a35e5694f..e3d944915 100644 --- a/test/test_room_web_http.py +++ b/test/test_room_web_http.py @@ -28,6 +28,7 @@ HARNESS = r''' #include #include #include +#include using String = std::string; #define PROGMEM @HTML@ @@ -153,9 +154,15 @@ struct WebConfigServer { static constexpr size_t MAX_BODY = @MAX_BODY@; static constexpr uint32_t STOP_WARN_MS = WebConfigBatch::kStopWarnMs; enum Mode { MODE_OFF, MODE_SETUP, MODE_LAN }; + enum { BATCH_IDLE, BATCH_PENDING }; Mode _mode = MODE_LAN; + unsigned _batch_state = BATCH_IDLE; + bool _setup_wifi_handoff_pending = false; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; bool _stopping = false, _stop_warned = false, _board_cmds_probed = true; uint32_t _handler_refs = 0, _stop_warn_at = 0, _last_activity = 0; + uint32_t _network_activity_until = 0; std::mutex mutex; SemaphoreHandle_t _mux = &mutex; mesh::RoomWebMailbox* _room_mailbox = nullptr; @@ -170,11 +177,16 @@ struct WebConfigServer { if (allocate) _room_mailbox = new mesh::RoomWebMailbox; registerRoutes(); _active = this; } - ~WebConfigServer() { if (_active == this) _active = nullptr; delete _room_mailbox; } + ~WebConfigServer() { + if (_active == this) _active = nullptr; + delete _room_mailbox; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + } void finalizeTeardown() { // Dependency stub: production tick decides when reclaim is safe. assert(_handler_refs == 0); ++finalizations; delete _room_mailbox; _room_mailbox = nullptr; _server = nullptr; _stopping = false; _cb->onWebConfigStopped(); + _network_activity_until = 0; refreshNetworkPowerHold(millis()); } void probeBoardCommands() { _board_cmds_probed = true; } void serviceTerminal(uint32_t) {} @@ -183,6 +195,9 @@ struct WebConfigServer { void requestStop(); void detachRoutes(); uint32_t handlerRefCount() const; + void noteNetworkActivity(); + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); void registerRoutes(); static void dispatchRequest(AsyncWebServerRequest*, RequestHandler); static void collectBody(AsyncWebServerRequest*, uint8_t*, size_t, size_t, size_t); @@ -342,6 +357,8 @@ static void boundsAndFloors() { static void pagesAndAvailability() { Callbacks cb; WebConfigServer server(cb); AsyncWebServerRequest page; server.host.invoke("/room", HTTP_GET, page); assert(page.status == 200); + assert(server.isNetworkForegroundActive(ticks)); + assert(mesh::wifi::networkPower().foregroundActive()); assert(page.reply.find("MeshCore Room") != String::npos && page.response_type == "text/html; charset=utf-8"); assert(page.response_headers["Cache-Control"] == "no-store"); assert(page.response_headers["X-Content-Type-Options"] == "nosniff" && page.response_headers["Referrer-Policy"] == "no-referrer"); @@ -360,12 +377,14 @@ static void lifecycle() { assert(server.handlerRefCount() == 1); server.requestStop(); server.tick(ticks); assert(server._stopping && server._room_mailbox && server.finalizations == 0); + assert(mesh::wifi::networkPower().foregroundActive()); assert(cb.calls == 0); }; submit(server, pending); assert(pending.status == 202 && server.host.ended); assert(server.handlerRefCount() == 0 && server._stopping && cb.calls == 0); AsyncWebServerRequest unavailable; identity(unavailable); result(server, unavailable); assert(unavailable.status == 503); server.tick(ticks); assert(!server._room_mailbox && !server._stopping && cb.stops == 1 && server.finalizations == 1); + assert(!mesh::wifi::networkPower().foregroundActive()); assert(cb.calls == 0); // Pending work is canceled, never executed after stop. Callbacks off_cb; WebConfigServer off(off_cb); AsyncWebServerRequest post; identity(post); body(post, "{}"); submit(off, post); off._mode = WebConfigServer::MODE_OFF; off.tick(ticks); assert(off_cb.calls == 0); @@ -435,6 +454,9 @@ class RoomWebHttpTests(unittest.TestCase): "void WebConfigServer::handleRoomResult(", "void WebConfigServer::collectBody(", "void WebConfigServer::dispatchRequest(", "void WebConfigServer::detachRoutes(", "uint32_t WebConfigServer::handlerRefCount(", "void WebConfigServer::requestStop(", + "void WebConfigServer::noteNetworkActivity(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(", "void WebConfigServer::registerRoutes(")] tick = extract_braced(source, "void WebConfigServer::tick(") # Execute the exact stop/OFF guards and loop service calls; SDK Wi-Fi, diff --git a/test/test_rs232_mqtt_runtime.py b/test/test_rs232_mqtt_runtime.py index 8168a6c90..c56f49425 100644 --- a/test/test_rs232_mqtt_runtime.py +++ b/test/test_rs232_mqtt_runtime.py @@ -187,9 +187,14 @@ class RS232MQTTRuntimeTests(unittest.TestCase): MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh); expect_get(cli, "bridge.type", "> mqtt+rs232"); assert(mesh.isMqttBridgeRunning() && !mesh.isRs232BridgeRunning()); + mesh.mqtt_bridge->setNetworkSuspended(true); + assert(mesh.isMqttBridgeRunning() && mesh._prefs.bridge_enabled); + assert(!mesh.bridgesPreventSleep()); char reply[256] = {}; cli.set("rs232.enabled on", reply); assert(strcmp(reply, "OK") == 0); assert(mesh.isRs232BridgeRunning() && mesh.isMqttBridgeRunning()); + assert(mesh.bridgesPreventSleep()); // Paused MQTT cannot mask an active UART. + mesh.mqtt_bridge->setNetworkSuspended(false); const unsigned starts = mesh.mqtt_bridge->begins; cli.set("bridge.baud 57600", reply); assert(strcmp(reply, "OK") == 0); assert(mesh.isRs232BridgeRunning() && mesh.mqtt_bridge->begins == starts); diff --git a/test/test_tls_download_clock_gates.py b/test/test_tls_download_clock_gates.py index d73a9f850..76e36fdd9 100644 --- a/test/test_tls_download_clock_gates.py +++ b/test/test_tls_download_clock_gates.py @@ -190,7 +190,9 @@ class TlsDownloadClockGateTest(unittest.TestCase): ) self.assertIn("SntpOperationCoordinator.h", implementation) - self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 2) + # Background refresh has one ESP32 three-server branch and one portable + # single-server branch; the explicit administrator sync is separate. + self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 3) self.assertEqual( len(re.findall(r"(?m)^\s*sntp_set_sync_status\(", implementation)), 2, @@ -321,10 +323,15 @@ class TlsDownloadClockGateTest(unittest.TestCase): header, ) self.assertIn("MQTT_NTP_REFRESH_TIMEOUT_MS = 15000UL", implementation) - self.assertEqual(implementation.count("pollNtpRefresh("), 3) - self.assertGreaterEqual( - implementation.count("&& !_ntp_refresh_pending"), 2 + self.assertEqual(implementation.count("pollNtpRefresh("), 4) + background = section( + implementation, + "void MQTTBridge::serviceBackgroundNtp(", + "void MQTTBridge::mqttTaskLoop()", ) + self.assertIn("if (_ntp_refresh_pending) return;", background) + self.assertLess(background.index("if (_ntp_refresh_pending) return;"), + background.index("refreshNTP();")) reset = refresh.index("sntp_set_sync_status(SNTP_SYNC_STATUS_RESET)") configure = refresh.index( @@ -351,7 +358,7 @@ class TlsDownloadClockGateTest(unittest.TestCase): self.assertIn("_ntp_refresh_operation.owns()", poll) timeout = poll.index("elapsed >= MQTT_NTP_REFRESH_TIMEOUT_MS") - retry = poll.index("mqttNtpRetryAt(now)", timeout) + retry = poll.index("mqttNtpFailureRetryAt(now)", timeout) self.assertLess(timeout, retry) self.assertNotIn("_last_ntp_sync =", poll[timeout:]) diff --git a/test/test_webconfig_network_power.py b/test/test_webconfig_network_power.py new file mode 100644 index 000000000..1e01cf41f --- /dev/null +++ b/test/test_webconfig_network_power.py @@ -0,0 +1,151 @@ +"""Exercise the production portal's bounded foreground ownership policy.""" +from pathlib import Path +import unittest + +from test_replay_reset_integration import extract_braced +import test_wifi_ota_start as wifi_start + +ROOT = Path(__file__).resolve().parents[1] + + +class WebConfigNetworkPowerTest(unittest.TestCase): + compile_and_run = wifi_start.WiFiOtaStartTest.compile_and_run + + def fixture(self): + source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() + methods = "\n".join(extract_braced(source, signature) for signature in ( + "void WebConfigServer::noteNetworkActivity(", + "void WebConfigServer::requestNetworkAccess(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(")) + return r''' +#include +#include +#include +#include +#include +uint32_t now = 1000; +uint32_t millis() { return now; } +struct WCLock { explicit WCLock(void*) {} }; +struct WebConfigServer { + enum { MODE_OFF, MODE_SETUP, MODE_CONNECTING, MODE_LAN }; + enum { BATCH_IDLE, BATCH_PENDING, BATCH_DONE }; + int _mode = MODE_LAN, _batch_state = BATCH_IDLE; + bool _stopping = false, _setup_wifi_handoff_pending = false; + uint32_t _last_activity = 1000, _network_activity_until = 0; + unsigned _handler_refs = 0; + void* _mux = nullptr; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; + mesh::RoomWebMailbox* _room_mailbox = nullptr; + unsigned handlerRefCount() const { return _handler_refs; } + void noteNetworkActivity(); + void requestNetworkAccess(); + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); +}; +@METHODS@ +int main() { + WebConfigServer portal; + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + // Boot/connection timestamps and a listener alone never hold failed WiFi. + portal._last_activity = now; + assert(!portal.isNetworkForegroundActive(now)); + portal.refreshNetworkPowerHold(now); + assert(!power.foregroundActive()); + portal.requestNetworkAccess(); + assert(power.consumeProbeRequest() && power.foregroundActive()); + now += 119999; + assert(portal.isNetworkForegroundActive(now)); + ++now; portal.refreshNetworkPowerHold(now); + assert(!portal.isNetworkForegroundActive(now) && !power.foregroundActive()); + // The bounded lease works across millis() rollover. + now = UINT32_MAX - 100; + portal.noteNetworkActivity(); + now += 1000; + assert(portal.isNetworkForegroundActive(now)); + now += 119000; + assert(!portal.isNetworkForegroundActive(now)); + // Setup, teardown and in-flight handlers/batches survive lease expiry. + portal._mode = WebConfigServer::MODE_SETUP; + assert(portal.isNetworkForegroundActive(now)); + portal._mode = WebConfigServer::MODE_LAN; + portal._handler_refs = 1; + assert(portal.isNetworkForegroundActive(now)); + portal._handler_refs = 0; portal._batch_state = WebConfigServer::BATCH_PENDING; + assert(portal.isNetworkForegroundActive(now)); + portal._batch_state = WebConfigServer::BATCH_DONE; + assert(!portal.isNetworkForegroundActive(now)); + portal._setup_wifi_handoff_pending = true; + assert(portal.isNetworkForegroundActive(now)); + portal._setup_wifi_handoff_pending = false; + WebConfigServer::Terminal terminal; + portal._terminal = &terminal; terminal.running = true; + assert(portal.isNetworkForegroundActive(now)); + terminal.running = false; terminal.queue.pending = true; + assert(portal.isNetworkForegroundActive(now)); + terminal.queue.pending = false; + assert(!portal.isNetworkForegroundActive(now)); + mesh::RoomWebMailbox mailbox; uint8_t token[32] = {1}, digest[32] = {2}; + portal._room_mailbox = &mailbox; + assert(mailbox.submit(token, 1, digest, "{}", 2, now) + == mesh::RoomWebMailbox::Result::Accepted); + assert(portal.isNetworkForegroundActive(now)); + assert(mailbox.begin() && portal.isNetworkForegroundActive(now)); + mailbox.finish(now); + assert(!portal.isNetworkForegroundActive(now)); + portal._mode = WebConfigServer::MODE_OFF; portal._stopping = true; + assert(portal.isNetworkForegroundActive(now)); + portal._stopping = false; + portal.refreshNetworkPowerHold(now); + assert(!power.foregroundActive()); + power.setManaged(false); +} +'''.replace("@METHODS@", methods) + + def test_actual_activity_setup_and_inflight_work_hold_with_bounded_idle(self): + self.compile_and_run(self.fixture()) + + def test_status_reports_shared_cooldown_before_connecting_or_off(self): + source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() + formatter = extract_braced(source, "bool WebConfigServer::formatWiFiStatus(") + block = extract_braced(formatter, + " if (mesh::wifi::networkPower().backgroundSuspended()") + self.assertLess(formatter.index(block), + formatter.index("active->_mode == MODE_CONNECTING")) + fixture = r''' +#include +#include +#include +#include +void appendOtaSeederStatus(char* reply, size_t capacity) { + size_t used = strlen(reply); + snprintf(reply + used, capacity - used, "; seeder status"); +} +bool format(char* reply, size_t reply_len, bool configured) { + const char* ssid = "working-network"; +@BLOCK@ + snprintf(reply, reply_len, "> connecting"); + return true; +} +int main() { + char reply[160]; auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + assert(format(reply, sizeof(reply), true)); + assert(strcmp(reply, "> suspended; recovery every 30 minutes, SSID: working-network; seeder status") == 0); + assert(format(reply, sizeof(reply), false)); + assert(strstr(reply, "SSID: (not configured); seeder status")); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + power.setSuspended(false); + format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0); + power.setManaged(false); +} +'''.replace("@BLOCK@", block) + self.compile_and_run(fixture) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_webconfig_recovery_auth.py b/test/test_webconfig_recovery_auth.py index 0651d7b77..092de93e1 100644 --- a/test/test_webconfig_recovery_auth.py +++ b/test/test_webconfig_recovery_auth.py @@ -54,7 +54,7 @@ class WebConfigRecoveryAuthTest(unittest.TestCase): def test_old_interface_only_fallback_fails_executable_regression(self): fixture = self.fixture() - old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) return true;", "return true;", 1) + old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) { noteNetworkActivity(); return true; }", "return true;", 1) self.assertNotEqual(old, fixture) self.compile_run(old, expect_rejection=True) diff --git a/test/test_webconfig_station_ownership.py b/test/test_webconfig_station_ownership.py index ef0125815..f5cf0ea39 100644 --- a/test/test_webconfig_station_ownership.py +++ b/test/test_webconfig_station_ownership.py @@ -87,6 +87,10 @@ class WebConfigStationOwnershipTest(unittest.TestCase): def test_non_mqtt_portal_retains_saved_station_recovery(self): self.compile_and_run(self.station_fixture()) + def test_shared_outage_blocks_idle_portal_reconnect_and_allows_foreground(self): + self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1", + "-DTEST_BACKGROUND_SUSPEND=1") + def test_manual_companion_start_preserves_legacy_join_and_15s_fallback(self): self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1", "-DTEST_LEGACY_COMPANION=1") diff --git a/test/test_wifi_network_power_policy.py b/test/test_wifi_network_power_policy.py new file mode 100644 index 000000000..b3b2f7e51 --- /dev/null +++ b/test/test_wifi_network_power_policy.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Run the shared WiFi outage policy against deterministic clocks and owners. + +The production header is compiled directly, without Arduino or PlatformIO. +Independent translation units and concurrent service holds exercise the actual +handoff used between the network worker and foreground service owners. +""" + +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include + +using mesh::wifi::NetworkPowerControl; +using mesh::wifi::NetworkPowerHold; +using mesh::wifi::NetworkPowerPolicy; +using mesh::wifi::networkPower; +using Action = NetworkPowerPolicy::Action; + +extern NetworkPowerControl* otherControl(); +extern void otherSetWebHold(bool value); +extern bool otherSuspended(); + +static void require(bool value, const char* reason) { + if (!value) throw std::runtime_error(reason); +} + +static void expect(NetworkPowerPolicy& policy, uint32_t now, + bool useful, bool foreground, Action expected, + bool probe = false) { + require(policy.update(now, useful, foreground, probe) == expected, + "unexpected network power transition"); + require(policy.suspended() == (expected == Action::Suspend), + "transition and suspension state disagree"); +} + +// Keep expected timings independent of the production constants. Altering a +// timeout must be an intentional policy change, rather than a self-passing test. +static const uint32_t grace = 300000U; +static const uint32_t probe_budget = 120000U; +static const uint32_t cooldown = 1800000U; + +static void graceBoundary() { + for (uint32_t start : {0U, 1234U, UINT32_MAX - 10000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + expect(policy, start + grace - 1U, false, false, Action::Run); + expect(policy, start + grace, false, false, Action::Suspend); + expect(policy, start + grace + 1U, false, false, Action::Suspend); + } +} + +static void failedServicesCannotRenew() { + // Serial event stream: associations succeed, SNTP alternates success/failure, + // and MQTT handshakes keep failing. None means the configured broker works. + enum Event { GotIp, NtpTimeout, NtpReply, MqttConnectFailure, MqttRetry }; + const Event events[] = {GotIp, NtpTimeout, MqttConnectFailure, + NtpReply, MqttRetry}; + for (uint32_t start : {0U, UINT32_MAX - 70000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + for (uint32_t offset = 1000U; offset < grace; offset += 1000U) { + const Event event = events[(offset / 1000U) % 5U]; + require(event == GotIp || event == NtpTimeout || event == NtpReply + || event == MqttConnectFailure || event == MqttRetry, + "invalid simulated network event"); + expect(policy, start + offset, false, false, Action::Run); + } + expect(policy, start + grace, false, false, Action::Suspend); + // Both services keep wanting retries. Their failure notifications cannot + // restart either the grace period or the 30-minute suspension clock. + for (uint32_t offset = 1000U; offset < cooldown; offset += 1000U) { + expect(policy, start + grace + offset, false, false, Action::Suspend); + } + expect(policy, start + grace + cooldown, false, false, Action::Resume); + } +} + +static void successfulBrokerRenews() { + for (uint32_t start : {0U, UINT32_MAX - 200000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + expect(policy, start + grace - 1U, true, false, Action::Run); + const uint32_t last_healthy = start + 2U * grace; + expect(policy, last_healthy, true, false, Action::Run); + expect(policy, last_healthy + grace - 1U, false, false, Action::Run); + expect(policy, last_healthy + grace, false, false, Action::Suspend); + // A genuine healthy connection observed during suspension cancels it and + // receives the normal grace budget, rather than a short recovery probe. + const uint32_t recovered = last_healthy + grace + 20U; + expect(policy, recovered, true, false, Action::Resume); + expect(policy, recovered + probe_budget, false, false, Action::Run); + expect(policy, recovered + grace, false, false, Action::Suspend); + } +} + +static void boundedRecoveryCycles() { + for (uint32_t start : {0U, UINT32_MAX - 10000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + uint32_t suspended_at = start + grace; + expect(policy, suspended_at, false, false, Action::Suspend); + for (unsigned cycle = 0; cycle < 20; ++cycle) { + expect(policy, suspended_at + cooldown - 1U, false, false, + Action::Suspend); + const uint32_t resumed_at = suspended_at + cooldown; + expect(policy, resumed_at, false, false, Action::Resume); + for (uint32_t offset = 1000U; offset < probe_budget; offset += 1000U) { + expect(policy, resumed_at + offset, false, false, Action::Run); + } + expect(policy, resumed_at + probe_budget - 1U, false, false, + Action::Run); + suspended_at = resumed_at + probe_budget; + expect(policy, suspended_at, false, false, Action::Suspend); + } + } +} + +static void foregroundPreservesAccess() { + NetworkPowerPolicy policy; + expect(policy, 0, false, true, Action::Run); + expect(policy, grace, false, true, Action::Run); + expect(policy, 3U * grace, false, true, Action::Run); + // Closing the foreground session does not grant a new background grace. + expect(policy, 3U * grace + 1U, false, false, Action::Suspend); + expect(policy, 3U * grace + 2U, false, true, Action::Resume); + expect(policy, 3U * grace + 2U + probe_budget, false, true, Action::Run); + expect(policy, 3U * grace + 3U + probe_budget, false, false, + Action::Suspend); +} + +static void manualProbeOnlyOnDemand() { + NetworkPowerPolicy policy; + expect(policy, 100U, false, false, Action::Run); + expect(policy, 100U + grace, false, false, Action::Suspend); + expect(policy, 101U + grace, false, false, Action::Suspend); + const uint32_t requested_at = 102U + grace; + expect(policy, requested_at, false, false, Action::Resume, true); + // Repeated manual requests while running cannot renew the probe deadline. + expect(policy, requested_at + probe_budget - 1U, false, false, + Action::Run, true); + expect(policy, requested_at + probe_budget, false, false, + Action::Suspend, true); + expect(policy, requested_at + probe_budget + 1U, false, false, + Action::Suspend); + expect(policy, requested_at + probe_budget + cooldown - 1U, false, false, + Action::Suspend); + expect(policy, requested_at + probe_budget + cooldown, false, false, + Action::Resume); +} + +static void usefulProbeBecomesNormalOperation() { + NetworkPowerPolicy policy; + expect(policy, 0, false, false, Action::Run); + expect(policy, grace, false, false, Action::Suspend); + const uint32_t resumed_at = grace + cooldown; + expect(policy, resumed_at, false, false, Action::Resume); + const uint32_t recovered_at = resumed_at + probe_budget - 1U; + expect(policy, recovered_at, true, false, Action::Run); + expect(policy, recovered_at + probe_budget, false, false, Action::Run); + expect(policy, recovered_at + grace - 1U, false, false, Action::Run); + expect(policy, recovered_at + grace, false, false, Action::Suspend); +} + +static void clearControl(NetworkPowerControl& control) { + control.setManaged(false); + control.setHold(NetworkPowerHold::CompanionClient, false); + control.setHold(NetworkPowerHold::WebConfig, false); + control.setHold(NetworkPowerHold::OtaSeeder, false); + control.consumeProbeRequest(); +} + +static void programWideRegistry() { + NetworkPowerControl& control = networkPower(); + clearControl(control); + require(&control == otherControl(), "service translation units split registry"); + control.setManaged(true); + control.setSuspended(true); + require(otherSuspended(), "worker suspension invisible to station owner"); + otherSetWebHold(true); + require(control.foregroundActive(), "foreground hold invisible to worker"); + control.setHold(NetworkPowerHold::OtaSeeder, true); + otherSetWebHold(false); + require(control.foregroundActive(), "closing web session cleared OTA hold"); + control.setHold(NetworkPowerHold::OtaSeeder, false); + require(!control.foregroundActive(), "closed services retained WiFi lease"); + control.requestProbe(); + require(otherControl()->consumeProbeRequest(), "probe invisible across tasks"); + require(!control.consumeProbeRequest(), "probe request was not one-shot"); + control.setManaged(false); + require(!otherSuspended(), "disabled manager retained suspension"); +} + +static void controlLifecycle() { + NetworkPowerControl control; + require(!control.managed() && !control.backgroundSuspended() + && !control.foregroundActive() && !control.consumeProbeRequest(), + "new control has active service state"); + control.setSuspended(true); + require(!control.backgroundSuspended(), "unmanaged owner imposed suspension"); + control.setManaged(true); + require(control.backgroundSuspended(), "managed suspension was lost"); + control.requestProbe(); + control.setManaged(false); + require(!control.backgroundSuspended() && !control.consumeProbeRequest(), + "stopping worker preserved stale suspension or probe"); + control.setHold(NetworkPowerHold::CompanionClient, true); + control.setManaged(true); + require(!control.backgroundSuspended() && control.foregroundActive(), + "restarting manager lost foreground owner or kept stale suspension"); + control.setHold(NetworkPowerHold::CompanionClient, false); + require(!control.foregroundActive(), "last foreground owner failed to release"); +} + +static void concurrentServiceHolds() { + NetworkPowerControl control; + const NetworkPowerHold owners[] = {NetworkPowerHold::CompanionClient, + NetworkPowerHold::WebConfig, + NetworkPowerHold::OtaSeeder}; + for (unsigned round = 0; round < 256; ++round) { + std::atomic ready{0}; + std::atomic go{false}; + std::atomic release[3] = {}; + std::atomic removed[3] = {}; + std::vector threads; + for (unsigned index = 0; index < 3; ++index) { + threads.emplace_back([&, index]() { + ready.fetch_add(1, std::memory_order_release); + while (!go.load(std::memory_order_acquire)) std::this_thread::yield(); + control.setHold(owners[index], true); + ready.fetch_add(1, std::memory_order_release); + while (!release[index].load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + control.setHold(owners[index], false); + removed[index].store(true, std::memory_order_release); + }); + } + while (ready.load(std::memory_order_acquire) != 3U) std::this_thread::yield(); + go.store(true, std::memory_order_release); + while (ready.load(std::memory_order_acquire) != 6U) std::this_thread::yield(); + const bool opens_preserved = control.foregroundActive(); + bool remaining_holds_preserved = true; + for (unsigned index = 0; index < 3; ++index) { + release[index].store(true, std::memory_order_release); + while (!removed[index].load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (index < 2 && !control.foregroundActive()) remaining_holds_preserved = false; + } + for (auto& thread : threads) thread.join(); + require(opens_preserved, "simultaneous opens lost all leases"); + require(remaining_holds_preserved, "one owner cleared another owner's lease"); + require(!control.foregroundActive(), "concurrent closes leaked WiFi hold"); + } +} + +static void serviceLeaseEventSequence() { + NetworkPowerPolicy policy; + NetworkPowerControl control; + control.setManaged(true); + auto update = [&](uint32_t now, bool useful = false) { + const Action action = policy.update(now, useful, control.foregroundActive(), + control.consumeProbeRequest()); + control.setSuspended(action == Action::Suspend); + return action; + }; + require(update(0) == Action::Run, "network failed to start"); + require(update(grace) == Action::Suspend, "outage left WiFi active"); + control.setHold(NetworkPowerHold::WebConfig, true); + require(update(grace + 1U) == Action::Resume, "admin did not wake network"); + control.setHold(NetworkPowerHold::OtaSeeder, true); + control.setHold(NetworkPowerHold::WebConfig, false); + require(update(grace + probe_budget + 1U) == Action::Run, + "closing admin page interrupted active OTA"); + control.setHold(NetworkPowerHold::OtaSeeder, false); + require(update(grace + probe_budget + 2U) == Action::Suspend, + "completed OTA kept failed background network on"); + control.requestProbe(); + require(update(grace + probe_budget + 3U) == Action::Resume, + "explicit retry could not recover network"); + require(!control.consumeProbeRequest(), "owner probe did not get consumed"); + require(update(grace + probe_budget + 4U, true) == Action::Run, + "healthy broker did not resume normal service"); + require(!control.backgroundSuspended(), "healthy broker left station suspended"); +} + +int main(int argc, char** argv) { + if (argc != 2) return 2; + try { + const char* name = argv[1]; + if (!strcmp(name, "grace")) graceBoundary(); + else if (!strcmp(name, "failures")) failedServicesCannotRenew(); + else if (!strcmp(name, "healthy")) successfulBrokerRenews(); + else if (!strcmp(name, "cycles")) boundedRecoveryCycles(); + else if (!strcmp(name, "foreground")) foregroundPreservesAccess(); + else if (!strcmp(name, "manual")) manualProbeOnlyOnDemand(); + else if (!strcmp(name, "recovery")) usefulProbeBecomesNormalOperation(); + else if (!strcmp(name, "registry")) programWideRegistry(); + else if (!strcmp(name, "lifecycle")) controlLifecycle(); + else if (!strcmp(name, "threads")) concurrentServiceHolds(); + else if (!strcmp(name, "leases")) serviceLeaseEventSequence(); + else return 2; + } catch (const std::exception& error) { + std::cerr << argv[1] << ": " << error.what() << '\n'; + return 1; + } + std::cout << "PASS: " << argv[1] << '\n'; + return 0; +} +''' + +OTHER_OWNER = r''' +#include + +mesh::wifi::NetworkPowerControl* otherControl() { + return &mesh::wifi::networkPower(); +} +void otherSetWebHold(bool value) { + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, value); +} +bool otherSuspended() { + return mesh::wifi::networkPower().backgroundSuspended(); +} +''' + + +class WiFiNetworkPowerPolicyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + raise RuntimeError("a host C++ compiler is required") + cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-wifi-outage-") + cls.addClassCleanup(cls.temporary.cleanup) + work = Path(cls.temporary.name) + source = work / "policy.cpp" + owner = work / "owner.cpp" + source.write_text(HARNESS, encoding="ascii") + owner.write_text(OTHER_OWNER, encoding="ascii") + cls.binary = work / "policy" + built = subprocess.run( + [compiler, "-std=c++11", "-pthread", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(source), str(owner), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60, + ) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + + def run_case(self, name): + result = subprocess.run([str(self.binary), name], capture_output=True, + text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("PASS: " + name, result.stdout) + + def test_initial_five_minute_grace_and_millis_rollover(self): + self.run_case("grace") + + def test_ntp_replies_association_and_mqtt_failures_do_not_renew_outage(self): + self.run_case("failures") + + def test_healthy_broker_renews_grace_and_cancels_suspension(self): + self.run_case("healthy") + + def test_two_minute_probes_and_thirty_minute_cooldowns_repeat_bounded(self): + self.run_case("cycles") + + def test_foreground_access_resumes_and_protects_wifi_until_released(self): + self.run_case("foreground") + + def test_requested_probe_is_explicit_and_does_not_extend_active_probe(self): + self.run_case("manual") + + def test_successful_probe_returns_to_normal_grace(self): + self.run_case("recovery") + + def test_two_translation_units_share_owner_holds_and_probe_requests(self): + self.run_case("registry") + + def test_worker_lifecycle_clears_stale_suspension_and_probe(self): + self.run_case("lifecycle") + + def test_concurrent_foreground_owners_preserve_each_others_holds(self): + self.run_case("threads") + + def test_admin_and_ota_lease_sequence_during_combined_background_outage(self): + self.run_case("leases") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_wifi_ota_start.py b/test/test_wifi_ota_start.py index 67099d349..788783ced 100644 --- a/test/test_wifi_ota_start.py +++ b/test/test_wifi_ota_start.py @@ -343,7 +343,9 @@ int main() { source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() methods = "\n".join(extract_braced(source, signature) for signature in ( "void WebConfigServer::requestStop(", "bool WebConfigServer::stopForOTA(", - "void WebConfigServer::finalizeTeardown(")) + "void WebConfigServer::finalizeTeardown(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(")) fixture = (ROOT / "test/fixtures/webconfig_ota_handoff.cpp").read_text() self.compile_and_run(fixture.replace("@METHODS@", methods))