diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 82c882a28..296bfe57a 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -222,10 +222,15 @@ jobs: python3 -B test/test_mqtt_transport_results.py -v python3 -B test/test_mqtt_tls_security.py -v python3 -B test/test_mqtt_network_timeout.py -v + python3 -B test/test_mqtt_disconnect.py -v python3 -B test/test_mqtt_connection_admission.py -v python3 -B test/test_mqtt_handshake_scheduler.py -v python3 -B test/test_mqtt_diagnostics.py -v python3 -B test/test_mqtt_publish_ack.py -v + python3 -B test/test_wifi_network_power_policy.py -v + python3 -B test/test_mqtt_wifi_network_power.py -v + python3 -B test/test_companion_wifi_network_power.py -v + python3 -B test/test_webconfig_network_power.py -v - name: Verify DNS lease restoration and bounded diagnostics run: python3 -B test/test_network_dns.py -v diff --git a/docs/WiFi.md b/docs/WiFi.md index bea2b7a2a..694b375db 100644 --- a/docs/WiFi.md +++ b/docs/WiFi.md @@ -102,6 +102,32 @@ best effort rather than durable storage. Each enabled slot publishes independently, so one failed broker does not intentionally stop the other slots. +## Automatic network outage power saving + +ESP32 MQTT firmware uses one shared outage timer for MQTT and NTP. If no broker +demonstrates a working connection for five minutes, the MQTT worker stops its +clients and NTP request before suspending background WiFi. Saved WiFi and broker +settings remain enabled. Failed requests, a WiFi association, and NTP replies +alone do not restart the outage timer or keep a failed MQTT uplink awake. + +After 30 minutes, the firmware opens a recovery window lasting up to two +minutes. A broker acknowledgment or a connection that stays up for 30 seconds +returns the node to normal operation. Otherwise WiFi is suspended for another +30 minutes. An explicit management request can ask for an earlier probe. +Automatic NTP requests have a 15-second timeout; failed requests wait 30 minutes +instead of repeatedly running a blocking server search. Successful NTP retains +the daily refresh schedule. + +Active browser administration, setup/OTA access points, connected Companion +clients, and an attached WiFi OTA seeder protect their network access. An idle +WebConfig listener does not prevent the cooldown. An enabled SNMP agent keeps a +working LAN connection available. ESP-NOW keeps its radio and channel; only the +failed infrastructure connection is suspended. LoRa continues operating, and +when CPU power saving is enabled, its normal sleep eligibility returns after +background WiFi shuts down. USB console and logging sleep rules still apply +separately. This policy does not automatically turn off working services or +expire an active setup access point. + See [MQTT_IMPLEMENTATION.md](https://github.com/mikecarper/MeshCore/blob/keymindCascade/MQTT_IMPLEMENTATION.md) for the complete preset list, custom broker configuration, topic formats, authentication, diagnostics, and memory limits. diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index a24f467cb..8538dc4a6 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -4721,7 +4721,10 @@ void MyMesh::serviceMQTT(const char* wifi_ssid, const char* wifi_password) { } } - if (WiFi.status() == WL_CONNECTED && _mqtt_configured && !_mqtt_started && _mqtt_bridge + // Start the recovery scheduler even when the router is down at boot. The + // Companion still owns association; waiting for GOT_IP here would leave its + // failed station outside MQTT/NTP's shared power budget indefinitely. + if (_mqtt_configured && !_mqtt_started && _mqtt_bridge #ifdef WITH_WEBCONFIG && !isWebConfigSetupActive() #endif diff --git a/examples/companion_radio/main.cpp b/examples/companion_radio/main.cpp index 364621874..69d27e5d3 100644 --- a/examples/companion_radio/main.cpp +++ b/examples/companion_radio/main.cpp @@ -100,6 +100,7 @@ MultiSerialInterface interface_manager; #include #include #include + #include #include #include #include @@ -1508,6 +1509,12 @@ void halt() { static bool companion_wifi_active = false; static bool companion_wifi_disable_in_progress = false; static bool companion_wifi_services_stopped = false; + static bool companion_wifi_background_stopped = false; + static bool companion_wifi_espnow_restore_pending = false; + static uint32_t companion_wifi_espnow_restore_at = 0; +#ifdef WITH_WEBCONFIG + static bool companion_wifi_webconfig_start_pending = false; +#endif static volatile bool companion_wifi_setup_requested = false; static volatile bool companion_wifi_setup_stop_requested = false; static bool companion_wifi_credential_reload_pending = false; @@ -1634,6 +1641,10 @@ void halt() { } static void serviceCompanionWiFiNtp() { + if (mesh::wifi::networkPower().backgroundSuspended()) { + cancelCompanionWiFiNtp(false); + return; + } // Configured MQTT builds already perform a fresh startup NTP sync and a // daily refresh. The common Companion owner covers every non-MQTT or // runtime-unconfigured WiFi node without racing MQTT's SNTP proof. @@ -1832,6 +1843,7 @@ void halt() { void scheduleCompanionWiFiCredentialReload() { loadCompanionWiFiCredentials(); + mesh::wifi::networkPower().requestProbe(); companion_wifi_credential_reload_pending = true; companion_wifi_credential_reload_at = millis() + 1500UL; if (companion_wifi_credential_reload_at == 0) { @@ -1898,6 +1910,7 @@ void halt() { void requestCompanionWiFiSetup() { if (mesh::wireless::control().blocked(mesh::wireless::WiFi)) return; + mesh::wifi::networkPower().requestProbe(); companion_wifi_setup_stop_requested = false; companion_wifi_setup_requested = true; } @@ -1921,6 +1934,7 @@ void halt() { return false; } companion_wifi_requested = enabled; + if (enabled) mesh::wifi::networkPower().requestProbe(); if (!companion_wifi_requested && companion_wifi_active) { companion_wifi_disable_in_progress = true; } @@ -2230,6 +2244,60 @@ void halt() { #endif #if defined(ESP32) && defined(WIFI_SSID) + static bool companionWiFiBackgroundSuspended() { + // MQTT publishes this flag only after its own TLS and SNTP work is quiet. + // A foreground OTA command can acquire the radio before its next pass. + return mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive() + && !board.isOTAUpdateRunning(); + } + + static bool companionWiFiAutomaticRecoveryAllowed() { + // MQTT owns one combined outage/cooldown clock. Raising an automatic AP + // during that grace period would create a foreground owner indefinitely. + return !mesh::wifi::networkPower().managed(); + } + +#ifdef WITH_WEBCONFIG + static void startCompanionWiFiAutoWebConfig() { + companion_wifi_webconfig_start_pending = false; + if (!WebConfigServer::loadEnabled(true)) return; + if (companion_wifi_has_credentials && WiFi.status() != WL_CONNECTED) { + // The Companion owns the initial saved-SSID attempt. Do not ask the + // portal to start its separate fifteen-second AP fallback at boot. + companion_wifi_webconfig_start_pending = true; + return; + } + char web_reply[160]; + the_mesh.startWebConfig(!companion_wifi_has_credentials, web_reply); + WIFI_DEBUG_PRINTLN("%s", web_reply); + } + + static void serviceCompanionWiFiAutoWebConfig() { + if (!companion_wifi_webconfig_start_pending || !companion_wifi_requested + || !companion_wifi_active || companionWiFiBackgroundSuspended()) return; + if (!WebConfigServer::loadEnabled(true)) { + companion_wifi_webconfig_start_pending = false; + return; + } + if (WiFi.status() != WL_CONNECTED + || the_mesh.isWebConfigActiveOrStopping()) return; + startCompanionWiFiAutoWebConfig(); + } +#endif + + static void serviceCompanionWiFiNetworkPowerHold() { + bool active = wifi_interface.isConnected() + || companion_wifi_credential_reload_pending + || companion_wifi_setup_requested + || board.isOTAUpdateRunning(); +#ifdef ENABLE_OTA + active = active || ota_console_client.connected(); +#endif + mesh::wifi::networkPower().setHold( + mesh::wifi::NetworkPowerHold::CompanionClient, active); + } + static void resetCompanionWiFiRecoveryState() { wifi_reconnect_tracker = WiFiReconnectPolicy::Tracker(); wifi_setup_attempted = false; @@ -2242,6 +2310,7 @@ void halt() { || static_cast(millis() - companion_wifi_credential_reload_at) < 0) { return; } + if (companionWiFiBackgroundSuspended()) return; #ifdef WITH_WEBCONFIG if (the_mesh.isWebConfigActiveOrStopping()) return; #endif @@ -2250,6 +2319,7 @@ void halt() { companion_wifi_credential_reload_pending = false; companion_wifi_credential_reload_at = 0; + companion_wifi_espnow_restore_pending = false; loadCompanionWiFiCredentials(); resetCompanionWiFiRecoveryState(); if (!companion_wifi_requested || !companion_wifi_active @@ -2270,9 +2340,11 @@ void halt() { static void startCompanionWiFi() { if (mesh::wireless::control().blocked(mesh::wireless::WiFi)) return; if (companion_wifi_active) return; + if (companionWiFiBackgroundSuspended()) return; if (!mesh::wifi::stationMutationAllowed()) return; board.setInhibitSleep(true); + companion_wifi_espnow_restore_pending = false; mesh::wifi::setStationAutoReconnect(true); resetCompanionWiFiRecoveryState(); @@ -2290,11 +2362,7 @@ void halt() { } #endif #ifdef WITH_WEBCONFIG - if (WebConfigServer::loadEnabled(true)) { - char web_reply[160]; - the_mesh.startWebConfig(!companion_wifi_has_credentials, web_reply); - WIFI_DEBUG_PRINTLN("%s", web_reply); - } + startCompanionWiFiAutoWebConfig(); #endif applyCompanionWiFiPowerSave(); @@ -2305,6 +2373,7 @@ void halt() { #endif companion_wifi_active = true; companion_wifi_services_stopped = false; + companion_wifi_background_stopped = false; WIFI_DEBUG_PRINTLN("WiFi enabled by BOOT/GPIO 0 control"); } @@ -2329,17 +2398,80 @@ void halt() { static void stopCompanionInfrastructureWiFi() { WiFi.setAutoReconnect(false); -#if defined(MESH_PRIMARY_ESPNOW) && MESH_PRIMARY_ESPNOW - // ESP-NOW is the mesh radio on this target. Drop the station/AP services - // without stopping the driver, then restore its fixed LR mesh channel. - WiFi.disconnect(false, false); - WiFi.mode(WIFI_STA); - mesh::wifi::applyProtocolMask(WIFI_IF_STA); - mesh::wifi::restoreEspNowChannel(); -#else - WiFi.disconnect(true, false); - WiFi.mode(WIFI_OFF); + if (mesh::wifi::espNowChannelConstrained()) { + // Primary ESP-NOW and a live optional bridge both share this driver. + // Drop only station association and keep their fixed mesh channel. + WiFi.disconnect(false, false); + WiFi.mode(WIFI_STA); + mesh::wifi::applyProtocolMask(WIFI_IF_STA); + companion_wifi_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + companion_wifi_espnow_restore_at = millis() + 1000UL; + } else { + companion_wifi_espnow_restore_pending = false; + WiFi.disconnect(true, false); + WiFi.mode(WIFI_OFF); + } + } + + static void serviceCompanionEspNowChannelRestore() { + if (!companion_wifi_espnow_restore_pending) return; + if (!mesh::wifi::espNowChannelConstrained()) { + companion_wifi_espnow_restore_pending = false; + return; + } + if (board.isOTAUpdateRunning() + || mesh::wifi::networkPower().foregroundActive() + || !mesh::wifi::stationMutationAllowed()) return; + const uint32_t now = millis(); + if (static_cast(now - companion_wifi_espnow_restore_at) < 0) return; + // esp_wifi_set_channel may fail until an asynchronous disconnect/scan + // completes. Retry the channel only; repeated disconnects would prevent + // the shared ESP-NOW receiver from recovering during a long cooldown. + companion_wifi_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + companion_wifi_espnow_restore_at = now + 1000UL; + } + + static void serviceCompanionWiFiBackgroundPower() { + serviceCompanionEspNowChannelRestore(); + if (!companion_wifi_active || !companion_wifi_requested) return; + if (companionWiFiBackgroundSuspended()) { + if (companion_wifi_background_stopped) return; + // Settings and MQTT's scheduling task stay enabled. These idle socket + // listeners and the common NTP request own no useful foreground work. + cancelCompanionWiFiNtp(false); + wifi_interface.end(); +#ifdef ENABLE_OTA + ota_console_stop(); #endif + stopCompanionInfrastructureWiFi(); + board.setInhibitSleep(mesh::wifi::espNowChannelConstrained()); + companion_wifi_background_stopped = true; + WIFI_DEBUG_PRINTLN("Network unavailable; WiFi suspended until the next recovery probe"); + return; + } + if (!companion_wifi_background_stopped) return; + if (!mesh::wifi::stationMutationAllowed()) return; + + board.setInhibitSleep(true); + companion_wifi_espnow_restore_pending = false; + resetCompanionWiFiRecoveryState(); + if (companion_wifi_has_credentials) { + WiFi.mode(WIFI_STA); + mesh::wifi::applyProtocolMask(WIFI_IF_STA); + mesh::wifi::setStationAutoReconnect(true); + wifi_reconnect_tracker.noteDisconnected(millis()); + mesh::wifi::beginStation(configured_wifi_ssid, configured_wifi_password); + } + applyCompanionWiFiPowerSave(); + wifi_interface.begin(TCP_PORT); + wifi_interface.enable(); +#ifdef ENABLE_OTA + ota_console_start(); +#endif + companion_wifi_background_stopped = false; + WIFI_DEBUG_PRINTLN("WiFi resumed for a bounded network recovery probe"); } static bool finishStoppingCompanionWiFi() { @@ -2368,6 +2500,10 @@ void halt() { board.setInhibitSleep(false); resetCompanionWiFiRecoveryState(); companion_wifi_active = false; + companion_wifi_background_stopped = false; +#ifdef WITH_WEBCONFIG + companion_wifi_webconfig_start_pending = false; +#endif #if defined(MESH_PRIMARY_ESPNOW) && MESH_PRIMARY_ESPNOW WIFI_DEBUG_PRINTLN( "Companion WiFi services disabled; ESP-NOW mesh radio remains active"); @@ -2388,6 +2524,10 @@ void halt() { if (companion_wifi_requested && !companion_wifi_active) { startCompanionWiFi(); } + serviceCompanionWiFiBackgroundPower(); +#ifdef WITH_WEBCONFIG + serviceCompanionWiFiAutoWebConfig(); +#endif } #endif @@ -3014,7 +3154,25 @@ static bool companionNativeUsbLightSleepReady() { // Full compiles every transport, but a compiled driver is not an active // receiver. Preserve live services and pending replies while permitting // the same short sleep slice when those services have been turned off. - if (companion_wireless.enabled() != 0 || interface_manager.hasPendingIO() + uint8_t enabled_services = companion_wireless.enabled(); +#if defined(ESP32) && defined(WIFI_SSID) + // The user's saved WiFi preference and an idle WebConfig listener remain + // enabled during automatic suspension. Only a confirmed physical power-off + // releases their sleep blocker; foreground work and shared ESP-NOW retain it. + if (companion_wifi_background_stopped + && mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive() + && !mesh::wifi::espNowChannelConstrained()) { + wifi_mode_t parked_mode = WIFI_MODE_NULL; + const esp_err_t parked_result = esp_wifi_get_mode(&parked_mode); + if ((parked_result == ESP_OK && parked_mode == WIFI_MODE_NULL) + || parked_result == ESP_ERR_WIFI_NOT_INIT + || parked_result == ESP_ERR_WIFI_NOT_STARTED) { + enabled_services &= static_cast(~mesh::wireless::WiFi); + } + } +#endif + if (enabled_services != 0 || interface_manager.hasPendingIO() || board.isOTAUpdateRunning() || board.isRadioTestActive() || sensors.gpsUsesSerialUart(1)) return false; #if defined(BLE_PIN_CODE) && !CONFIG_IDF_TARGET_ESP32C6 @@ -3624,6 +3782,9 @@ void loop() { } #endif mesh::wireless::control().service(millis()); +#if defined(ESP32) && defined(WIFI_SSID) + serviceCompanionWiFiNetworkPowerHold(); +#endif #if defined(NRF52_PLATFORM) board.feedWatchdog(); #endif @@ -3800,8 +3961,11 @@ void loop() { } #endif serviceCompanionWiFiCredentialReload(); + serviceCompanionWiFiNetworkPowerHold(); serviceCompanionWiFiState(); - if (companion_wifi_requested && companion_wifi_active) { + if (companion_wifi_requested && companion_wifi_active + && !companionWiFiBackgroundSuspended() + && !companion_wifi_background_stopped) { #ifdef ENABLE_OTA ota_console_loop(); // service the local text console (port 5002) #endif @@ -3847,7 +4011,7 @@ void loop() { } } else if (configured_wifi_ssid[0]) { wifi_reconnect_tracker.noteDisconnected(wifi_now); - if (!wifi_setup_recovery_mode + if (!wifi_setup_recovery_mode && companionWiFiAutomaticRecoveryAllowed() #ifdef WITH_WEBCONFIG && !the_mesh.isWebConfigSetupActive() #else diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index a346e6348..6f742ec36 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1,6 +1,7 @@ #include "MyMesh.h" #if defined(ESP32_PLATFORM) #include +#include #if defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ || defined(LIGHTWEIGHT_WIFI_OTA) || (defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA)) #include @@ -11125,6 +11126,9 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup return true; } if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + if (!automatic_setup && !_webconfig->isStopping()) { + _webconfig->requestNetworkAccess(); + } strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); return true; @@ -11163,6 +11167,7 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup } } + if (!automatic_setup) _webconfig->requestNetworkAccess(); const bool setup_was_suspended = _unconfigured_setup_espnow_suspended; #ifdef WITH_ESPNOW_BRIDGE const bool espnow_was_running = isEspNowBridgeRunning(); @@ -13819,11 +13824,15 @@ void MyMesh::serviceIdleWiFi() { && (!defined(MESH_PRIMARY_ESPNOW) || !MESH_PRIMARY_ESPNOW) \ && (!defined(MESH_ESPNOW_RADIO) || !MESH_ESPNOW_RADIO) if (_cli.getBoard()->isOTAUpdateRunning()) return; + if (mesh::wifi::networkPower().foregroundActive()) return; #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return; #endif #ifdef WITH_MQTT_BRIDGE if (_ota_update_at) return; + // The MQTT worker owns physical shutdown and resume, including cooldown. + // Never race its Core 0 Arduino/SDK interface transition from this loop. if (mqtt_bridge && (mqtt_bridge->isRunning() || mqtt_bridge->isStopping())) return; #endif #ifdef WITH_ESPNOW_BRIDGE @@ -13852,8 +13861,12 @@ void MyMesh::serviceIdleWiFi() { bool MyMesh::hasPendingWork() const { if (isDualRadioActive()) return true; if (hasPendingOtaApply()) return true; +#if defined(ESP32_PLATFORM) + if (mesh::wifi::networkPower().foregroundActive()) return true; +#endif #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return true; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return true; #endif #if defined(ESP32_PLATFORM) \ && (defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ @@ -13871,7 +13884,13 @@ bool MyMesh::hasPendingWork() const { if (deferred_cli_command.pending || pending_self_advert || _cli.hasActiveUserGpioTimer()) return true; #if defined(WITH_BRIDGE) const AbstractBridge* active_bridge = activeBridge(); - if (active_bridge && active_bridge->isRunning()) return true; + if (active_bridge && active_bridge->isRunning()) { +#ifdef WITH_MQTT_BRIDGE + if (active_bridge != mqtt_bridge || !mqtt_bridge->isNetworkSuspended()) return true; +#else + return true; +#endif + } #ifdef WITH_MQTT_BRIDGE if (mqtt_bridge && mqtt_bridge->isStopping()) return true; #endif diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 5b4d1dc32..75a384848 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1,6 +1,7 @@ #include "MyMesh.h" #if defined(ESP32_PLATFORM) #include +#include #if defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ || defined(LIGHTWEIGHT_WIFI_OTA) || (defined(ADMIN_PASSWORD) && !defined(DISABLE_WIFI_OTA)) #include @@ -3119,6 +3120,9 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup return true; } if (_webconfig && (_webconfig->isRunning() || _webconfig->isStopping())) { + if (!automatic_setup && !_webconfig->isStopping()) { + _webconfig->requestNetworkAccess(); + } strcpy(reply, _webconfig->isStopping() ? "Err: webconfig still stopping, retry shortly" : "Err: webconfig already running"); return true; @@ -3154,6 +3158,7 @@ bool MyMesh::startWebConfigImpl(bool force_ap, char* reply, bool automatic_setup } } + if (!automatic_setup) _webconfig->requestNetworkAccess(); const bool setup_was_suspended = _unconfigured_setup_espnow_suspended; #ifdef WITH_ESPNOW_BRIDGE const bool espnow_was_running = isEspNowBridgeRunning(); @@ -4198,11 +4203,15 @@ void MyMesh::serviceIdleWiFi() { && (!defined(MESH_PRIMARY_ESPNOW) || !MESH_PRIMARY_ESPNOW) \ && (!defined(MESH_ESPNOW_RADIO) || !MESH_ESPNOW_RADIO) if (_cli.getBoard()->isOTAUpdateRunning()) return; + if (mesh::wifi::networkPower().foregroundActive()) return; #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return; #endif #ifdef WITH_MQTT_BRIDGE if (_ota_update_at) return; + // The MQTT worker also owns its suspended station: Core 1 must not race + // the worker's shutdown or the next bounded probe on Core 0. if (bridge && (bridge->isRunning() || bridge->isStopping())) return; #endif #ifdef WITH_ESPNOW_BRIDGE @@ -4226,8 +4235,12 @@ void MyMesh::serviceIdleWiFi() { bool MyMesh::hasPendingWork() const { if (isDualRadioActive()) return true; if (hasPendingOtaApply()) return true; +#if defined(ESP32_PLATFORM) + if (mesh::wifi::networkPower().foregroundActive()) return true; +#endif #ifdef WITH_WEBCONFIG - if (isWebConfigActive()) return true; + if (isWebConfigActive() && (!mesh::wifi::networkPower().backgroundSuspended() + || _webconfig->isNetworkForegroundActive(millis()))) return true; #endif #if defined(ESP32_PLATFORM) \ && (defined(WITH_WEBCONFIG) || defined(WITH_MQTT_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) \ @@ -4243,7 +4256,11 @@ bool MyMesh::hasPendingWork() const { if (_local_cli_output.busy()) return true; #endif #if defined(WITH_BRIDGE) - if (bridge && bridge->isRunning()) return true; // bridge needs WiFi radio, can't sleep +#ifdef WITH_MQTT_BRIDGE + if (bridge && bridge->isRunning() && !bridge->isNetworkSuspended()) return true; +#else + if (bridge && bridge->isRunning()) return true; +#endif #ifdef WITH_MQTT_BRIDGE if (bridge && bridge->isStopping()) return true; #endif diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp index 6f0f16f36..54dc75b49 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.cpp +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.cpp @@ -471,7 +471,7 @@ esp_err_t PsychicMqttClient::reconnect() void PsychicMqttClient::disconnect() { - if (_client == nullptr) + if (_client == nullptr || !_started) { ESP_LOGW(TAG, "MQTT client not started."); return; @@ -481,16 +481,41 @@ void PsychicMqttClient::disconnect() { ESP_LOGI(TAG, "Disconnecting MQTT client."); _stopMqttClient = false; - esp_mqtt_client_disconnect(_client); - - // Wait for all disconnect events to be processed - while (!_stopMqttClient) + const esp_err_t disconnect_result = esp_mqtt_client_disconnect(_client); + if (disconnect_result == ESP_OK) { - vTaskDelay(10 / portTICK_PERIOD_MS); + // The SDK can lose its disconnect notification during a failed + // transport. Give callbacks one second, then use SDK stop to close + // the transport instead of waiting forever on that notification. + const uint32_t started_at = static_cast(millis()); + const uint32_t poll_ticks = 10 / portTICK_PERIOD_MS; + while (!_stopMqttClient + && static_cast(millis() - started_at) < 1000U) + { + vTaskDelay(poll_ticks ? poll_ticks : 1); + } + if (!_stopMqttClient) + { + ESP_LOGW(TAG, "Disconnect notification timed out; stopping MQTT client."); + } + } + else + { + ESP_LOGW(TAG, "MQTT disconnect request failed: %s; stopping client.", + esp_err_to_name(disconnect_result)); } } - esp_mqtt_client_stop(_client); + const esp_err_t stop_result = esp_mqtt_client_stop(_client); + if (stop_result != ESP_OK) + { + // A failed stop is not permission for the station owner to disable + // WiFi or reclaim transport buffers. isStarted() keeps that handoff + // blocked until an owning-task retry actually stops the SDK client. + ESP_LOGW(TAG, "MQTT client stop failed: %s", esp_err_to_name(stop_result)); + return; + } + _connected = false; _started = false; ESP_LOGI(TAG, "MQTT client stopped."); } diff --git a/lib/PsychicMqttClient/src/PsychicMqttClient.h b/lib/PsychicMqttClient/src/PsychicMqttClient.h index f5b29f31b..e1bced0fa 100644 --- a/lib/PsychicMqttClient/src/PsychicMqttClient.h +++ b/lib/PsychicMqttClient/src/PsychicMqttClient.h @@ -366,7 +366,8 @@ public: /** * @brief Disconnects the MQTT client from the server. - * This call might be blocking until the client is stopped cleanly + * Disconnect notification waits at most one second before SDK stop. + * A failed SDK stop leaves isStarted() true so callers retain ownership. */ void disconnect(); @@ -494,10 +495,10 @@ private: esp_mqtt_client_handle_t _client = nullptr; esp_mqtt_client_config_t _mqtt_cfg; esp_mqtt_error_codes_t _lastError; - bool _connected = false; - bool _stopMqttClient = false; + std::atomic _connected{false}; + std::atomic _stopMqttClient{false}; bool _config_dirty = true; - bool _started = false; + std::atomic _started{false}; // Runtime cap on the esp-mqtt outbox for QoS 0 async publishes (bytes). // 0 = disabled. Enforced in publish(); not an esp-mqtt config field. diff --git a/src/helpers/WiFiNetworkPowerPolicy.h b/src/helpers/WiFiNetworkPowerPolicy.h new file mode 100644 index 000000000..0a1538ced --- /dev/null +++ b/src/helpers/WiFiNetworkPowerPolicy.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include + +namespace mesh { +namespace wifi { + +static constexpr uint32_t kNetworkFailureGraceMs = 5UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkRecoveryProbeMs = 2UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkRecoveryIntervalMs = 30UL * 60UL * 1000UL; +static constexpr uint32_t kNetworkHealthyMs = 30000UL; + +// MQTT and its NTP bootstrap share one outage clock. A successful WiFi +// association or time reply alone is not proof that an MQTT destination works. +// Failed attempts must never extend either the grace period or the cooldown. +class NetworkPowerPolicy { +public: + enum class Action { Run, Suspend, Resume }; + + Action update(uint32_t now, bool useful_network, bool foreground, + bool requested_probe = false) { + if (!_started) { + _started = true; + _since = now; + } + if (useful_network) { + const bool was_suspended = _suspended; + _suspended = false; + _probing = false; + _since = now; + return was_suspended ? Action::Resume : Action::Run; + } + if (_suspended) { + if (foreground || requested_probe + || now - _since >= kNetworkRecoveryIntervalMs) { + _suspended = false; + _probing = true; + _since = now; + return Action::Resume; + } + return Action::Suspend; + } + if (foreground) return Action::Run; + const uint32_t budget = _probing + ? kNetworkRecoveryProbeMs : kNetworkFailureGraceMs; + if (now - _since >= budget) { + _suspended = true; + _probing = false; + _since = now; + return Action::Suspend; + } + return Action::Run; + } + + bool suspended() const { return _suspended; } + +private: + bool _started = false; + bool _suspended = false; + bool _probing = false; + uint32_t _since = 0; +}; + +enum class NetworkPowerHold : uint8_t { + CompanionClient = 1, + WebConfig = 2, + OtaSeeder = 4 +}; + +// One program-wide handoff between the MQTT owner task and the station/UI +// owners. Only the worker publishes suspension, after stopping TLS and SNTP. +// Foreground owners publish holds; they never stop the worker's transports. +class NetworkPowerControl { +public: + bool managed() const { return _managed.load(std::memory_order_acquire); } + void setManaged(bool value) { + _managed.store(value, std::memory_order_release); + if (!value) { + _suspended.store(false, std::memory_order_release); + _probe.store(false, std::memory_order_release); + } + } + bool backgroundSuspended() const { + return managed() && _suspended.load(std::memory_order_acquire); + } + void setSuspended(bool value) { + _suspended.store(value, std::memory_order_release); + } + void setHold(NetworkPowerHold owner, bool value) { + const uint8_t mask = static_cast(owner); + if (value) _holds.fetch_or(mask, std::memory_order_acq_rel); + else _holds.fetch_and(static_cast(~mask), std::memory_order_acq_rel); + } + bool foregroundActive() const { + return _holds.load(std::memory_order_acquire) != 0; + } + void requestProbe() { _probe.store(true, std::memory_order_release); } + bool consumeProbeRequest() { + return _probe.exchange(false, std::memory_order_acq_rel); + } + +private: + std::atomic _managed{false}; + std::atomic _suspended{false}; + std::atomic _probe{false}; + std::atomic _holds{0}; +}; + +inline NetworkPowerControl& networkPower() { + static NetworkPowerControl control; + return control; +} + +} // namespace wifi +} // namespace mesh diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 564d670e6..21754a061 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -89,6 +89,15 @@ static constexpr uint32_t MQTT_NTP_NON_ESP_SETTLE_MS = 5000UL; #endif static constexpr unsigned long MQTT_NTP_MIN_VALID_EPOCH = 1767225600UL; +static uint32_t mqttNtpFailureRetryAt(uint32_t now) { +#ifdef ESP_PLATFORM + const uint32_t deadline = now + mesh::wifi::kNetworkRecoveryIntervalMs; + return deadline == 0 ? 1 : deadline; +#else + return MQTTConnectionPolicy::ntpRetryAt(now); +#endif +} + static bool mqttNtpRefreshDue(uint32_t now, uint32_t last_sync, uint32_t retry_at) { return MQTTConnectionPolicy::ntpRefreshDue(now, last_sync, retry_at); @@ -1346,6 +1355,10 @@ void MQTTBridge::finishStopped() { // Timezone is inline class storage (_timezone_storage) - nothing to delete. // The shared JSON document's pools were freed by releaseRuntimeBuffers() above. _initialized = false; +#ifdef ESP_PLATFORM + _network_suspended.store(false, std::memory_order_release); + mesh::wifi::networkPower().setManaged(false); +#endif // A clean task stop already cancels this on its owning core. The second // cancellation is intentionally idempotent and also releases the lease // after the lifecycle's bounded forced-stop fallback. @@ -1449,6 +1462,16 @@ void MQTTBridge::LifecycleOps::onStopComplete(bool clean) { // --------------------------------------------------------------------------- // FreeRTOS task entry point // --------------------------------------------------------------------------- +void MQTTBridge::applyTimezoneRules() { + TimeChangeRule dst_rule, std_rule; + if (!timezoneRulesFromString(_obs->timezone_string, dst_rule, std_rule)) { + TimeChangeRule utc = {"UTC", Last, Sun, Mar, 0, 0}; + dst_rule = utc; + std_rule = utc; + } + _timezone_storage.setRules(dst_rule, std_rule); +} + #ifdef ESP_PLATFORM void MQTTBridge::mqttTask(void* parameter) { MQTTBridge* bridge = static_cast(parameter); @@ -1559,10 +1582,168 @@ bool MQTTBridge::waitUnlessStopping(uint32_t delay_ms) { return false; } +bool MQTTBridge::serviceNetworkPower(uint32_t now) { + auto& control = mesh::wifi::networkPower(); + bool useful_network = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + const uint32_t acknowledgments = _slots[i].client + ? _slots[i].client->getPublishAck() : 0; + const bool broker_replied = acknowledgments != 0 + && acknowledgments != _network_publish_acks[i]; + _network_publish_acks[i] = acknowledgments; + // A flapping CONNACK must not buy another full outage window forever. + // A PUBACK proves delivery immediately; a quiet connection must stay up + // for 30 seconds, leaving time to prove recovery within the probe window. + if (_slots[i].enabled && _slots[i].connected + && (broker_replied || (now - static_cast(_slots[i].connected_at_ms) + >= mesh::wifi::kNetworkHealthyMs))) { + useful_network = true; + } + } + wifi_mode_t mode = WIFI_MODE_NULL; + const bool ap_active = esp_wifi_get_mode(&mode) == ESP_OK && (mode & WIFI_MODE_AP); + bool foreground = control.foregroundActive() || ap_active + || (_board && _board->isOTAUpdateRunning()) + || _ntp_force_requested || _ntp_diag_requested; +#ifdef WITH_SNMP + foreground = foreground || (_snmp_agent && _snmp_agent->isRunning() + && WiFi.status() == WL_CONNECTED); +#endif + const auto action = _network_power_policy.update( + now, useful_network, foreground, control.consumeProbeRequest()); + if (action == mesh::wifi::NetworkPowerPolicy::Action::Suspend) { + const bool newly_suspended = !_network_suspended.load(std::memory_order_acquire); + if (newly_suspended) { + cancelNtpRefresh(); + _ntp_sync_pending = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + _slot_attempt_pending[i] = false; + if (_slots[i].client && _slots[i].client->isStarted()) { + _slots[i].client->disconnect(); + // The wrapper only clears isStarted after SDK stop succeeds. Never + // hand station ownership off while any transport still runs. + if (_slots[i].client->isStarted()) return false; + } + _slots[i].connected = false; + _slots[i].connected_at_ms = 0; + _status_publish_pending[i] = false; + } + updateCachedConnectionStatus(); + _network_suspended.store(true, std::memory_order_release); + control.setSuspended(true); // TLS/SNTP teardown precedes the handoff. + MQTT_DEBUG_PRINTLN("Network unavailable; suspending background WiFi for 30 minutes"); + } + if (_manage_wifi && !control.foregroundActive() + && !(_board && _board->isOTAUpdateRunning()) + && mqttStationWiFiMutationAllowed() + && (newly_suspended || (!mesh::wifi::espNowChannelConstrained() + && WiFi.getMode() != WIFI_OFF))) { + WiFi.setAutoReconnect(false); + WiFi.disconnect(false, false); + if (mesh::wifi::espNowChannelConstrained()) { + // ESP-NOW is an independent radio owner. Stop failed STA attempts, + // keeping its driver, protocol and fixed channel available. + _network_espnow_restore_pending = + mesh::wifi::restoreEspNowChannel() != ESP_OK; + _network_espnow_restore_at = static_cast(millis()) + 1000U; + } else if (WiFi.getMode() != WIFI_OFF) { + WiFi.mode(WIFI_OFF); + } + } + serviceEspNowChannelRestore(static_cast(millis())); + return false; + } + _network_espnow_restore_pending = false; + if (_network_suspended.exchange(false, std::memory_order_acq_rel)) { + // Publish before restarting: the Companion/role loop must not reclaim + // WiFi underneath a new association or TLS handshake. + control.setSuspended(false); + _last_wifi_check = now - 10001UL; + _wifi_status_initialized = false; + _last_slot_reconnect_ms = now - MQTTConnectionPolicy::kReconnectGuardMs; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + _slots[i].circuit_breaker_tripped = false; + _slots[i].last_reconnect_attempt = now - MQTTConnectionPolicy::reconnectDelayMs( + _slots[i].reconnect_backoff, static_cast(i)); + } + _ntp_refresh_retry_at = 0; + _ntp_sync_pending = !_fresh_ntp_this_boot.load(std::memory_order_acquire); + if (_manage_wifi) initializeWiFiInTask(); + MQTT_DEBUG_PRINTLN("Starting bounded network recovery probe"); + } + return true; +} + +void MQTTBridge::serviceEspNowChannelRestore(uint32_t now) { + if (!_network_espnow_restore_pending) return; + if (!_manage_wifi || !_network_suspended.load(std::memory_order_acquire) + || !mesh::wifi::espNowChannelConstrained()) { + _network_espnow_restore_pending = false; + return; + } + // Disconnect and scan completion are asynchronous. Retry only a failed + // channel write, once per second, without another station disconnect or + // driver mode change. Foreground owners retain their channel throughout. + if (mesh::wifi::networkPower().foregroundActive() + || (_board && _board->isOTAUpdateRunning()) + || !mqttStationWiFiMutationAllowed() + || static_cast(now - _network_espnow_restore_at) < 0) return; + _network_espnow_restore_pending = mesh::wifi::restoreEspNowChannel() != ESP_OK; + _network_espnow_restore_at = static_cast(millis()) + 1000U; +} + +void MQTTBridge::serviceBackgroundNtp(uint32_t now) { + if (WiFi.status() != WL_CONNECTED) { + pollNtpRefresh(now); + return; + } + // A retained/admin-set clock can start MQTT without a blocking multi-server + // NTP walk. It is not credited as a fresh internet reply or useful network. + if (!_ntp_synced.load(std::memory_order_acquire)) { + uint32_t system_time = 0; + MQTTConnectionPolicy::checkedRtcEpoch(static_cast(time(nullptr)), + MQTT_NTP_MIN_VALID_EPOCH, system_time); + const uint32_t rtc_time = _rtc ? _rtc->getCurrentTime() : 0; + const auto source = MQTTConnectionPolicy::chooseFallbackClock( + false, system_time, rtc_time, MQTT_NTP_MIN_VALID_EPOCH); + if (source != MQTTConnectionPolicy::ClockSource::None) { + const uint32_t epoch = source == MQTTConnectionPolicy::ClockSource::Rtc + ? rtc_time : system_time; + struct timeval accepted = {static_cast(epoch), 0}; + settimeofday(&accepted, nullptr); + if (_rtc) { + _rtc->setCurrentTime(epoch); + if (epoch < rtc_time) { + _backward_clock_reset_epoch.store(epoch, std::memory_order_release); + } + } + _ntp_synced.store(true, std::memory_order_release); + } + } + pollNtpRefresh(now); + if (_ntp_refresh_pending) return; + const bool bootstrap = _ntp_sync_pending + || !_fresh_ntp_this_boot.load(std::memory_order_acquire); + const bool retry_due = _ntp_refresh_retry_at == 0 + || mqttNtpRefreshDue(now, _last_ntp_sync, _ntp_refresh_retry_at); + if ((bootstrap && retry_due) + || mqttNtpRefreshDue(now, _last_ntp_sync, _ntp_refresh_retry_at)) { + _ntp_sync_pending = false; + refreshNTP(); + } +} + // --------------------------------------------------------------------------- // mqttTaskLoop() - main loop running on Core 0 // --------------------------------------------------------------------------- void MQTTBridge::mqttTaskLoop() { + _network_power_policy = mesh::wifi::NetworkPowerPolicy(); + _network_suspended.store(false, std::memory_order_release); + _network_espnow_restore_pending = false; + mesh::wifi::networkPower().setManaged(true); + // Timezone configuration is independent of whether an NTP server answers. + // The old blocking bootstrap applied these rules as a side effect. + applyTimezoneRules(); // An active OTA/setup AP can outlive MQTT's start request. Stay stoppable // while waiting, and leave slot/TLS work until station initialization ran. while (!_stop_requested && !initializeWiFiInTask()) { @@ -1586,10 +1767,21 @@ void MQTTBridge::mqttTaskLoop() { if (_stop_requested) { MQTT_DEBUG_PRINTLN("MQTT task: cooperative stop - tearing down clients on Core 0"); cancelNtpRefresh(); + bool transports_stopped = true; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { teardownSlot(i); + if (_slots[i].client && _slots[i].client->isStarted()) transports_stopped = false; + } + if (!transports_stopped) { + // Retain clients, tokens and the stop acknowledgment until SDK stop + // actually succeeds. end() still owns the explicitly dirty timeout + // fallback; this worker must never label a failed stop as clean. + vTaskDelay(pdMS_TO_TICKS(100)); + continue; } destroySlotClients(); + _network_suspended.store(false, std::memory_order_release); + mesh::wifi::networkPower().setManaged(false); _stop_acked.store(true, std::memory_order_release); // teardown precedes Core 1 release return; } @@ -1613,6 +1805,15 @@ void MQTTBridge::mqttTaskLoop() { unsigned long now = millis(); + // The MQTT and NTP schedulers share this single WiFi outage budget. + // Quiesce on the owner core before an external Companion station owner + // sees the suspension flag; failed timers cannot wake each other up. + if (!serviceNetworkPower(static_cast(now))) { + processPacketQueue(); + vTaskDelay(pdMS_TO_TICKS(250)); + continue; + } + // Periodic heap + publish-health snapshot. Gated behind MQTT_MEMORY_DEBUG (a // dedicated diagnostics flag, NOT enabled on production or plain MQTT_DEBUG builds) // so it stays off by default -- the same data is available on demand via the @@ -1644,21 +1845,7 @@ void MQTTBridge::mqttTaskLoop() { } } - // Check for pending NTP sync (triggered from WiFi event handler) - if (_ntp_sync_pending && WiFi.status() == WL_CONNECTED) { - _ntp_sync_pending = false; - syncTimeWithNTP(); - } - - // Retry NTP every 30s if initial sync failed (slots can't start without valid time) - if (!_ntp_synced.load(std::memory_order_acquire) - && WiFi.status() == WL_CONNECTED) { - static unsigned long last_ntp_retry = 0; - if (now - last_ntp_retry >= 30000) { - last_ntp_retry = now; - syncTimeWithNTP(); - } - } + serviceBackgroundNtp(static_cast(now)); // Process a CLI-requested forced NTP sync (queued from Core 1). Running it here // keeps all NTP I/O on Core 0; requestForcedNtpSync() blocks the CLI thread until @@ -1793,16 +1980,8 @@ void MQTTBridge::mqttTaskLoop() { // Periodic configuration check (throttled to avoid spam) checkConfigurationMismatch(); - // Periodic NTP refresh (once per day) - lightweight, non-blocking. - // Uses async SNTP instead of the heavy syncTimeWithNTP() which blocks Core 0 - // for up to 20+ seconds with DNS lookups, UDP sockets, and retry loops. - pollNtpRefresh(now); - if (WiFi.status() == WL_CONNECTED - && !_ntp_refresh_pending - && mqttNtpRefreshDue(now, _last_ntp_sync, - _ntp_refresh_retry_at)) { - refreshNTP(); - } + // Automatic boot/reconnect/daily NTP is serviced above as one bounded + // asynchronous operation. Explicit administrator diagnostics remain separate. // Publish status updates (handle millis() overflow correctly). // Read the toggle live from prefs (like mqtt.packets/rx/tx below) so a @@ -2142,10 +2321,14 @@ bool MQTTBridge::setupSlot(int index) { // transport handles are reused; setCredentials / setServer overwrite the config // fields in place before connect() restarts the ESP-IDF client. if (slot.initial_connect_done) { - if (slot.client->connected()) { + if (slot.client->isStarted()) { _slot_attempt_pending[index] = false; slot.client->disconnect(); } + if (slot.client->isStarted()) { + slot.last_reconnect_attempt = millis(); + return false; + } // Clear TLS verification fields so a stale CA-bundle attach or cert // pointer from a prior preset doesn't override the new one. esp_mqtt_client_config_t* cfg = slot.client->getMqttConfig(); @@ -2375,6 +2558,7 @@ void MQTTBridge::teardownSlot(int index, bool force) { delay(50); #endif } + if (slot.client && !force && slot.client->isStarted()) return; // Invalidate the token but keep the buffer: the client survives teardown and still // holds this pointer in its config (see MQTTSlot::auth_token). @@ -3062,6 +3246,7 @@ void MQTTBridge::setSlotPreset(int slot_index, const char* preset_name) { #ifdef ESP_PLATFORM if (_mqtt_task_handle != nullptr) { _slot_reconfigure_pending[slot_index] = true; + mesh::wifi::networkPower().requestProbe(); MQTT_DEBUG_PRINTLN("MQTT%d reconfigure queued (preset: %s)", slot_index + 1, preset_name); return; } @@ -3077,6 +3262,12 @@ void MQTTBridge::applySlotPreset(int slot_index, const char* preset_name) { MQTTSlot& slot = _slots[slot_index]; teardownSlot(slot_index); + if (slot.client && slot.client->isStarted()) { + // Keep the requested preset queued and the running SDK's current config + // untouched until its owning task can complete the failed stop. + _slot_reconfigure_pending[slot_index] = true; + return; + } if (strcmp(preset_name, MQTT_PRESET_NONE) == 0 || preset_name[0] == '\0') { slot.enabled = false; @@ -4460,7 +4651,21 @@ void MQTTBridge::refreshNTP() { #endif _ntp_refresh_previous_epoch = _rtc ? (unsigned long)_rtc->getCurrentTime() : 0; +#ifdef ESP_PLATFORM + const char* servers[kMaxNtpServers]; + int server_count = 0; + fillNtpServerList(_obs, servers, server_count); + const int first = _background_ntp_server % server_count; + // Let SNTP try three servers within one 15-second request, rotating through + // the configured list across probes instead of blocking on every server. + configTime(0, 0, servers[first], servers[(first + 1) % server_count], + servers[(first + 2) % server_count]); + // A blackholed leading server can consume the whole SDK receive timeout. + // Advance by one so every server eventually leads, even with six entries. + _background_ntp_server = static_cast((first + 1) % server_count); +#else configTime(0, 0, effectiveNtpPrimary(_obs)); +#endif _ntp_refresh_retry_at = 0; _ntp_refresh_started_at = millis(); _ntp_refresh_pending = true; @@ -4473,7 +4678,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { const uint32_t elapsed = now - _ntp_refresh_started_at; if (WiFi.status() != WL_CONNECTED) { cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN("NTP refresh aborted; WiFi disconnected"); return; } @@ -4514,7 +4719,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { // The ESP status is edge-like and may reset after it is read, so retry the // whole operation instead of waiting on a completion that was consumed. cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN( "NTP refresh returned implausible time %lld; retry scheduled", (long long)raw_system_time); @@ -4523,7 +4728,7 @@ void MQTTBridge::pollNtpRefresh(uint32_t now) { if (elapsed >= MQTT_NTP_REFRESH_TIMEOUT_MS) { cancelNtpRefresh(); - _ntp_refresh_retry_at = mqttNtpRetryAt(now); + _ntp_refresh_retry_at = mqttNtpFailureRetryAt(now); MQTT_DEBUG_PRINTLN("NTP refresh timed out; retry scheduled"); } } @@ -4845,6 +5050,7 @@ void MQTTBridge::runNtpEstimateProbe() { bool MQTTBridge::requestNtpTimeEstimate() { if (!isRunning() || _stop_requested.load(std::memory_order_acquire) + || isNetworkSuspended() || WiFi.status() != WL_CONNECTED || _ntp_estimate_requested) return false; _ntp_estimate_done = false; _ntp_estimate_ok = false; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 783d69cab..daa672239 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -25,6 +25,7 @@ class MeshSNMPAgent; // Forward declaration #ifdef ESP_PLATFORM #include "helpers/esp32/SntpOperationCoordinator.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #include #include #include @@ -447,6 +448,17 @@ private: unsigned long _last_wifi_reconnect_attempt; uint8_t _wifi_reconnect_backoff_attempt; // 0..5 -> 15s, 30s, 60s, 120s, 300s; reset on connect unsigned long _last_slot_reconnect_ms; // guards against concurrent TLS handshakes (15 s inter-slot gap) +#ifdef ESP_PLATFORM + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + uint8_t _background_ntp_server = 0; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void serviceBackgroundNtp(uint32_t now); +#endif AlertFaultPolicy::OutageSnapshot wifiOutage() const { return AlertFaultPolicy::unpackOutageSnapshot( @@ -550,6 +562,7 @@ private: void getClientVersion(char* buffer, size_t buffer_size) const; void logMemoryStatus(); void refreshOriginFromPrefs(); + void applyTimezoneRules(); // begin()/end()-scoped PSRAM buffers. Each allocation is independent so a // transient heap shortage degrades to the existing stack fallback instead // of making the bridge unusable. @@ -606,6 +619,15 @@ public: void requestStop(); bool isStopping() const { return _lifecycle.isStopInProgress(); } bool isRunning() const override { return _initialized; } + // Configured and still scheduling recovery, but transports are quiesced. + // Do not conflate automatic suspension with a persisted bridge disable. + bool isNetworkSuspended() const { +#ifdef ESP_PLATFORM + return _network_suspended.load(std::memory_order_acquire); +#else + return false; +#endif + } void loop() override; void onPacketReceived(mesh::Packet *packet) override; void sendPacket(mesh::Packet *packet) override; diff --git a/src/helpers/esp32/WebConfigServer.cpp b/src/helpers/esp32/WebConfigServer.cpp index 686c12f5d..694b0c865 100644 --- a/src/helpers/esp32/WebConfigServer.cpp +++ b/src/helpers/esp32/WebConfigServer.cpp @@ -368,6 +368,7 @@ WebConfigServer::WebConfigServer(Callbacks* callbacks, void* mqtt_prefs, bool ow WebConfigServer::~WebConfigServer() { detachRoutes(); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); closeTerminal(); delete _room_mailbox; if (_mux) vSemaphoreDelete(_mux); @@ -653,6 +654,14 @@ bool WebConfigServer::formatWiFiStatus( appendOtaSeederStatus(reply, reply_len); return true; } + if (mesh::wifi::networkPower().backgroundSuspended() + && !mesh::wifi::networkPower().foregroundActive()) { + snprintf(reply, reply_len, + "> suspended; recovery every 30 minutes, SSID: %s", + configured ? ssid : "(not configured)"); + appendOtaSeederStatus(reply, reply_len); + return true; + } if (active && active->_mode == MODE_CONNECTING) { snprintf(reply, reply_len, "> connecting, SSID: %s", configured ? ssid : "(not configured)"); @@ -754,6 +763,35 @@ bool WebConfigServer::getSetupInfo(char* ssid, size_t ssid_len, char* ip, size_t return true; } +void WebConfigServer::noteNetworkActivity() { + // This is separate from _last_activity, which also tracks boot and captive + // portal events. Those events must not renew an idle LAN listener's lease. + _network_activity_until = WebConfigBatch::scheduleAt(millis(), 120000UL); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); +} + +void WebConfigServer::requestNetworkAccess() { + noteNetworkActivity(); + mesh::wifi::networkPower().requestProbe(); +} + +bool WebConfigServer::isNetworkForegroundActive(uint32_t now) const { + if (_mode == MODE_OFF && !_stopping) return false; + if (_mode == MODE_SETUP || _stopping || handlerRefCount() != 0) return true; + WCLock lock(_mux); + if (_batch_state == BATCH_PENDING || _setup_wifi_handoff_pending) return true; + if (_terminal && (_terminal->running || _terminal->queue.pending)) return true; + if (_room_mailbox && (_room_mailbox->state() == mesh::RoomWebMailbox::State::Pending + || _room_mailbox->state() == mesh::RoomWebMailbox::State::Running)) return true; + return _network_activity_until + && !WebConfigBatch::deadlineReached(now, _network_activity_until); +} + +void WebConfigServer::refreshNetworkPowerHold(uint32_t now) { + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, + isNetworkForegroundActive(now)); +} + // --------------------------------------------------------------------------- // Lifecycle // --------------------------------------------------------------------------- @@ -817,6 +855,9 @@ bool WebConfigServer::startSetupMode(char reply[]) { strcpy(reply, "Err: webconfig busy"); return false; } + // Publish the foreground owner before changing interfaces. The background + // worker must not power down the station underneath this AP startup. + requestNetworkAccess(); _retry_saved_wifi_in_setup = false; _setup_reconnect_in_progress = false; _setup_reconnect_deadline = 0; @@ -888,6 +929,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { } if (!ap_ok) { if (!promote_lan) stopOwnedWiFiRadio(); + refreshNetworkPowerHold(millis()); strcpy(reply, "Err: failed to start AP"); return false; } @@ -905,6 +947,7 @@ bool WebConfigServer::startSetupMode(char reply[]) { _initial_setup = _wifi_ssid[0] == 0 && node.admin_password[0] != 0; _setup_started_at = millis(); _last_activity = _setup_started_at; + refreshNetworkPowerHold(_setup_started_at); // Keep the pre-AP scan results for the picker. Only an explicit browser // rescan needs to leave the AP's home channel after clients have joined. @@ -942,6 +985,8 @@ bool WebConfigServer::startLanMode(char reply[]) { } bool WebConfigServer::beginSavedStation(uint32_t now) { + if (mesh::wifi::networkPower().backgroundSuspended() + && !isNetworkForegroundActive(now)) return false; WiFi.mode(WIFI_STA); if (mesh::wifi::applyProtocolMask(WIFI_IF_STA) != ESP_OK) { return false; @@ -1020,6 +1065,7 @@ void WebConfigServer::requestStop() { _stopping = true; _stop_warn_at = WebConfigBatch::scheduleAt(millis(), STOP_WARN_MS); _stop_warned = false; + refreshNetworkPowerHold(millis()); } bool WebConfigServer::stopForOTA(char reply[]) { @@ -1095,6 +1141,8 @@ void WebConfigServer::finalizeTeardown() { _stats_json[0] = 0; _board_cmds = 0; _board_cmds_probed = false; + _network_activity_until = 0; + refreshNetworkPowerHold(millis()); if (_cb) _cb->onWebConfigStopped(); } @@ -1113,6 +1161,7 @@ void WebConfigServer::probeBoardCommands() { } void WebConfigServer::tick(uint32_t now) { + refreshNetworkPowerHold(now); if (_stopping) { uint32_t refs = handlerRefCount(); switch (WebConfigBatch::stopStep(refs, _stop_warned, _stop_warn_at, now)) { @@ -1137,6 +1186,12 @@ void WebConfigServer::tick(uint32_t now) { serviceTerminal(now); serviceRoom(now); + // The MQTT/NTP owner has stopped its transports before publishing this + // state. An idle portal must not start an independent reconnect loop while + // those consumers are in their shared thirty-minute recovery cooldown. + if (mesh::wifi::networkPower().backgroundSuspended() + && !isNetworkForegroundActive(now)) return; + // A primary ESP-NOW radio and its WiFi station cannot remain on different // channels. Reject a router-driven channel move (for example a CSA) and put // the mesh radio back on its boot channel before any reconnect work runs. @@ -1614,6 +1669,8 @@ void WebConfigServer::dispatchRequest(AsyncWebServerRequest* req, return; } + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + (target->*handler)(req); portENTER_CRITICAL(&s_wc_route_mux); @@ -1729,6 +1786,7 @@ static bool roomRequestIdentity(AsyncWebServerRequest* req, uint8_t (&token)[32] void WebConfigServer::handleRoomPage(AsyncWebServerRequest* req) { if (_mode == MODE_OFF || !_cb || !_cb->supportsRoomService()) { req->send(404); return; } _last_activity = millis(); + noteNetworkActivity(); auto* response = new (std::nothrow) WebConfigPacedProgmemResponse( "text/html; charset=utf-8", reinterpret_cast(ROOM_WEB_HTML), sizeof(ROOM_WEB_HTML) - 1); if (!response) { req->send(503); return; } @@ -1765,6 +1823,7 @@ void WebConfigServer::handleRoomPost(AsyncWebServerRequest* req) { } if (result == mesh::RoomWebMailbox::Result::Accepted) { _last_activity = millis(); + noteNetworkActivity(); req->send(202, "application/json", "{\"accepted\":true}"); } else if (result == mesh::RoomWebMailbox::Result::Busy) { req->send(429, "application/json", "{\"error\":\"room service busy\"}"); @@ -1814,7 +1873,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { // saved WiFi network must not grant that same password-reset privilege. if (!req || !req->client() || req->client()->localIP() != WiFi.softAPIP()) return false; - if (_initial_setup || _wifi_ssid[0] == 0) return true; + if (_initial_setup || _wifi_ssid[0] == 0) { noteNetworkActivity(); return true; } } else if (_mode != MODE_LAN) { return false; } @@ -1823,7 +1882,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { WCLock lock(_mux); _cb->getNodeSnapshot(node); } - if (node.admin_password[0] == 0) return true; + if (node.admin_password[0] == 0) { noteNetworkActivity(); return true; } if (_session_token[0] == 0) return false; if (!req->hasHeader("Cookie")) return false; const String& cookies = req->getHeader("Cookie")->value(); @@ -1834,6 +1893,7 @@ bool WebConfigServer::checkAuth(AsyncWebServerRequest* req) { if ((uint32_t)(now - _session_last_seen) > WEBCONFIG_SESSION_TTL_MS) return false; if (!fixedTimeEquals(token.c_str(), _session_token, 32)) return false; _session_last_seen = now; // sliding expiry + noteNetworkActivity(); return true; } @@ -1857,6 +1917,7 @@ static void webConfigRejectEncoding(AsyncWebServerRequest* req) { void WebConfigServer::handleRoot(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); if (!webConfigClientAcceptsGzip(req)) { webConfigRejectEncoding(req); return; } if (req->hasHeader("If-None-Match") && req->getHeader("If-None-Match")->value() == WEBCONFIG_HTML_ETAG) { @@ -1876,6 +1937,7 @@ void WebConfigServer::handleRoot(AsyncWebServerRequest* req) { void WebConfigServer::handleUi(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); if (!req->hasParam("v") || req->getParam("v")->value() != WEBCONFIG_HTML_VERSION) { // Part URLs are immutable. Never serve a new layout under an old loader's @@ -1988,6 +2050,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { return; } if (_initial_setup || _wifi_ssid[0] == 0) { + noteNetworkActivity(); req->send(200, "application/json", "{\"ok\":true}"); return; } @@ -1998,6 +2061,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { _cb->getNodeSnapshot(node); } if (node.admin_password[0] == 0) { + noteNetworkActivity(); req->send(200, "application/json", "{\"ok\":true}"); return; } @@ -2026,6 +2090,7 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) { _login_lock_until = 0; for (int i = 0; i < 4; i++) sprintf(&_session_token[i * 8], "%08lx", (unsigned long)esp_random()); _session_last_seen = now; + noteNetworkActivity(); AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}"); char cookie[80]; @@ -2951,6 +3016,7 @@ void WebConfigServer::handleScan(AsyncWebServerRequest* req) { void WebConfigServer::handlePresets(AsyncWebServerRequest* req) { if (_mode == MODE_OFF) { req->send(503); return; } _last_activity = millis(); + noteNetworkActivity(); #ifdef WITH_MQTT_BRIDGE if (_mqtt_prefs == NULL) { req->send(200, "application/json", "{\"presets\":[]}"); diff --git a/src/helpers/esp32/WebConfigServer.h b/src/helpers/esp32/WebConfigServer.h index aa67aa0f5..4974c7439 100644 --- a/src/helpers/esp32/WebConfigServer.h +++ b/src/helpers/esp32/WebConfigServer.h @@ -36,6 +36,7 @@ #include #include #include +#include #include class AsyncWebServer; @@ -252,6 +253,12 @@ public: bool stopForOTA(char reply[]); void tick(uint32_t now); // call every loop iteration + // A listening LAN portal is not itself a reason to keep a failed background + // network awake. Explicit starts and real browser activity get a bounded + // window; AP setup and in-flight work retain the radio until they complete. + void requestNetworkAccess(); + bool isNetworkForegroundActive(uint32_t now) const; + Mode mode() const { return _mode; } bool isRunning() const { return _mode != MODE_OFF; } bool isStopping() const { return _stopping; } @@ -367,6 +374,7 @@ private: uint32_t _diag_last = 0; volatile uint32_t _last_activity = 0; + volatile uint32_t _network_activity_until = 0; uint32_t _reboot_at = 0; // 0 = none scheduled uint32_t _stop_warn_at = 0; bool _stop_warned = false; @@ -378,6 +386,8 @@ private: bool _board_cmds_probed = false; void createServer(); + void noteNetworkActivity(); + void refreshNetworkPowerHold(uint32_t now); bool beginSavedStation(uint32_t now); void registerRoutes(); typedef void (WebConfigServer::*RequestHandler)(AsyncWebServerRequest*); diff --git a/src/helpers/esp32/WiFiOtaSeeder.cpp b/src/helpers/esp32/WiFiOtaSeeder.cpp index 1924e1b09..c4b18eef6 100644 --- a/src/helpers/esp32/WiFiOtaSeeder.cpp +++ b/src/helpers/esp32/WiFiOtaSeeder.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,7 @@ void detachTcpFolder(bool disconnected) { void stopClient(bool disconnected = false) { detachTcpFolder(disconnected); if (seeder_client) seeder_client.stop(); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); } } // namespace @@ -89,6 +91,7 @@ void WiFiOtaSeeder::loop() { // A different host transport may already own the replacement context. if (active && !active->folder_active) active->clear_folder_dest(); tcp_folder_attached = false; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); } if (seeder_client && seeder_client.connected()) return; @@ -125,6 +128,7 @@ void WiFiOtaSeeder::loop() { } tcp_folder_attached = true; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, true); char link_info[24]; snprintf(link_info, sizeof(link_info), "tcp %s", seeder_client.remoteIP().toString().c_str()); diff --git a/test/fixtures/mqtt_ap_ownership.cpp b/test/fixtures/mqtt_ap_ownership.cpp index bb4aed59b..5f0a40020 100644 --- a/test/fixtures/mqtt_ap_ownership.cpp +++ b/test/fixtures/mqtt_ap_ownership.cpp @@ -7,6 +7,7 @@ #include "helpers/AlertFaultPolicy.h" #include "helpers/MQTTConnectionPolicy.h" #include "helpers/WiFiPowerSave.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #define MQTT_DEBUG_PRINTLN(...) ((void)0) #define pdMS_TO_TICKS(value) (value) using TickType_t=uint32_t; @@ -69,6 +70,9 @@ namespace mesh{namespace wifi{ int s_wifi_disconnect_reason=0;uint32_t s_wifi_disconnect_time=0; unsigned long s_wifi_connected_at=0; struct MQTTBridge{ + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic_network_suspended{false}; + bool _network_espnow_restore_pending=false; bool _wifi_event_registered=false,_manage_wifi=true,_ntp_sync_pending=false; int _wifi_reconnect_backoff_attempt=0; std::atomic_ntp_synced{true},_stop_requested{false},_stop_acked{false}; @@ -78,13 +82,14 @@ struct MQTTBridge{ void setWifiOutage(AlertFaultPolicy::OutageSnapshot value){outage=value;} char _wifi_ssid[8]="mesh",_wifi_password[8]="secret"; void beginWiFiStation(); + void applyTimezoneRules() {} bool _wifi_status_initialized=true; int _last_wifi_status=WL_DISCONNECTED; unsigned long _last_wifi_check=0,_last_wifi_reconnect_attempt=0; uint8_t _wifi_power_save=1; struct{bool canonical_wifi=true;}_node_info; struct{uint8_t wifi_power_save=1;}prefs,*_obs=&prefs; - struct Client{void disconnect(){}}; + struct Client{void disconnect(){}bool isStarted()const{return false;}}; struct Slot{Client* client=nullptr;bool connected=false;}_slots[RUNTIME_MQTT_SLOTS]; bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS]={}; std::vector cleanup; @@ -96,8 +101,11 @@ struct MQTTBridge{ bool handleWiFiConnection(unsigned long now); void mqttTaskLoop(){ @TASK_START@ + while(true){ @TASK_STOP@ - ++slot_work; + ++slot_work; + break; + } } }; @INITIALIZE@ diff --git a/test/fixtures/rs232_mqtt_runtime.cpp b/test/fixtures/rs232_mqtt_runtime.cpp index af380185b..2904ec461 100644 --- a/test/fixtures/rs232_mqtt_runtime.cpp +++ b/test/fixtures/rs232_mqtt_runtime.cpp @@ -113,6 +113,7 @@ struct MQTTNodeInfo { class MQTTBridge : public AbstractBridge { bool running = false; bool stopping = false; + bool network_suspended = false; public: inline static unsigned live = 0, fail_starts = 0; unsigned begins = 0, ends = 0, loops = 0, sent = 0, received = 0; @@ -127,12 +128,21 @@ public: assert(filters); ++begins; if (fail_starts) { --fail_starts; return; } running = true; + network_suspended = false; } - void end() override { ++ends; running = stopping = false; } + void end() override { ++ends; running = stopping = network_suspended = false; } void requestStop() { stopping = running; } bool isStopping() const { return stopping; } bool isRunning() const override { return running; } - void loop() override { ++loops; if (stopping) { ++ends; running = stopping = false; } } + bool isNetworkSuspended() const { return network_suspended; } + void setNetworkSuspended(bool paused) { + assert(running && !stopping); + network_suspended = paused; + } + void loop() override { + ++loops; + if (stopping) { ++ends; running = stopping = network_suspended = false; } + } void sendPacket(mesh::Packet* p) override { if (allow(p)) ++sent; } void onPacketReceived(mesh::Packet* p) { if (allow(p)) ++received; } }; @@ -270,6 +280,29 @@ int main() { expect_get(cli, "bridge.type", "> mqtt+rs232+espnow"); expect_get(cli, "rs232.enabled", "> off"); expect_get(cli, "rs232.running", "> off"); + fresh.mqtt_bridge->setNetworkSuspended(true); + assert(fresh.isMqttBridgeRunning() && fresh._prefs.bridge_enabled); + assert(fresh.bridgesPreventSleep()); // ESP-NOW retains its radio owner. + assert(fresh.setEspNowBridgeState(false)); + assert(!fresh.bridgesPreventSleep()); // Paused MQTT is still configured. + assert(fresh.setRs232BridgeState(true)); + assert(fresh.bridgesPreventSleep()); // An active UART remains independent. + assert(fresh.setRs232BridgeState(false)); + assert(!fresh.bridgesPreventSleep()); + fresh.mqtt_bridge->setNetworkSuspended(false); + assert(fresh.bridgesPreventSleep()); // The recovery probe owns the radio. + char pause_reply[256] = {}; + cli.set("espnow.enabled off", pause_reply); + assert(strcmp(pause_reply, "OK") == 0); + fresh.mqtt_bridge->setNetworkSuspended(true); + assert(fresh.requestMqttBridgeStop()); + assert(fresh.isMqttBridgeStopping() && fresh.bridgesPreventSleep()); + fresh.serviceBridges(); + assert(!fresh.isMqttBridgeRunning() && !fresh.bridgesPreventSleep()); + assert(fresh.setBridgeState(true)); + assert(fresh.isMqttBridgeRunning() && fresh.bridgesPreventSleep()); + cli.set("espnow.enabled on", pause_reply); + assert(strcmp(pause_reply, "OK") == 0 && fresh.isEspNowBridgeRunning()); } MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh); char reply[256] = {}; diff --git a/test/fixtures/webconfig_ap_start.cpp b/test/fixtures/webconfig_ap_start.cpp index 9ab27d962..65f3a294e 100644 --- a/test/fixtures/webconfig_ap_start.cpp +++ b/test/fixtures/webconfig_ap_start.cpp @@ -337,6 +337,8 @@ struct WebConfigServer { bool auth_allowed = true; ~WebConfigServer() { delete _dns; } void createServer() { ++servers; } + void requestNetworkAccess() {} + void refreshNetworkPowerHold(uint32_t) {} bool startSetupMode(char reply[]); bool checkAuth(AsyncWebServerRequest*) const { return auth_allowed; } void handleScan(AsyncWebServerRequest* req); diff --git a/test/fixtures/webconfig_ota_handoff.cpp b/test/fixtures/webconfig_ota_handoff.cpp index 25929c8cd..1d9bdd0f1 100644 --- a/test/fixtures/webconfig_ota_handoff.cpp +++ b/test/fixtures/webconfig_ota_handoff.cpp @@ -5,11 +5,14 @@ #include #include #include +#include static uint32_t millis() { return 123; } struct WebConfigBatch { static uint32_t scheduleAt(uint32_t now, uint32_t delay) { return now + delay; } + static bool deadlineReached(uint32_t now, uint32_t at) { return int32_t(now - at) >= 0; } }; +struct WCLock { explicit WCLock(void*) {} }; constexpr int WIFI_STA = 1; constexpr int WIFI_AP = 2; struct WiFiFake { @@ -29,7 +32,7 @@ struct DNS { void stop() {} }; struct Callbacks { unsigned stops = 0; void onWebConfigStopped() { ++stops; } }; struct WebConfigServer { enum Mode { MODE_OFF, MODE_LAN, MODE_SETUP }; - enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_CONFIG = 0 }; + enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_PENDING = 1, BATCH_CONFIG = 0 }; Server server; Server* _server = &server; DNS* _dns = nullptr; @@ -46,11 +49,18 @@ struct WebConfigServer { uint32_t _setup_reconnect_deadline = 0, _reboot_at = 0, _batch_state = 0; uint32_t _batch_next = 0, _setup_wifi_handoff_deadline = 0, _batch_kind = 0; uint32_t _board_cmds = 0; + uint32_t _network_activity_until = 0; + void* _mux = nullptr; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; char _wifi_ssid[32] = {}, _setup_wifi_handoff_ip[16] = {}; char _session_token[16] = {}, _stats_json[16] = {}; bool routes = true; void detachRoutes() { routes = false; } void closeTerminal() {} + uint32_t handlerRefCount() const { return 0; } + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); bool isRunning() const { return _mode != MODE_OFF; } bool isStopping() const { return _stopping; } void requestStop(); @@ -75,6 +85,7 @@ int main() { assert(portal.stopForOTA(reply)); assert(strcmp(reply, "WebConfig stopped") == 0); assert(!portal.routes && !portal.server.listening && portal.isStopping()); + assert(mesh::wifi::networkPower().foregroundActive()); assert(WiFi.sta && !WiFi.ap && WiFi.stops == 0); assert(WiFi.connected == !setup); // OTA can now bind port 80 and create its own AP, while HTTP request @@ -86,6 +97,7 @@ int main() { assert(WiFi.ap && WiFi.sta && WiFi.stops == 0); assert(WiFi.ap_stops == unsigned(setup)); assert(portal.callbacks.stops == 1 && !portal._keep_wifi_on_stop); + assert(!mesh::wifi::networkPower().foregroundActive()); } } WebConfigServer ordinary; diff --git a/test/fixtures/webconfig_recovery_auth.cpp b/test/fixtures/webconfig_recovery_auth.cpp index 8833f2941..a23229e16 100644 --- a/test/fixtures/webconfig_recovery_auth.cpp +++ b/test/fixtures/webconfig_recovery_auth.cpp @@ -47,6 +47,8 @@ struct WebConfigServer { uint32_t _last_activity=0, _session_last_seen=0, _login_lock_until=0; int _login_fails=0; Callback callback; Callback* _cb=&callback; WebConfigServer() { strcpy(callback.node.admin_password, "existing-secret"); } + unsigned network_activity = 0; + void noteNetworkActivity() { ++network_activity; } bool checkAuth(AsyncWebServerRequest*); void handleLogin(AsyncWebServerRequest*); void handleLogout(AsyncWebServerRequest*); @@ -58,12 +60,15 @@ int main() { attacker.body="{\"set\":{\"password\":\"attacker\",\"wifi.ssid\":\"EvilNet\"}}"; attacker._tempObject=attacker.body.data(); assert(!portal.checkAuth(&attacker)); + assert(portal.network_activity==0); portal.checkConfigPost(&attacker); assert(attacker.status==401 && !portal.config_accepted); attacker.password("wrong"); portal.handleLogin(&attacker); assert(attacker.status==401 && portal._session_token[0]==0); + assert(portal.network_activity==0); AsyncWebServerRequest admin; admin.password("existing-secret"); portal.handleLogin(&admin); assert(admin.status==200); + assert(portal.network_activity>0); assert(strlen(portal._session_token)==32); assert(admin.response.headers["Set-Cookie"].find("HttpOnly; SameSite=Lax")!=std::string::npos); admin.header.cookies=std::string("wcs=")+portal._session_token; diff --git a/test/fixtures/webconfig_station_owner.cpp b/test/fixtures/webconfig_station_owner.cpp index bd96c7fdf..0888787e1 100644 --- a/test/fixtures/webconfig_station_owner.cpp +++ b/test/fixtures/webconfig_station_owner.cpp @@ -23,6 +23,7 @@ struct WebConfigServer { bool isRunning() const { return false; } bool isStopping() const { return false; } void updateWiFiOwnership(bool owner) { owns = owner; } + void requestNetworkAccess() {} bool startAutoMode(char* reply) { strcpy(reply, "starting"); return true; } bool startSetupMode(char* reply) { strcpy(reply, "setup"); return true; } }; diff --git a/test/fixtures/webconfig_station_recovery.cpp b/test/fixtures/webconfig_station_recovery.cpp index f0f5bbdaa..6857ac024 100644 --- a/test/fixtures/webconfig_station_recovery.cpp +++ b/test/fixtures/webconfig_station_recovery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include static uint32_t clock_ms = 1000; uint32_t millis() { return clock_ms; } void delay(unsigned) {} @@ -96,6 +97,7 @@ public: bool _stopping = false, _retry_saved_wifi_in_setup = false; bool _setup_reconnect_in_progress = false, _board_cmds_probed = false; bool _was_setup_ap = false, _initial_setup = false; + bool foreground = false; uint32_t _setup_reconnect_deadline = 0, _connect_deadline = 0; uint32_t _setup_started_at = 0, _last_activity = 0; WiFiReconnectPolicy::Tracker _wifi_reconnect_tracker; @@ -134,6 +136,7 @@ public: void probeBoardCommands() { _board_cmds_probed = true; } void serviceTerminal(uint32_t) {} void serviceRoom(uint32_t) {} + bool isNetworkForegroundActive(uint32_t) const { return foreground || _mode == MODE_SETUP; } bool bluetoothWiFiCoexistenceRequired() { return false; } bool acceptWiFi(Entry& e, const char* value) { @CREDENTIAL_BATCH@ @@ -197,6 +200,33 @@ int main() { companion.tick(16000); assert(companion.setup_starts == 1 && companion._retry_saved_wifi_in_setup); assert(!companion._runtime_wifi_ownership_managed); +#elif defined(TEST_BACKGROUND_SUSPEND) + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + portal._mode = WebConfigServer::MODE_LAN; + clock_ms = 301000; + portal.tick(clock_ms); + assert(WiFi.begins == 0 && portal.setup_starts == 0); + // A connecting listener must not reopen an AP or acquire STA while the + // background owner has deliberately stopped MQTT and NTP for the cooldown. + portal._mode = WebConfigServer::MODE_CONNECTING; + portal._connect_deadline = 16000; + portal.tick(clock_ms); + assert(WiFi.begins == 0 && portal.setup_starts == 0); + portal._connect_deadline = 0; + portal.foreground = true; + portal.tick(++clock_ms); + assert(WiFi.begins == 1 && portal._connect_deadline == clock_ms + 15000); + portal.foreground = false; + clock_ms += 16000; + portal.tick(clock_ms); + assert(portal.setup_starts == 0 && WiFi.begins == 1); + // Resuming the worker permits ordinary ownership/reconnect logic again. + power.setSuspended(false); + portal._mode = WebConfigServer::MODE_LAN; + clock_ms += 301000; + portal.tick(clock_ms); + assert(WiFi.begins == 2); #elif defined(TEST_EXTERNAL_OWNER) portal.updateWiFiOwnership(false); assert(portal._runtime_wifi_ownership_managed); diff --git a/test/test_companion_wifi_network_power.py b/test/test_companion_wifi_network_power.py new file mode 100644 index 000000000..85ee389c4 --- /dev/null +++ b/test/test_companion_wifi_network_power.py @@ -0,0 +1,474 @@ +"""Execute Companion's production outage transitions without hardware.""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from test_replay_reset_integration import extract_braced + + +ROOT = Path(__file__).resolve().parents[1] +MAIN = ROOT / "examples/companion_radio/main.cpp" + + +HARNESS = r''' +#include +#include +#include +#include + +namespace mesh { namespace wifi { +bool espnow = false, station_allowed = true; +int protocols = 0, channels = 0, associations = 0, restore_result = 0; +bool espNowChannelConstrained() { return espnow; } +bool stationMutationAllowed() { return station_allowed; } +void applyProtocolMask(int) { ++protocols; } +int restoreEspNowChannel() { ++channels; return restore_result; } +void setStationAutoReconnect(bool); +void beginStation(const char*, const char*) { ++associations; } +}} + +constexpr int WIFI_STA = 1, WIFI_OFF = 0, WIFI_IF_STA = 0, TCP_PORT = 5000; +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6; +constexpr int ESP_OK = 0; +uint32_t test_now = 10000; +uint32_t millis() { return test_now; } +struct Board { + bool ota = false, inhibit = true; + bool isOTAUpdateRunning() const { return ota; } + void setInhibitSleep(bool value) { inhibit = value; } +} board; +struct Interface { + bool connected = false, enabled = true; + int ends = 0, begins = 0; + bool isConnected() const { return connected; } + void end() { enabled = false; ++ends; } + void begin(int) { ++begins; } + void enable() { enabled = true; } +} wifi_interface; +struct ConsoleClient { + bool active = false; + bool connected() const { return active; } +} ota_console_client; +struct Wifi { + int current_mode = WIFI_STA; + int current_status = WL_DISCONNECTED; + bool reconnect = true, last_disconnect_stopped = false; + int disconnects = 0; + void setAutoReconnect(bool value) { reconnect = value; } + void disconnect(bool stop, bool) { + last_disconnect_stopped = stop; + ++disconnects; + } + void mode(int value) { current_mode = value; } + int status() const { return current_status; } +} WiFi; +void mesh::wifi::setStationAutoReconnect(bool value) { WiFi.setAutoReconnect(value); } + +struct Tracker { void noteDisconnected(uint32_t) {} } wifi_reconnect_tracker; +bool companion_wifi_active = true, companion_wifi_requested = true; +bool companion_wifi_background_stopped = false; +bool companion_wifi_espnow_restore_pending = false; +uint32_t companion_wifi_espnow_restore_at = 0; +bool companion_wifi_webconfig_start_pending = false; +bool companion_wifi_credential_reload_pending = false; +bool companion_wifi_setup_requested = false; +bool companion_wifi_has_credentials = true; +char configured_wifi_ssid[] = "saved-network"; +char configured_wifi_password[] = "saved-password"; +int ntp_cancels = 0, resets = 0, ps_applied = 0, console_starts = 0, console_stops = 0; +void cancelCompanionWiFiNtp(bool retry) { assert(!retry); ++ntp_cancels; } +void resetCompanionWiFiRecoveryState() { ++resets; } +void applyCompanionWiFiPowerSave() { ++ps_applied; } +void ota_console_start() { ++console_starts; } +void ota_console_stop() { ++console_stops; } +#define WIFI_DEBUG_PRINTLN(...) do {} while (0) +struct WebConfigServer { + static bool enabled; + static bool loadEnabled(bool) { return enabled; } +}; +bool WebConfigServer::enabled = true; +struct Mesh { + bool web_active = false, last_force_ap = false; + int web_starts = 0; + bool isWebConfigActiveOrStopping() const { return web_active; } + bool startWebConfig(bool force_ap, char*) { + ++web_starts; last_force_ap = force_ap; web_active = true; return true; + } +} the_mesh; + +@METHODS@ + +void reset() { + auto& power = mesh::wifi::networkPower(); + power.setManaged(false); + power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + power.setManaged(true); + power.setSuspended(false); + mesh::wifi::espnow = false; + mesh::wifi::station_allowed = true; + mesh::wifi::protocols = mesh::wifi::channels = mesh::wifi::associations = 0; + mesh::wifi::restore_result = 0; + test_now = 10000; + board = {}; wifi_interface = {}; ota_console_client = {}; WiFi = {}; + companion_wifi_active = companion_wifi_requested = true; + companion_wifi_background_stopped = false; + companion_wifi_espnow_restore_pending = false; + companion_wifi_espnow_restore_at = 0; + companion_wifi_webconfig_start_pending = false; + companion_wifi_credential_reload_pending = companion_wifi_setup_requested = false; + companion_wifi_has_credentials = true; + ntp_cancels = resets = ps_applied = console_starts = console_stops = 0; + the_mesh = {}; WebConfigServer::enabled = true; +} + +int main() { + reset(); + // Merely listening for a client does not hold the radio on. + serviceCompanionWiFiNetworkPowerHold(); + assert(!mesh::wifi::networkPower().foregroundActive()); + wifi_interface.connected = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + wifi_interface.connected = false; +#ifdef ENABLE_OTA + ota_console_client.active = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + ota_console_client.active = false; +#endif + companion_wifi_credential_reload_pending = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + companion_wifi_credential_reload_pending = false; + companion_wifi_setup_requested = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + companion_wifi_setup_requested = false; + board.ota = true; + serviceCompanionWiFiNetworkPowerHold(); + assert(mesh::wifi::networkPower().foregroundActive()); + + reset(); + mesh::wifi::networkPower().setSuspended(true); + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped); + assert(companion_wifi_active && companion_wifi_requested); + assert(!board.inhibit && !WiFi.reconnect); + assert(WiFi.current_mode == WIFI_OFF && WiFi.last_disconnect_stopped); + assert(wifi_interface.ends == 1 && ntp_cancels == 1); +#ifdef ENABLE_OTA + assert(console_stops == 1); +#endif + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(wifi_interface.ends == 1 && ntp_cancels == 1); + assert(mesh::wifi::associations == 0); + assert(strcmp(configured_wifi_ssid, "saved-network") == 0); + assert(strcmp(configured_wifi_password, "saved-password") == 0); + + // The worker owns the cooldown. Once it opens a recovery window, the + // externally owned Companion station starts exactly one saved association. + mesh::wifi::networkPower().setSuspended(false); + mesh::wifi::station_allowed = false; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped && mesh::wifi::associations == 0); + mesh::wifi::station_allowed = true; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && board.inhibit); + assert(wifi_interface.enabled && wifi_interface.begins == 1); + assert(mesh::wifi::associations == 1 && WiFi.reconnect); + assert(WiFi.current_mode == WIFI_STA && ps_applied == 1); +#ifdef ENABLE_OTA + assert(console_starts == 1); +#endif + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::associations == 1 && wifi_interface.begins == 1); + + // An active updater remains powered even if it starts between the worker's + // last ownership check and publication of the suspension flag. + reset(); + mesh::wifi::networkPower().setSuspended(true); + board.ota = true; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && board.inhibit); + assert(wifi_interface.ends == 0 && WiFi.disconnects == 0); + + // Primary and dynamically enabled ESP-NOW both need their driver/channel. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_background_stopped && !WiFi.reconnect); + assert(board.inhibit); + assert(WiFi.current_mode == WIFI_STA && !WiFi.last_disconnect_stopped); + assert(mesh::wifi::protocols == 1 && mesh::wifi::channels == 1); + + // Disconnect is asynchronous. The first restoration can fail until the + // station/scan ends; keep retrying the channel, never the disconnect. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + assert(companion_wifi_espnow_restore_pending && WiFi.disconnects == 1); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1 && WiFi.disconnects == 1); + test_now += 999; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1); + ++test_now; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && companion_wifi_espnow_restore_pending); + assert(WiFi.disconnects == 1 && mesh::wifi::protocols == 1); + test_now += 1000; + mesh::wifi::station_allowed = false; // a foreground AP owns the channel + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2); + mesh::wifi::station_allowed = true; + board.ota = true; + serviceCompanionEspNowChannelRestore(); + assert(mesh::wifi::channels == 2); + board.ota = false; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, true); + serviceCompanionEspNowChannelRestore(); + assert(mesh::wifi::channels == 2); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 3 && !companion_wifi_espnow_restore_pending); + test_now += 10000; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 3 && WiFi.disconnects == 1); + + // Retry deadlines remain safe when millis wraps during the disconnect. + reset(); + test_now = UINT32_MAX - 500; + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + test_now += 999; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 1); + ++test_now; + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending); + + // The same deferred repair runs after an intentional WiFi disable, without + // restarting that station or its listeners. A removed ESP-NOW owner cancels it. + reset(); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + stopCompanionInfrastructureWiFi(); + companion_wifi_requested = companion_wifi_active = false; + test_now += 1000; + mesh::wifi::restore_result = ESP_OK; + serviceCompanionWiFiBackgroundPower(); + assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending); + assert(WiFi.disconnects == 1 && mesh::wifi::associations == 0); + mesh::wifi::restore_result = 1; + stopCompanionInfrastructureWiFi(); + mesh::wifi::espnow = false; + test_now += 1000; + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_espnow_restore_pending && mesh::wifi::channels == 3); + + // Beginning a recovery association cancels a delayed channel write. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::espnow = true; + mesh::wifi::restore_result = 1; + serviceCompanionWiFiBackgroundPower(); + mesh::wifi::networkPower().setSuspended(false); + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_espnow_restore_pending && mesh::wifi::associations == 1); + + // A client can be accepted immediately before a stale worker suspension + // flag becomes visible. Its fresh foreground hold prevents teardown. + reset(); + mesh::wifi::networkPower().setSuspended(true); + wifi_interface.connected = true; + serviceCompanionWiFiNetworkPowerHold(); + serviceCompanionWiFiBackgroundPower(); + assert(!companion_wifi_background_stopped && WiFi.disconnects == 0); + assert(wifi_interface.ends == 0 && board.inhibit); + + // User-disabled or not-yet-active WiFi is never restarted by recovery. + reset(); + companion_wifi_requested = false; + mesh::wifi::networkPower().setSuspended(true); + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + companion_wifi_requested = true; + companion_wifi_active = false; + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + + // A plain WiFi Companion with no managed MQTT worker keeps its existing + // connection and standalone NTP policy; a stale flag cannot suspend it. + reset(); + mesh::wifi::networkPower().setSuspended(true); + mesh::wifi::networkPower().setManaged(false); + serviceCompanionWiFiBackgroundPower(); + assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped); + + // A saved-SSID boot cannot start the portal's independent fifteen-second + // AP fallback before the MQTT worker acquires its network policy. + reset(); + mesh::wifi::networkPower().setManaged(false); + startCompanionWiFiAutoWebConfig(); + assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig(); + assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0); + mesh::wifi::networkPower().setManaged(true); + assert(!companionWiFiAutomaticRecoveryAllowed()); + for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 0); + WiFi.current_status = WL_CONNECTED; + serviceCompanionWiFiAutoWebConfig(); + assert(!companion_wifi_webconfig_start_pending && the_mesh.web_starts == 1); + assert(!the_mesh.last_force_ap); + serviceCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 1); + + // Unconfigured devices still open setup, and unmanaged standalone WiFi + // retains its ordinary recovery AP. Explicit manual setup is independent. + reset(); + companion_wifi_has_credentials = false; + startCompanionWiFiAutoWebConfig(); + assert(the_mesh.web_starts == 1 && the_mesh.last_force_ap); + assert(!companion_wifi_webconfig_start_pending); + mesh::wifi::networkPower().setManaged(false); + assert(companionWiFiAutomaticRecoveryAllowed()); +} +''' + + +class CompanionWiFiNetworkPowerTests(unittest.TestCase): + def test_router_down_boot_starts_the_configured_mqtt_scheduler_once(self): + source = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text(encoding="ascii") + actual = extract_braced(source, "static void copyMqttString(") + actual += extract_braced(source, "void MyMesh::serviceMQTT(") + fixture = r''' +#define WITH_WEBCONFIG 1 +#include +#include +#include +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6; +struct { int status() const { return WL_DISCONNECTED; } } WiFi; +namespace mesh { struct Log { void println(const char*) {} }; +inline Log& usbLoggingPort() { static Log log; return log; } } +struct Bridge { + int starts = 0; bool running = false; + void begin() { ++starts; running = true; mesh::wifi::networkPower().setManaged(true); } + bool isRunning() const { return running; } +}; +struct MyMesh { + bool _mqtt_enabled = true, _wc_batch_active = false; + bool _mqtt_started = false, _mqtt_configured = true, setup = false; + int stops = 0; + struct { char wifi_ssid[33] = {}, wifi_password[65] = {}; } _mqtt_prefs; + Bridge* _mqtt_bridge; + void stopMQTT() { ++stops; } + bool isWebConfigSetupActive() const { return setup; } + void serviceMQTT(const char*, const char*); +}; +''' + scenarios = r''' +int main() { + Bridge bridge; MyMesh node; node._mqtt_bridge = &bridge; + assert(WiFi.status() == WL_DISCONNECTED); + node._wc_batch_active = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node._wc_batch_active = false; node.setup = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node.setup = false; node._mqtt_configured = false; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 0); + node._mqtt_configured = true; + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 1 && node._mqtt_started); + assert(mesh::wifi::networkPower().managed()); + assert(strcmp(node._mqtt_prefs.wifi_ssid, "saved-network") == 0); + assert(strcmp(node._mqtt_prefs.wifi_password, "saved-password") == 0); + node.serviceMQTT("saved-network", "saved-password"); + assert(bridge.starts == 1); + node._mqtt_enabled = false; + node.serviceMQTT("saved-network", "saved-password"); + assert(node.stops == 1); +} +''' + with tempfile.TemporaryDirectory(prefix="companion-router-outage-") as directory: + path = Path(directory) + cpp, binary = path / "fixture.cpp", path / "fixture" + cpp.write_text(fixture + actual + scenarios, encoding="ascii") + result = subprocess.run(["g++", "-std=c++11", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_production_transitions_with_and_without_ota_console(self): + compiler = os.environ.get("CXX") or shutil.which("g++") + self.assertIsNotNone(compiler) + source = MAIN.read_text(encoding="ascii") + methods = "\n".join(extract_braced(source, signature) for signature in ( + "static bool companionWiFiBackgroundSuspended(", + "static bool companionWiFiAutomaticRecoveryAllowed(", + "static void startCompanionWiFiAutoWebConfig(", + "static void serviceCompanionWiFiAutoWebConfig(", + "static void serviceCompanionWiFiNetworkPowerHold(", + "static void stopCompanionInfrastructureWiFi(", + "static void serviceCompanionEspNowChannelRestore(", + "static void serviceCompanionWiFiBackgroundPower(", + )) + sanitizer = (["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] if sys.platform.startswith("linux") else []) + with tempfile.TemporaryDirectory(prefix="companion-network-power-") as directory: + path = Path(directory) + cpp = path / "fixture.cpp" + cpp.write_text(HARNESS.replace("@METHODS@", methods), encoding="ascii") + for ota in (False, True): + with self.subTest(ota=ota): + binary = path / ("with-ota" if ota else "no-ota") + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-Werror", *sanitizer, *(["-DENABLE_OTA=1"] if ota else []), + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + + def test_background_paths_and_manual_recovery_use_the_shared_guard(self): + source = MAIN.read_text(encoding="ascii") + for signature in ("static void serviceCompanionWiFiCredentialReload(", + "static void startCompanionWiFi("): + body = extract_braced(source, signature) + self.assertLess(body.index("companionWiFiBackgroundSuspended()"), + body.index("WiFi.mode(WIFI_STA)")) + ntp = extract_braced(source, "static void serviceCompanionWiFiNtp(") + self.assertLess(ntp.index("networkPower().backgroundSuspended()"), + ntp.index("companion_wifi_ntp_operation.tryAcquire()")) + for signature in ("void requestCompanionWiFiSetup(", + "void scheduleCompanionWiFiCredentialReload(", + "bool setCompanionWiFiEnabled("): + self.assertIn("networkPower().requestProbe()", extract_braced(source, signature)) + self.assertIn("&& !companionWiFiBackgroundSuspended()\n" + " && !companion_wifi_background_stopped", source) + self.assertIn("if (!wifi_setup_recovery_mode && companionWiFiAutomaticRecoveryAllowed()", source) + start = extract_braced(source, "static void startCompanionWiFi(") + self.assertIn("startCompanionWiFiAutoWebConfig();", start) + self.assertNotIn("startWebConfig(", start) + self.assertNotIn("companion_wifi_webconfig_start_pending = false", start) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_companion_wifi_ntp.py b/test/test_companion_wifi_ntp.py index 837ee59ce..7ebe40280 100644 --- a/test/test_companion_wifi_ntp.py +++ b/test/test_companion_wifi_ntp.py @@ -119,7 +119,14 @@ class CompanionWiFiNtpIntegrationTests(unittest.TestCase): self.assertIn("_ntp_sync_pending = true", initialize) self.assertIn("_ntp_reconnect_latch.consumeIfConnected", task_loop) self.assertIn("_ntp_sync_pending = true", task_loop) - self.assertIn("syncTimeWithNTP();", task_loop) + self.assertIn("serviceBackgroundNtp(static_cast(now));", task_loop) + self.assertNotIn("syncTimeWithNTP();", task_loop) + background = mqtt[ + mqtt.index("void MQTTBridge::serviceBackgroundNtp(") : + mqtt.index("void MQTTBridge::mqttTaskLoop()") + ] + self.assertIn("pollNtpRefresh(now);", background) + self.assertIn("refreshNTP();", background) self.assertIn("_last_ntp_sync = millis();", sync) self.assertIn( "fresh_ntp ? 0 : mqttNtpRetryAt(_last_ntp_sync)", sync diff --git a/test/test_esp32_full_power_lifecycle.py b/test/test_esp32_full_power_lifecycle.py index 2c1f58f52..ff337c89d 100644 --- a/test/test_esp32_full_power_lifecycle.py +++ b/test/test_esp32_full_power_lifecycle.py @@ -24,6 +24,7 @@ FIXTURE = r''' #include #include #include +#include #include uint32_t now = 0; uint32_t millis() { return now; } @@ -143,6 +144,8 @@ struct MQTTNodeInfo { uint8_t* sf; uint8_t* cr; uint8_t* repeat_flag; bool repeat_when_nonzero; bool canonical_wifi = false; }; struct MQTTBridge : AbstractBridge { + bool network_suspended = false; + bool isNetworkSuspended() const { return network_suspended; } template explicit MQTTBridge(Args...) {} void setDeviceID(const char*) {} void setFirmwareVersion(const char*) {} void setBoardModel(const char*) {} void setBuildDate(const char*) {} @@ -156,6 +159,8 @@ struct WebConfigServer { uint32_t _setup_started_at = 0; int _mode = 0; static constexpr int MODE_SETUP = 1; + static constexpr int MODE_LAN = 2; + uint32_t foreground_until = 0; char _wifi_ssid[33] = {}; template WebConfigServer(void*, void* prefs, bool owner, Args...) : owns(owner) { if (prefs) strcpy(_wifi_ssid, static_cast(prefs)->wifi_ssid); @@ -165,6 +170,14 @@ struct WebConfigServer { bool isRunning() const { return running; } bool isStopping() const { return stopping; } void updateWiFiOwnership(bool owner) { owns = owner; } + void requestNetworkAccess() { + foreground_until = millis() + 120000; + mesh::wifi::networkPower().requestProbe(); + } + bool isNetworkForegroundActive(uint32_t tick) const { + return _mode == MODE_SETUP || stopping + || (foreground_until && int32_t(tick - foreground_until) < 0); + } bool startSetupMode(char* reply) { if (!allow_start) { if (owns) stopOwnedWiFiRadio(); @@ -406,6 +419,51 @@ class ESP32FullPowerLifecycleTest(unittest.TestCase): with self.subTest(role=role): self.compile_and_run(self.fixture(role), *FLAGS) + def test_suspended_mqtt_idle_portal_and_foreground_ownership(self): + checks = r''' +int main() { + MyMesh node; char reply[160] = {}; + strcpy(node._cli.prefs.wifi_ssid, "working-network"); + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); + assert(node.setMqttBridgeState(true)); + assert(node.startWebConfig(false, reply)); + node._webconfig->_mode = WebConfigServer::MODE_LAN; + now += 120001; // Explicit-start lease has elapsed; listener is idle. + power.consumeProbeRequest(); + node.@WORKER@->network_suspended = true; + power.setSuspended(true); + unsigned off_before = WiFi.off_modes; + node.serviceIdleWiFi(); + assert(WiFi.off_modes == off_before && sdk_started); + assert(node.hasPendingWork()); // Cannot sleep before worker shutdown completes. + WiFi.mode(WIFI_OFF); // Only the Core 0 network owner performs this shutdown. + node.serviceIdleWiFi(); + assert(!sdk_initialized && !node.hasPendingWork()); + assert(node.getPowerSaveSleepSeconds(30) == 30); + // An operator can wake the existing listener without creating a second + // server, taking station ownership, or deinitializing the worker's driver. + assert(node.startWebConfig(false, reply)); + assert(power.consumeProbeRequest()); + assert(node._webconfig->isNetworkForegroundActive(now)); + assert(node.hasPendingWork()); + now += 120001; + assert(!node.hasPendingWork()); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, true); + assert(node.hasPendingWork()); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); + assert(!node.hasPendingWork()); + power.setSuspended(false); + node.@WORKER@->network_suspended = false; + node.serviceIdleWiFi(); + assert(node.hasPendingWork()); // Resume is owned by the MQTT worker too. + power.setManaged(false); +} +''' + for role in ("simple_repeater", "simple_room_server"): + with self.subTest(role=role): + self.compile_and_run(self.fixture(role, checks), *FLAGS) + def test_primary_radios_and_wifi_disabled_images_do_not_reclaim_wifi(self): for role in ("simple_repeater", "simple_room_server"): source = (ROOT / "examples" / role / "MyMesh.cpp").read_text() diff --git a/test/test_esp32_full_sleep.py b/test/test_esp32_full_sleep.py index 02711b304..67c991bf8 100644 --- a/test/test_esp32_full_sleep.py +++ b/test/test_esp32_full_sleep.py @@ -98,6 +98,7 @@ int main() { #include #include #include +#include static uint32_t now = 1000; uint32_t millis() { return now; } using esp_err_t = int; @@ -135,15 +136,22 @@ struct Interfaces { bool pending = false; bool hasPendingIO() const { return pen struct Sensors { bool gps_uart = false; bool gpsUsesSerialUart(uint8_t uart) const { return uart == 1 && gps_uart; } } sensors; struct Button { bool polling = false; bool needsPolling() const { return polling; } } user_btn; struct Wireless { unsigned services = 0; unsigned enabled() const { return services; } } companion_wireless; +static bool companion_wifi_background_stopped = false; namespace mesh { bool logging = false, watchdog = false; bool isUsbLoggingEnabled() { return logging; } bool isUsbLoggingWatchdogArmed() { return watchdog; } +namespace wifi { +bool espnow_constrained = false; +bool espNowChannelConstrained() { return espnow_constrained; } +} namespace wireless { +constexpr unsigned WiFi = 1, Bluetooth = 2, EspNow = 4; struct Control { bool transition = false; bool pending() const { return transition; } } controller; Control& control() { return controller; } } } +@SUSPENDED@ @READY@ void idle() { @BRANCH@ } template void blocks(T& flag) { @@ -170,6 +178,49 @@ int main() { blocks(mesh::watchdog); blocks(mesh::wireless::controller.transition); prefs.powersaving_enabled = false; before = sleeps; idle(); assert(sleeps == before); prefs.powersaving_enabled = true; idle(); assert(sleeps == before + 1); + // A requested WiFi service and idle WebConfig listener remain logically + // enabled during MQTT cooldown. Sleep needs transport quiescence and the + // station owner's physical stop, rather than a disabled saved preference. + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + companion_wireless.services = mesh::wireless::WiFi; + companion_wifi_background_stopped = true; + sdk_result = ESP_ERR_WIFI_NOT_INIT; + before = sleeps; idle(); assert(sleeps == before + 1); + for (int stopped_result : {ESP_ERR_WIFI_NOT_STARTED, ESP_OK}) { + sdk_result = stopped_result; sdk_mode = WIFI_MODE_NULL; + before = sleeps; idle(); assert(sleeps == before + 1); + } + sdk_result = ESP_ERR_WIFI_NOT_INIT; + blocks(mesh::wifi::espnow_constrained); + for (wifi_mode_t mode : {WIFI_MODE_STA, WIFI_MODE_AP, WIFI_MODE_APSTA}) { + sdk_mode = mode; sdk_result = ESP_OK; + before = sleeps; idle(); assert(sleeps == before); + } + sdk_mode = WIFI_MODE_NULL; sdk_result = 99; + before = sleeps; idle(); assert(sleeps == before); // Failed physical cleanup/read. + sdk_result = ESP_ERR_WIFI_NOT_INIT; + companion_wifi_background_stopped = false; + before = sleeps; idle(); assert(sleeps == before); // Owner has not stopped STA yet. + companion_wifi_background_stopped = true; + for (unsigned service : {mesh::wireless::Bluetooth, mesh::wireless::EspNow}) { + companion_wireless.services = mesh::wireless::WiFi | service; + before = sleeps; idle(); assert(sleeps == before); + } + companion_wireless.services = mesh::wireless::WiFi; + for (auto hold : {mesh::wifi::NetworkPowerHold::WebConfig, + mesh::wifi::NetworkPowerHold::CompanionClient, + mesh::wifi::NetworkPowerHold::OtaSeeder}) { + power.setHold(hold, true); + before = sleeps; idle(); assert(sleeps == before); + power.setHold(hold, false); + } + blocks(board.ota); blocks(bt_controller_active); blocks(bt_host_active); + power.setSuspended(false); + before = sleeps; idle(); assert(sleeps == before); // Worker has begun its probe. + power.setManaged(false); + companion_wifi_background_stopped = false; + companion_wireless.services = 0; sdk_mode = WIFI_MODE_NULL; sdk_result = ESP_OK; // The real board policy must observe USB even while pending work/logging // inhibit this branch. A rebooting host cannot disappear for one SOF gap // and cause Full to shut down its console immediately. @@ -179,6 +230,8 @@ int main() { now += 120000; idle(); assert(sleeps == before + 1); } '''.replace("@READY@", ready).replace("@BRANCH@", branch).replace("@USB_HELD@", usb_held) + source = source.replace("@SUSPENDED@", function(main, + "static bool companionWiFiBackgroundSuspended()")) source = "#include \n" + source flags = ("ESP32_PLATFORM", "ESP32", "ENABLE_USB_INTERFACE", "ARDUINO_USB_CDC_ON_BOOT=1", 'WIFI_SSID=""', "BLE_PIN_CODE=123456", "MESH_USB_LOGGING_AVAILABLE=1", @@ -189,7 +242,7 @@ int main() { with self.subTest(pm=pm): result = run_cpp(source, (*flags, f"COMPANION_IDF_PM_AVAILABLE={pm}")) self.assertEqual(result.returncode, 0, result.stderr) - broken = run_cpp(source.replace("companion_wireless.enabled() != 0", "false"), + broken = run_cpp(source.replace("enabled_services != 0", "false"), (*flags, "COMPANION_IDF_PM_AVAILABLE=0")) self.assertNotEqual(broken.returncode, 0) diff --git a/test/test_mqtt_disconnect.py b/test/test_mqtt_disconnect.py new file mode 100644 index 000000000..10fbbc7c3 --- /dev/null +++ b/test/test_mqtt_disconnect.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Run the real MQTT wrapper stop and outage handoff under SDK failures.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from test_mqtt_publish_ack import ARDUINO_HEADER, MQTT_HEADER +from test_mqtt_transport_results import method +from test_replay_reset_integration import extract_braced + + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER = "lib/PsychicMqttClient/src/PsychicMqttClient.cpp" +BRIDGE = "src/helpers/bridges/MQTTBridge.cpp" + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define private public +#include "PsychicMqttClient.h" +#undef private + +template void log_stub(Args&&...) {} +static const char* TAG = "test"; +#define ESP_LOGI(...) log_stub(__VA_ARGS__) +#define ESP_LOGW(...) log_stub(__VA_ARGS__) +#define ESP_ERROR_CHECK_WITHOUT_ABORT(value) ((void)(value)) +#define MQTT_DEBUG_PRINTLN(...) ((void)0) +#define pdMS_TO_TICKS(ms) ((ms) / portTICK_PERIOD_MS) +static uint32_t now_ms = 0, elapsed_ms = 0, tick_ms = 1; +static uint32_t callback_after_ms = UINT32_MAX; +static int disconnect_result = ESP_OK, stop_result = ESP_OK; +static int disconnect_calls = 0, stop_calls = 0, sleeps = 0; +static PsychicMqttClient* active_client = nullptr; +static std::function on_delay; +unsigned long millis() { return now_ms; } +const char* esp_err_to_name(int) { return "mock"; } + +static void deliverDisconnect() { + if (!active_client || elapsed_ms < callback_after_ms) return; + active_client->_connected = false; + active_client->_stopMqttClient = true; + callback_after_ms = UINT32_MAX; +} +void vTaskDelay(unsigned ticks) { + assert(ticks != 0); + ++sleeps; + now_ms += ticks * tick_ms; + elapsed_ms += ticks * tick_ms; + deliverDisconnect(); + if (on_delay) on_delay(); + assert(elapsed_ms <= 2000U + tick_ms); +} +int esp_mqtt_client_disconnect(void*) { + ++disconnect_calls; + if (disconnect_result == ESP_OK) deliverDisconnect(); + return disconnect_result; +} +int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; } +// Destruction is outside this test's stop/handoff seam; failed SDK resources +// remain owned by the surrounding bridge lifecycle in the production code. +PsychicMqttClient::~PsychicMqttClient() {} + +@ADAPTER_METHODS@ + +constexpr int RUNTIME_MQTT_SLOTS = 1, WIFI_MODE_NULL = 0; +constexpr int WIFI_MODE_AP = 2, WIFI_STA = 1, WIFI_OFF = 0; +using wifi_mode_t = int; +static int wifi_mode = WIFI_STA, wifi_stops = 0; +int esp_wifi_get_mode(int* mode) { *mode = wifi_mode; return ESP_OK; } +bool mqttStationWiFiMutationAllowed() { return !(wifi_mode & WIFI_MODE_AP); } +namespace mesh { namespace wifi { +bool espNowChannelConstrained() { return false; } +int restoreEspNowChannel() { assert(false); return ESP_ERR_INVALID_STATE; } +} } +struct { + void setAutoReconnect(bool enabled) { assert(!enabled); } + void disconnect(bool stop, bool erase) { assert(!stop && !erase); } + int getMode() const { return wifi_mode; } + void mode(int mode) { + assert(mode == WIFI_OFF && mesh::wifi::networkPower().backgroundSuspended()); + wifi_mode = mode; + ++wifi_stops; + } +} WiFi; + +struct MQTTBridge { + struct Slot { + PsychicMqttClient* client = nullptr; + bool enabled = true, connected = false, circuit_breaker_tripped = false; + uint32_t connected_at_ms = 0, last_reconnect_attempt = 0; + uint8_t reconnect_backoff = 0; + bool initial_connect_done = true; + char* auth_token = nullptr; + char broker_uri[32] = "mqtt://original"; + uint32_t token_expires_at = 1, last_token_renewal = 1; + uint32_t last_log_time = 1, last_deferred_log_ms = 1; + uint8_t max_backoff_failures = 0; + } _slots[RUNTIME_MQTT_SLOTS]; + using MQTTSlot = Slot; + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}, _fresh_ntp_this_boot{false}; + bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; + bool _status_publish_pending[RUNTIME_MQTT_SLOTS] = {}; + bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {}; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + bool _ntp_force_requested = false, _ntp_diag_requested = false; + bool _ntp_sync_pending = false, _manage_wifi = true, _wifi_status_initialized = true; + uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0; + uint32_t _ntp_refresh_retry_at = 0; + std::atomic _stop_requested{false}, _stop_acked{false}; + int destroyed = 0, queues = 0, ntp_starts = 0, slot_starts = 0; + struct Board { bool isOTAUpdateRunning() const { return false; } }; + Board* _board = nullptr; + void cancelNtpRefresh() {} + void updateCachedConnectionStatus() {} + void initializeWiFiInTask() {} + void processPacketQueue() { ++queues; } + void teardownSlot(int index, bool force = false); + void destroySlotClients() { + assert(!_stop_acked); + for (auto& slot : _slots) { + assert(!slot.client || !slot.client->isStarted()); + slot.client = nullptr; + } + ++destroyed; + } + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void workerStop() { + while (true) { +@TASK_STOP@ + } + } + void workerOutageOnce() { + for (unsigned iteration = 0; iteration < 1; ++iteration) { + const uint32_t now = now_ms; +@OUTAGE_GUARD@ + ++ntp_starts; + ++slot_starts; + } + } +}; + +@BRIDGE_METHOD@ + +int main(int argc, char** argv) { + assert(argc == 4); + const int scenario = std::atoi(argv[1]); + tick_ms = static_cast(std::atoi(argv[2])); + if (std::atoi(argv[3])) now_ms = UINT32_MAX - 511U; + PsychicMqttClient client; + active_client = &client; + client._client = reinterpret_cast(1); + client._started = true; + client._connected = true; + if (scenario == 0) { + client.disconnect(); // Lost event: SDK stop is sufficient proof. + assert(elapsed_ms >= 1000U && elapsed_ms < 1000U + tick_ms); + assert(stop_calls == 1 && !client.isStarted() && !client.connected()); + const int stopped = stop_calls; + client.disconnect(); + assert(stop_calls == stopped); // already stopped is idempotent + } else if (scenario == 1) { + stop_result = ESP_ERR_INVALID_STATE; + auto& control = mesh::wifi::networkPower(); + control.setManaged(true); + MQTTBridge bridge; + bridge._slots[0].client = &client; + const uint32_t start = now_ms; + assert(bridge.serviceNetworkPower(start)); + now_ms = start + 300000U; + assert(!bridge.serviceNetworkPower(now_ms)); + assert(elapsed_ms >= 1000U && stop_calls == 1 && client.isStarted()); + assert(!bridge._network_suspended && !control.backgroundSuspended()); + assert(wifi_stops == 0 && wifi_mode == WIFI_STA); + // An owning-task retry can now complete teardown without another wait if + // the disconnect request itself fails. Only the successful SDK stop grants + // the bridge permission to turn the station off. + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_OK; + assert(!bridge.serviceNetworkPower(now_ms)); + assert(!client.isStarted() && control.backgroundSuspended() && wifi_stops == 1); + } else if (scenario == 2) { + disconnect_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(sleeps == 0 && disconnect_calls == 1 && stop_calls == 1); + assert(!client.isStarted() && !client.connected()); + } else if (scenario == 3) { + callback_after_ms = 0; + stop_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(sleeps == 0 && stop_calls == 1 && client.isStarted()); + assert(!client.connected()); // Actual event state survives failed stop. + } else if (scenario == 4) { + callback_after_ms = 50U; + client.disconnect(); + assert(elapsed_ms >= 50U && elapsed_ms < 50U + tick_ms); + assert(stop_calls == 1 && !client.isStarted() && !client.connected()); + } else if (scenario == 5) { + client._started = false; + client.disconnect(); + client._client = nullptr; + client.disconnect(); + assert(disconnect_calls == 0 && stop_calls == 0 && sleeps == 0); + } else if (scenario == 6) { + client._connected = false; + stop_result = ESP_ERR_INVALID_STATE; + client.disconnect(); + assert(disconnect_calls == 0 && stop_calls == 1 && sleeps == 0); + assert(client.isStarted()); + } else if (scenario == 7) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + MQTTBridge bridge; + bridge._slots[0].client = &client; + bridge._stop_requested = true; + on_delay = [&]() { + assert(!bridge._stop_acked && bridge.destroyed == 0); + assert(bridge._slots[0].client == &client); + if (elapsed_ms >= 150U) stop_result = ESP_OK; + }; + bridge.workerStop(); + assert(stop_calls == 2 && !client.isStarted()); + assert(bridge._stop_acked && bridge.destroyed == 1); + assert(bridge._slots[0].client == nullptr); + on_delay = nullptr; + } else if (scenario == 8) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + MQTTBridge bridge; + char token[] = "protected-token"; + bridge._slots[0].client = &client; + bridge._slots[0].auth_token = token; + bridge.teardownSlot(0); + assert(client.isStarted() && strcmp(token, "protected-token") == 0); + assert(bridge._slots[0].initial_connect_done); + assert(strcmp(bridge._slots[0].broker_uri, "mqtt://original") == 0); + } else if (scenario == 9) { + disconnect_result = ESP_ERR_INVALID_STATE; + stop_result = ESP_ERR_INVALID_STATE; + auto& control = mesh::wifi::networkPower(); + control.setManaged(true); + MQTTBridge bridge; + bridge._slots[0].client = &client; + const uint32_t start = now_ms; + assert(bridge.serviceNetworkPower(start)); + now_ms = start + 300000U; + bridge.workerOutageOnce(); + assert(client.isStarted() && !bridge._network_suspended); + assert(!control.backgroundSuspended() && wifi_stops == 0); + assert(bridge.queues == 1 && bridge.ntp_starts == 0 && bridge.slot_starts == 0); + } else assert(false); + printf("PASS: %d\n", scenario); +} +''' + + +class MqttDisconnectTests(unittest.TestCase): + def test_real_wrapper_and_wifi_handoff_across_sdk_failures_ticks_and_wrap(self): + compiler = os.environ.get("CXX") or shutil.which("g++") + self.assertIsNotNone(compiler) + adapter = "\n".join(method(ADAPTER, signature) for signature in ( + "PsychicMqttClient::PsychicMqttClient()", + "bool PsychicMqttClient::connected()", + "void PsychicMqttClient::disconnect()", + "void PsychicMqttClient::forceStop()", + "unsigned long PsychicMqttClient::getPublishAck()", + )) + task = method(BRIDGE, "void MQTTBridge::mqttTaskLoop()") + source = (HARNESS.replace("@ADAPTER_METHODS@", adapter) + .replace("@TASK_STOP@", extract_braced(task, " if (_stop_requested)")) + .replace("@OUTAGE_GUARD@", extract_braced(task, " if (!serviceNetworkPower(")) + .replace("@BRIDGE_METHOD@", "\n".join(method(BRIDGE, signature) for signature in ( + "bool MQTTBridge::serviceNetworkPower(", + "void MQTTBridge::serviceEspNowChannelRestore(", + "void MQTTBridge::teardownSlot(", + )))) + with tempfile.TemporaryDirectory(prefix="meshcore-mqtt-disconnect-") as temporary: + directory = Path(temporary) + (directory / "Arduino.h").write_text(ARDUINO_HEADER, encoding="ascii") + (directory / "mqtt_client.h").write_text(MQTT_HEADER, encoding="ascii") + (directory / "esp_crt_bundle.h").write_text("#pragma once\n", encoding="ascii") + cpp = directory / "disconnect.cpp" + cpp.write_text(source, encoding="ascii") + for idf in (4, 5): + for tick in (1, 10, 50): + with self.subTest(idf=idf, tick=tick): + binary = directory / f"disconnect-{idf}-{tick}" + arduino = ARDUINO_HEADER.replace( + "constexpr unsigned portTICK_PERIOD_MS = 1;", + f"constexpr unsigned portTICK_PERIOD_MS = {tick};") + (directory / "Arduino.h").write_text(arduino, encoding="ascii") + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-Werror", "-DARDUINO_ARCH_ESP32=1", "-DESP_PLATFORM=1", + f"-DESP_IDF_VERSION_MAJOR={idf}", + "-I", str(directory), "-I", str(ROOT / "lib/PsychicMqttClient/src"), + "-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + for scenario in range(10): + for wrap in (0, 1): + with self.subTest(scenario=scenario, wrap=wrap): + checked = subprocess.run([str(binary), str(scenario), str(tick), str(wrap)], + capture_output=True, text=True, timeout=5) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_mqtt_network_timeout.py b/test/test_mqtt_network_timeout.py index 0b24217c8..0586b7252 100644 --- a/test/test_mqtt_network_timeout.py +++ b/test/test_mqtt_network_timeout.py @@ -150,6 +150,7 @@ struct MQTTBridge { } _slots[RUNTIME_MQTT_SLOTS]; std::atomic _stop_requested{false}, _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {}; + bool _slot_reconfigure_pending[RUNTIME_MQTT_SLOTS] = {}; bool _initialized = true; int _max_active_slots = 2; MQTTPrefs prefs; @@ -189,6 +190,14 @@ int main() { assert(slot.client->_buffer_capacity == 896); char token[8] = "old-jwt"; slot.auth_token = token; + stop_result = ESP_ERR_INVALID_STATE; + bridge.applySlotPreset(0, "jwt"); + assert(slot.client->isStarted() && slot.initial_connect_done); + assert(slot.preset == &plain_preset && strcmp(token, "old-jwt") == 0); + assert(bridge._slot_reconfigure_pending[0]); + assert(init_calls == 1 && destroy_calls == 0); + stop_result = ESP_OK; + bridge._slot_reconfigure_pending[0] = false; // worker consumes the retry flag bridge.applySlotPreset(0, "jwt"); assert(token[0] == '\0' && slot.preset == &jwt_preset); assert(slot.client->_buffer_capacity == 896); @@ -287,6 +296,10 @@ class MqttNetworkTimeoutTest(unittest.TestCase): preamble = preamble.replace( 'void vTaskDelay(unsigned) { assert(false && "stub disconnect must deliver its event"); }', 'void vTaskDelay(unsigned ticks) { assert(ticks == 50); }') + preamble = preamble.replace( + 'int esp_mqtt_client_stop(void*) { ++stop_calls; return ESP_OK; }', + 'static int stop_result = ESP_OK;\n' + 'int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; }') source = "#define ESP_PLATFORM 1\n" + preamble + RECONFIGURE_PREAMBLE source += "\n".join(method(adapter, signature) for signature in ( "PsychicMqttClient::PsychicMqttClient()", diff --git a/test/test_mqtt_transport_results.py b/test/test_mqtt_transport_results.py index 100993887..2068783eb 100644 --- a/test/test_mqtt_transport_results.py +++ b/test/test_mqtt_transport_results.py @@ -191,6 +191,7 @@ TASK_PREAMBLE = r''' #include #include #include +#include "helpers/WiFiNetworkPowerPolicy.h" #define PORTABLE_MQTT_OBSERVER 1 #define MQTT_DEBUG_PRINTLN(...) ((void)0) using TickType_t = uint32_t; @@ -215,7 +216,12 @@ struct MQTTPrefs { bool mqtt_status_enabled = true; }; struct MQTTBridge { + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + bool _network_espnow_restore_pending = false; struct Slot { + struct Client { bool isStarted() const { return false; } }; + Client* client = nullptr; bool enabled = true, initial_connect_done = false, connected = false; unsigned long last_reconnect_attempt = 0; } _slots[RUNTIME_MQTT_SLOTS]; @@ -244,6 +250,14 @@ struct MQTTBridge { assert(!_stop_acked); } bool initializeWiFiInTask() { return true; } + bool serviceNetworkPower(uint32_t) { work(); return true; } + void serviceBackgroundNtp(uint32_t) { + work(); + if (scenario == 7) { + _ntp_synced = true; + _stop_requested = true; + } + } bool handleWiFiConnection(unsigned long) { work(); return false; } bool syncTimeWithNTP(bool = false, bool = false) { work(); @@ -296,6 +310,7 @@ struct MQTTBridge { } bool waitUnlessStopping(uint32_t delay_ms); void mqttTaskLoop(); + void applyTimezoneRules() {} }; void vTaskDelay(TickType_t ticks) { assert(ticks > 0); @@ -375,6 +390,7 @@ ASYNC_STOP_PREAMBLE = r''' #include #include "helpers/MQTTLifecycle.h" #include "helpers/MQTTConnectionPolicy.h" +#include "helpers/WiFiNetworkPowerPolicy.h" #define ESP_PLATFORM 1 #define MQTT_DEBUG_PRINTLN(...) ((void)0) #define pdMS_TO_TICKS(ms) (ms) @@ -406,6 +422,7 @@ struct MQTTBridge { } ops{this}; MQTTLifecycle::Coordinator _lifecycle{ops, 1}; std::atomic _stop_requested{false}, _stop_acked{false}; + std::atomic _network_suspended{true}; bool _initialized = true, resources_owned = true; bool _slots_setup_done = true, _staged_raw_valid = true; bool _ntp_estimate_requested = true, _ntp_estimate_done = true, _ntp_estimate_ok = true; @@ -416,6 +433,8 @@ struct MQTTBridge { assert(_lifecycle.requestStart()); assert(_lifecycle.onTaskStarted()); s_mqtt_bridge_instance = this; + mesh::wifi::networkPower().setManaged(true); + mesh::wifi::networkPower().setSuspended(true); } void cancelNtpRefresh() { ++cancels; } void requestStop(); @@ -452,6 +471,7 @@ static void assert_finished(MQTTBridge& bridge, bool clean) { assert(!bridge._ntp_estimate_requested && !bridge._ntp_estimate_done); assert(!bridge._ntp_estimate_ok && bridge._ntp_estimate_epoch == 0); assert(bridge.cancels == 1); + assert(!bridge._network_suspended && !mesh::wifi::networkPower().managed()); } ''' @@ -547,6 +567,7 @@ class MqttTransportResultsTests(unittest.TestCase): path.write_text(source, encoding="utf-8") exe = Path(temp) / ("worker.exe" if os.name == "nt" else "worker") result = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", + "-I", str(ROOT / "src"), str(path), "-o", str(exe)], text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) for scenario in range(12): diff --git a/test/test_mqtt_wifi_network_power.py b/test/test_mqtt_wifi_network_power.py new file mode 100644 index 000000000..1a1672897 --- /dev/null +++ b/test/test_mqtt_wifi_network_power.py @@ -0,0 +1,856 @@ +#!/usr/bin/env python3 +"""Execute MQTT's production WiFi/SNTP outage handoff with controlled SDK calls. + +The actual bridge methods and shared policies run without hardware, network, +Arduino, or PlatformIO. SDK calls observe suspension publication ordering and +can fail independently of MQTT and NTP state. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from test_mqtt_transport_results import method + + +ROOT = Path(__file__).resolve().parents[1] +BRIDGE = "src/helpers/bridges/MQTTBridge.cpp" + +PREAMBLE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ESP_PLATFORM 1 +#define WITH_SNMP 1 +#define MQTT_DEBUG_PRINTLN(...) ((void)0) +using wifi_mode_t = int; +constexpr int ESP_OK = 0, ESP_FAIL = -1, WIFI_MODE_NULL = 0; +constexpr int WIFI_STA = 1, WIFI_MODE_AP = 2, WIFI_OFF = 0; +constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 0, RUNTIME_MQTT_SLOTS = 3; +constexpr int SNTP_SYNC_STATUS_RESET = 0, SNTP_SYNC_STATUS_COMPLETED = 1; +static constexpr uint32_t MQTT_NTP_REFRESH_TIMEOUT_MS = 15000U; +static constexpr uint32_t MQTT_NTP_NON_ESP_SETTLE_MS = 5000U; +static constexpr uint32_t MQTT_NTP_MIN_VALID_EPOCH = 1767225600U; +static uint32_t clock_ms = 100U; +static int64_t system_epoch = 0; +static int sdk_mode = WIFI_STA, sdk_mode_result = ESP_OK; +static bool station_mutation_allowed = true, espnow_constrained = false; +static int ntp_status = SNTP_SYNC_STATUS_RESET, ntp_configs = 0; +static int ntp_stops = 0, channel_restores = 0; +static int channel_restore_result = ESP_OK; +static bool require_quiescent_wifi_stop = false; +static std::vector events; +static std::vector ntp_primaries; +static std::function on_client_stop; + +static void require(bool value, const char* reason) { + if (!value) throw std::runtime_error(reason); +} +uint32_t millis() { return clock_ms; } +time_t fake_time(void*) { return static_cast(system_epoch); } +int fake_settimeofday(const timeval* value, void*) { + system_epoch = value->tv_sec; + return 0; +} +#define time fake_time +#define settimeofday fake_settimeofday + +int esp_wifi_get_mode(wifi_mode_t* mode) { + *mode = sdk_mode; + return sdk_mode_result; +} +bool mqttStationWiFiMutationAllowed() { + return station_mutation_allowed && sdk_mode_result == ESP_OK + && !(sdk_mode & WIFI_MODE_AP); +} +namespace mesh { namespace wifi { +bool espNowChannelConstrained() { return espnow_constrained; } +int restoreEspNowChannel() { + ++channel_restores; + events.push_back("espnow"); + return channel_restore_result; +} +} } + +struct WiFiModel { + int state = WL_CONNECTED; + bool reconnect = true; + int disconnects = 0, offs = 0; + int status() const { return state; } + int getMode() const { return sdk_mode; } + void setAutoReconnect(bool value) { + reconnect = value; + if (!value && require_quiescent_wifi_stop) { + require(mesh::wifi::networkPower().backgroundSuspended(), + "WiFi stopped before worker published transport quiescence"); + } + } + void disconnect(bool stop, bool erase) { + require(!stop && !erase, "suspension erased credentials or ESP-NOW driver"); + ++disconnects; + state = WL_DISCONNECTED; + events.push_back("wifi-disconnect"); + } + void mode(int value) { + require(value == WIFI_OFF, "unexpected station mode change"); + if (require_quiescent_wifi_stop) { + require(mesh::wifi::networkPower().backgroundSuspended(), + "WiFi driver stopped before the safe handoff"); + } + sdk_mode = value; + ++offs; + events.push_back("wifi-off"); + } +} WiFi; + +void sntp_set_sync_status(int value) { ntp_status = value; } +int sntp_get_sync_status() { return ntp_status; } +void esp_sntp_stop() { + if (require_quiescent_wifi_stop) { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "SNTP was stopped after publishing transport quiescence"); + } + ++ntp_stops; + events.push_back("sntp-stop"); +} +void configTime(int, int, const char* primary, const char*, const char*) { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "SNTP restarted during suspension"); + ++ntp_configs; + ntp_primaries.push_back(primary); + events.push_back("sntp-start"); +} +void configTime(int a, int b, const char* primary) { + configTime(a, b, primary, nullptr, nullptr); +} + +struct Client { + bool started = true, fail_stop = false; + int stops = 0; + uint32_t publish_acks = 0; + uint32_t getPublishAck() const { return publish_acks; } + bool isStarted() const { return started; } + void disconnect() { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "TLS stopped after handing station off"); + ++stops; + events.push_back("tls-stop"); + if (!fail_stop) started = false; + if (on_client_stop) on_client_stop(); + } +}; +struct Board { + bool ota = false; + bool isOTAUpdateRunning() const { return ota; } +}; +struct Snmp { + bool running = false; + bool isRunning() const { return running; } +}; +struct Rtc { + uint32_t epoch = 0; + uint32_t getCurrentTime() const { return epoch; } + void setCurrentTime(uint32_t value) { epoch = value; } +}; +struct Prefs {}; +const char* effectiveNtpPrimary(const Prefs*) { return "custom"; } +#ifdef ESP_PLATFORM +static void fillNtpServerList(const Prefs*, const char* servers[], int& count) { + static const char* names[] = {"custom", "pool", "google", "cloudflare", "aws", "nist"}; + count = 6; + for (int index = 0; index < count; ++index) servers[index] = names[index]; +} +#endif + +@HELPERS@ + +struct MQTTBridge { + static constexpr int kMaxNtpServers = 6; + struct Slot { + bool enabled = true, connected = false, circuit_breaker_tripped = true; + uint32_t connected_at_ms = 0, last_reconnect_attempt = 200U; + uint8_t reconnect_backoff = 5; + Client* client = nullptr; + } _slots[RUNTIME_MQTT_SLOTS]; + Client clients[RUNTIME_MQTT_SLOTS]; + std::atomic _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {}; + std::atomic _status_publish_pending[RUNTIME_MQTT_SLOTS] = {}; + uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {}; + bool _network_espnow_restore_pending = false; + uint32_t _network_espnow_restore_at = 0; + mesh::wifi::NetworkPowerPolicy _network_power_policy; + std::atomic _network_suspended{false}; + std::atomic _ntp_synced{false}, _fresh_ntp_this_boot{false}; + std::atomic _backward_clock_reset_epoch{0}; + bool _manage_wifi = true, _wifi_status_initialized = true; + bool _ntp_force_requested = false, _ntp_diag_requested = false; + bool _ntp_estimate_requested = false, _ntp_sync_pending = true; + bool _ntp_estimate_done = false, _ntp_estimate_ok = false; + uint32_t _ntp_estimate_epoch = 0; + std::atomic _stop_requested{false}; + bool _ntp_refresh_pending = false; + uint32_t _ntp_refresh_started_at = 0, _ntp_refresh_previous_epoch = 0; + uint32_t _ntp_refresh_retry_at = 0, _last_ntp_sync = 0; + uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0; + uint8_t _background_ntp_server = 0; + Board board; + Board* _board = &board; + Snmp snmp; + Snmp* _snmp_agent = &snmp; + Rtc rtc; + Rtc* _rtc = &rtc; + Prefs prefs; + Prefs* _obs = &prefs; + int initializes = 0, cache_updates = 0; + mesh::sntp_coord::OperationLease _ntp_refresh_operation{ + mesh::sntp_coord::processWideCoordinator(), stopMqttSntpOperation}; + MQTTBridge() { + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + _slots[index].client = &clients[index]; + _slot_attempt_pending[index] = true; + _status_publish_pending[index] = true; + } + } + bool initializeWiFiInTask() { + require(!mesh::wifi::networkPower().backgroundSuspended(), + "station resumed before clearing suspension handoff"); + ++initializes; + sdk_mode = WIFI_STA; + WiFi.reconnect = true; + return true; + } + void updateCachedConnectionStatus() { ++cache_updates; } + bool isRunning() const { return true; } + bool isNetworkSuspended() const { return _network_suspended.load(); } + bool requestNtpTimeEstimate(); + bool serviceNetworkPower(uint32_t now); + void serviceEspNowChannelRestore(uint32_t now); + void serviceBackgroundNtp(uint32_t now); + void cancelNtpRefresh(); + void refreshNTP(); + void pollNtpRefresh(uint32_t now); +}; + +@METHODS@ + +static void resetEnvironment() { + auto& power = mesh::wifi::networkPower(); + power.setManaged(false); + power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false); + power.consumeProbeRequest(); + power.setManaged(true); + clock_ms = 100U; + system_epoch = 0; + sdk_mode = WIFI_STA; + sdk_mode_result = ESP_OK; + station_mutation_allowed = true; + espnow_constrained = false; + ntp_status = SNTP_SYNC_STATUS_RESET; + ntp_configs = ntp_stops = channel_restores = 0; + channel_restore_result = ESP_OK; + require_quiescent_wifi_stop = false; + events.clear(); + ntp_primaries.clear(); + on_client_stop = nullptr; + WiFi = {}; +} + +static constexpr uint32_t grace = 300000U, interval = 1800000U, probe = 120000U; + +static void failureStopsBothServicesBeforeWifi() { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(clock_ms), "cold bridge was suspended"); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "initial NTP was not asynchronous"); + clock_ms += 15000U; + bridge.serviceBackgroundNtp(clock_ms); + require(!bridge._ntp_refresh_pending && ntp_stops == 1, + "timed-out NTP continued running"); + require(bridge._ntp_refresh_retry_at == clock_ms + interval, + "failed NTP was not deferred 30 minutes"); + // GOT_IP/NTP retry timers and failing SDK handshakes do not renew the clock. + for (uint32_t offset = 16000U; offset < grace; offset += 1000U) { + clock_ms = 100U + offset; + WiFi.state = WL_CONNECTED; + require(bridge.serviceNetworkPower(clock_ms), "outage grace ended early"); + bridge.serviceBackgroundNtp(clock_ms); + } + require(ntp_configs == 1, "failed SNTP retried during its cooldown"); + // Keep one live SNTP lease at the cutoff to prove teardown publication order. + bridge._ntp_refresh_retry_at = 0; + bridge.refreshNTP(); + require(bridge._ntp_refresh_operation.owns(), "SNTP request lacks ownership"); + events.clear(); + require_quiescent_wifi_stop = true; + clock_ms = 100U + grace; + require(!bridge.serviceNetworkPower(clock_ms), "failed services kept WiFi on"); + require(bridge._network_suspended && mesh::wifi::networkPower().backgroundSuspended(), + "worker did not hand off its transport suspension"); + require(events == std::vector({"sntp-stop", "tls-stop", "tls-stop", + "tls-stop", "wifi-disconnect", "wifi-off"}), + "WiFi was disabled before SNTP/TLS teardown"); + require(!bridge._ntp_refresh_pending && !bridge._ntp_refresh_operation.owns() + && !bridge._ntp_sync_pending && !WiFi.reconnect && sdk_mode == WIFI_OFF, + "a failed background service retained radio activity"); + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + require(!bridge.clients[index].started && !bridge._slots[index].connected + && !bridge._slot_attempt_pending[index] && !bridge._status_publish_pending[index], + "suspension retained a pending transport or publish"); + } +} + +static void failedSdkStopWithholdsHandoff() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge.clients[1].fail_stop = true; + require(!bridge.serviceNetworkPower(100U + grace), + "failed SDK stop permitted more background TLS or NTP work"); + require(!bridge._network_suspended && !mesh::wifi::networkPower().backgroundSuspended() + && sdk_mode == WIFI_STA && WiFi.disconnects == 0 && WiFi.offs == 0, + "station owner was allowed to stop a still-running TLS client"); + require(!bridge.clients[0].started && bridge.clients[1].started + && bridge.clients[2].started, "teardown did not stop at failed client"); + bridge.clients[1].fail_stop = false; + require(!bridge.serviceNetworkPower(101U + grace), + "retry after successful SDK stop failed to suspend"); + require(!bridge.clients[1].started && !bridge.clients[2].started && WiFi.offs == 1, + "successful retry leaked a transport or radio"); +} + +static void recoveryHasOneBoundedWindow() { + for (uint32_t start : {100U, UINT32_MAX - 10000U}) { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(start), "initial network did not run"); + const uint32_t suspended = start + grace; + require(!bridge.serviceNetworkPower(suspended), "network did not suspend"); + for (uint32_t offset = 1000U; offset < interval; offset += 1000U) { + require(!bridge.serviceNetworkPower(suspended + offset), + "a failed timer woke WiFi during the cooldown"); + } + require(bridge.initializes == 0, "cooldown restarted station"); + const uint32_t resumed = suspended + interval; + require(bridge.serviceNetworkPower(resumed), "30-minute recovery was not started"); + require(bridge.initializes == 1 && !bridge._network_suspended + && !mesh::wifi::networkPower().backgroundSuspended() + && sdk_mode == WIFI_STA, "recovery did not release station ownership"); + for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) { + require(!bridge._slots[index].circuit_breaker_tripped + && MQTTConnectionPolicy::reconnectDue(resumed, + bridge._slots[index].last_reconnect_attempt, + bridge._slots[index].reconnect_backoff, static_cast(index)), + "recovery left brokers waiting behind the old breaker"); + } + require(bridge._ntp_sync_pending && bridge._ntp_refresh_retry_at == 0, + "recovery did not admit one new NTP attempt"); + for (uint32_t offset = 1000U; offset < probe; offset += 1000U) { + require(bridge.serviceNetworkPower(resumed + offset), "probe ended early"); + } + require(bridge.initializes == 1, "probe restarted station repeatedly"); + require(!bridge.serviceNetworkPower(resumed + probe), "failed probe was unbounded"); + require(!bridge.serviceNetworkPower(resumed + probe + interval - 1U), + "failed probe did not start a fresh 30-minute cooldown"); + } +} + +static void foregroundAndEspNowArePreserved() { + for (int owner = 0; owner < 6; ++owner) { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + if (owner == 0) mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + if (owner == 1) bridge.board.ota = true; + if (owner == 2) sdk_mode = WIFI_STA | WIFI_MODE_AP; + if (owner == 3) bridge._ntp_force_requested = true; + if (owner == 4) bridge._ntp_diag_requested = true; + if (owner == 5) bridge.snmp.running = true; + require(bridge.serviceNetworkPower(100U + grace), "foreground owner lost radio"); + require(WiFi.disconnects == 0 && WiFi.offs == 0, + "background outage disrupted a foreground owner"); + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + require(!bridge.serviceNetworkPower(100U + grace), "failed STA did not suspend"); + require(!WiFi.reconnect && WiFi.disconnects == 1 && WiFi.offs == 0 + && sdk_mode == WIFI_STA && channel_restores == 1, + "suspending failed internet disabled independent ESP-NOW radio"); + } + // An AP can take ownership during the SDK's yielding client stop calls. + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + on_client_stop = [] { sdk_mode = WIFI_STA | WIFI_MODE_AP; }; + require(!bridge.serviceNetworkPower(100U + grace), "worker did not suspend transports"); + require(WiFi.disconnects == 0 && WiFi.offs == 0 && (sdk_mode & WIFI_MODE_AP), + "foreground AP race lost the shared WiFi driver"); + } +} + +static void automaticClockEstimateCannotWakeSuspendedNetwork() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._ntp_estimate_requested = true; + require(!bridge.serviceNetworkPower(100U + grace), + "automatic clock estimate bypassed combined outage grace"); + require(!bridge.serviceNetworkPower(100U + grace + 1U), + "automatic clock estimate woke failed network immediately"); + require(!bridge.serviceNetworkPower(100U + grace + interval - 1U), + "automatic clock estimate bypassed the 30-minute cooldown"); + require(bridge.initializes == 0, "background clock request restarted station"); + bridge._ntp_estimate_requested = false; + WiFi.state = WL_CONNECTED; // Even a stale connected facade cannot bypass it. + require(!bridge.requestNtpTimeEstimate() && !bridge._ntp_estimate_requested, + "automatic clock bootstrap queued new work during suspension"); + require(bridge.serviceNetworkPower(100U + grace + interval), + "scheduled recovery did not admit clock work"); + require(bridge.requestNtpTimeEstimate() && bridge._ntp_estimate_requested, + "scheduled recovery still blocked automatic clock bootstrap"); +} + +static void deferredEspNowChannelRepair() { + for (uint32_t start : {100U, UINT32_MAX - 500U, UINT32_MAX - 999U}) { + resetEnvironment(); + MQTTBridge bridge; + espnow_constrained = true; + clock_ms = start - grace; + bridge.serviceNetworkPower(clock_ms); + clock_ms = start; + channel_restore_result = ESP_FAIL; + require(!bridge.serviceNetworkPower(clock_ms), "failed STA did not suspend"); + require(bridge._network_espnow_restore_pending && channel_restores == 1 + && WiFi.disconnects == 1 && WiFi.offs == 0, + "failed channel write was not retained independently of WiFi stop"); + for (uint32_t offset : {100U, 250U, 500U, 999U}) { + clock_ms = start + offset; + require(!bridge.serviceNetworkPower(clock_ms), "channel repair resumed failed internet"); + require(channel_restores == 1 && WiFi.disconnects == 1, + "channel repair was unthrottled or repeated disconnect"); + } + clock_ms = start + 1000U; + channel_restore_result = ESP_OK; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 2 && !bridge._network_espnow_restore_pending + && WiFi.disconnects == 1 && WiFi.offs == 0, + "transient channel failure did not recover without toggling WiFi"); + for (uint32_t offset : {1100U, 2000U, 5000U}) { + clock_ms = start + offset; + require(!bridge.serviceNetworkPower(clock_ms), "repair canceled the outage cooldown"); + require(channel_restores == 2 && WiFi.disconnects == 1, + "successful channel restoration retained repeated work"); + } + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + clock_ms = 100U + grace; + channel_restore_result = ESP_FAIL; + bridge.serviceNetworkPower(clock_ms); + clock_ms += 1000U; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + bridge.serviceEspNowChannelRestore(clock_ms); + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + sdk_mode = WIFI_STA | WIFI_MODE_AP; + bridge.serviceEspNowChannelRestore(clock_ms); + sdk_mode = WIFI_STA; + bridge.board.ota = true; + bridge.serviceEspNowChannelRestore(clock_ms); + bridge.board.ota = false; + sdk_mode_result = ESP_FAIL; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 1 && bridge._network_espnow_restore_pending, + "deferred restoration changed a foreground or unknown driver channel"); + sdk_mode_result = ESP_OK; + channel_restore_result = ESP_OK; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 2 && !bridge._network_espnow_restore_pending, + "released foreground owner stranded channel restoration"); + } + resetEnvironment(); + { + MQTTBridge bridge; + espnow_constrained = true; + bridge.serviceNetworkPower(100U); + clock_ms = 100U + grace; + channel_restore_result = ESP_FAIL; + bridge.serviceNetworkPower(clock_ms); + espnow_constrained = false; + clock_ms += 1000U; + bridge.serviceEspNowChannelRestore(clock_ms); + require(channel_restores == 1 && !bridge._network_espnow_restore_pending, + "stopped ESP-NOW owner retained a channel mutation"); + } +} + +static void stableBrokerCountsButFlappingDoesNot() { + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 101U; + require(bridge.serviceNetworkPower(30101U), "30-second quiet broker was not useful"); + require(bridge.serviceNetworkPower(100U + 10U * grace), "working broker lost WiFi"); + require(WiFi.offs == 0, "NTP failure overrode working MQTT"); + const uint32_t failed_at = 100U + 10U * grace; + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(failed_at + grace - 1U), + "real broker success did not renew outage grace"); + require(!bridge.serviceNetworkPower(failed_at + grace), + "disconnected formerly healthy broker kept WiFi indefinitely"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + for (uint32_t offset = 1000U; offset < grace; offset += 1000U) { + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 100U + offset - 1U; + require(bridge.serviceNetworkPower(100U + offset), "grace ended early"); + } + bridge._slots[0].connected_at_ms = 100U + grace - 1U; + require(!bridge.serviceNetworkPower(100U + grace), + "flapping broker repeatedly renewed the background outage"); + } +} + +static void publishAcknowledgmentProvesRecoveryImmediately() { + resetEnvironment(); + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + require(!bridge.serviceNetworkPower(100U + grace), "initial outage did not suspend"); + const uint32_t resumed = 100U + grace + interval; + require(bridge.serviceNetworkPower(resumed), "recovery probe did not begin"); + // A broker reply during the final second proves useful service immediately, + // even though it has not met the quiet 30-second connection threshold. + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = resumed + probe - 1U; + bridge.clients[0].publish_acks = 1U; + require(bridge.serviceNetworkPower(resumed + probe - 1U), + "PUBACK failed to prove useful network"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(resumed + probe), + "successful recovery retained its old short probe deadline"); + require(bridge.serviceNetworkPower(resumed + probe - 1U + grace - 1U), + "PUBACK failed to renew a normal outage window"); + require(!bridge.serviceNetworkPower(resumed + probe - 1U + grace), + "an old PUBACK repeatedly renewed a disconnected broker"); +} + +static void quietConnectionHasExactThirtySecondHealthBoundary() { + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 200U; + require(bridge.serviceNetworkPower(30199U), "initial grace ended early"); + bridge._slots[0].connected = false; + require(!bridge.serviceNetworkPower(100U + grace), + "29.999-second connection falsely renewed outage grace"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 200U; + require(bridge.serviceNetworkPower(30200U), "30-second connection was not admitted"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(30200U + grace - 1U), + "quiet 30-second broker did not renew useful-service grace"); + require(!bridge.serviceNetworkPower(30200U + grace), + "formerly quiet broker kept failed service awake indefinitely"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(0U); + bridge._slots[0].connected = true; + bridge._slots[0].connected_at_ms = 0U; + require(bridge.serviceNetworkPower(30000U), "zero-time connection was not admitted"); + bridge._slots[0].connected = false; + require(bridge.serviceNetworkPower(30000U + grace - 1U), + "valid zero-time broker connection was mistaken for unset state"); + require(!bridge.serviceNetworkPower(30000U + grace), + "disconnected zero-time broker retained WiFi indefinitely"); + } +} + +static void everyNtpServerGetsTheLeadingTimeoutSlot() { + resetEnvironment(); + MQTTBridge bridge; + const char* expected[] = {"custom", "pool", "google", "cloudflare", "aws", "nist", "custom"}; + for (const char* primary : expected) { + bridge.refreshNTP(); + require(ntp_primaries.back() == primary, + "a blackholed leading server starved configured NTP fallback servers"); + clock_ms += 15000U; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending, + "failed rotating NTP request exceeded its bounded timeout"); + clock_ms = bridge._ntp_refresh_retry_at; + } + require(ntp_configs == 7, "NTP rotation skipped or repeated a timeout slot"); +} + +static void sntpFailureAndBusyLeaseRetries() { + for (uint32_t start : {100U, UINT32_MAX - 4000U, UINT32_MAX - interval + 1U}) { + resetEnvironment(); + MQTTBridge bridge; + clock_ms = start; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, "boot NTP did not start"); + clock_ms += 15000U; + bridge.serviceBackgroundNtp(clock_ms); + const uint32_t failed_at = clock_ms; + const uint32_t retry = mqttNtpFailureRetryAt(failed_at); + require(bridge._ntp_refresh_retry_at == retry && !bridge._ntp_refresh_pending, + "failed SNTP kept running or lost its bounded retry"); + clock_ms = failed_at + interval - 1U; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1, "failed SNTP retried before 30 minutes"); + clock_ms = retry; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 2 && bridge._ntp_refresh_pending, "SNTP retry never became due"); + } + resetEnvironment(); + { + MQTTBridge bridge; + mesh::sntp_coord::OperationLease foreground_ntp(mesh::sntp_coord::processWideCoordinator()); + require(foreground_ntp.tryAcquire(), "foreground SNTP owner did not acquire"); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 0 && !bridge._ntp_refresh_pending + && bridge._ntp_refresh_retry_at == clock_ms + 5000U, + "SNTP contention was treated as failed internet or busy-waited"); + clock_ms += 4999U; + foreground_ntp.release(); + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 0, "SNTP busy-owner retry was not bounded"); + clock_ms += 1U; + bridge.serviceBackgroundNtp(clock_ms); + require(ntp_configs == 1, "released SNTP owner never admitted retry"); + } +} + +static void retainedClockAndFreshNtpDoNotCreditFailedMqtt() { + resetEnvironment(); + MQTTBridge bridge; + bridge.rtc.epoch = MQTT_NTP_MIN_VALID_EPOCH + 100U; + require(bridge.serviceNetworkPower(100U), "initial policy did not run"); + bridge.serviceBackgroundNtp(100U); + require(bridge._ntp_synced && !bridge._fresh_ntp_this_boot, + "retained RTC became fake proof of fresh internet"); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "retained clock suppressed fresh NTP request"); + system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U; + ntp_status = SNTP_SYNC_STATUS_COMPLETED; + clock_ms = 101U; + bridge.serviceBackgroundNtp(clock_ms); + require(bridge._fresh_ntp_this_boot && bridge._last_ntp_sync == clock_ms + && !bridge._ntp_refresh_pending && ntp_stops == 1, + "fresh NTP did not update the RTC and release SNTP"); + require(bridge.serviceNetworkPower(100U + grace - 1U), "outage grace ended early"); + require(!bridge.serviceNetworkPower(100U + grace), + "successful NTP kept failed MQTT's WiFi on indefinitely"); +} + +int main(int argc, char** argv) { + require(argc == 2, "one case is required"); + const std::string name = argv[1]; + try { + if (name == "handoff") failureStopsBothServicesBeforeWifi(); + else if (name == "failed-stop") failedSdkStopWithholdsHandoff(); + else if (name == "recovery") recoveryHasOneBoundedWindow(); + else if (name == "owners") foregroundAndEspNowArePreserved(); + else if (name == "background-estimate") automaticClockEstimateCannotWakeSuspendedNetwork(); + else if (name == "channel-repair") deferredEspNowChannelRepair(); + else if (name == "healthy") stableBrokerCountsButFlappingDoesNot(); + else if (name == "puback") publishAcknowledgmentProvesRecoveryImmediately(); + else if (name == "quiet-boundary") quietConnectionHasExactThirtySecondHealthBoundary(); + else if (name == "ntp-retry") sntpFailureAndBusyLeaseRetries(); + else if (name == "ntp-rotation") everyNtpServerGetsTheLeadingTimeoutSlot(); + else if (name == "ntp-proof") retainedClockAndFreshNtpDoNotCreditFailedMqtt(); + else return 2; + } catch (const std::exception& error) { + std::cerr << name << ": " << error.what() << '\n'; + return 1; + } + std::cout << "PASS: " << name << '\n'; + return 0; +} +''' + +PORTABLE_MAIN = r''' +int main() { + try { + resetEnvironment(); + MQTTBridge bridge; + clock_ms = UINT32_MAX - 1000U; + bridge.refreshNTP(); + require(ntp_configs == 1 && bridge._ntp_refresh_pending, + "portable async SNTP request did not start"); + require(!bridge._ntp_refresh_operation.owns(), + "portable stack unexpectedly acquired ESP32 SNTP lease"); + clock_ms += 4999U; + bridge.pollNtpRefresh(clock_ms); + require(bridge._ntp_refresh_pending && !bridge._ntp_synced, + "portable stack accepted clock before five-second settlement"); + system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U; + clock_ms += 1U; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending && bridge._ntp_synced + && bridge._fresh_ntp_this_boot && bridge.rtc.epoch == system_epoch + && bridge._last_ntp_sync == clock_ms, + "portable stack lost its existing settled-clock behavior"); + bridge.refreshNTP(); + WiFi.state = WL_DISCONNECTED; + bridge.pollNtpRefresh(clock_ms); + require(!bridge._ntp_refresh_pending + && bridge._ntp_refresh_retry_at == clock_ms + 5000U, + "ESP32 outage policy changed portable WiFi-failure retry"); + require(ntp_stops == 0, "portable SNTP called ESP32 SDK cleanup"); + require(mqttNtpFailureRetryAt(UINT32_MAX - 4999U) == 1U, + "portable retry lost its wrap-safe zero-sentinel handling"); + require(mqttNtpRetryAt(clock_ms) == clock_ms + 5000U, + "portable busy-owner retry unexpectedly changed"); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + std::cout << "PASS: portable\n"; + return 0; +} +''' + + +class MqttWiFiNetworkPowerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + raise RuntimeError("a host C++ compiler is required") + helpers = "\n".join(method(BRIDGE, signature) for signature in ( + "static uint32_t mqttNtpFailureRetryAt(", + "static bool mqttNtpRefreshDue(", + "static uint32_t mqttNtpRetryAt(", + "static void stopMqttSntpOperation(", + )) + methods = "\n".join(method(BRIDGE, signature) for signature in ( + "bool MQTTBridge::serviceNetworkPower(", + "void MQTTBridge::serviceEspNowChannelRestore(", + "void MQTTBridge::serviceBackgroundNtp(", + "void MQTTBridge::cancelNtpRefresh(", + "void MQTTBridge::refreshNTP(", + "void MQTTBridge::pollNtpRefresh(", + "bool MQTTBridge::requestNtpTimeEstimate(", + )) + cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-mqtt-network-power-") + cls.addClassCleanup(cls.temporary.cleanup) + directory = Path(cls.temporary.name) + cpp = directory / "power.cpp" + source = PREAMBLE.replace("@HELPERS@", helpers).replace("@METHODS@", methods) + cpp.write_text(source, encoding="ascii") + cls.binary = directory / "power" + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(cpp), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + # The portable stack keeps its previous five-second settle and retry; + # the new 30-minute failed-network policy is ESP32-only. + portable_methods = "\n".join(method(BRIDGE, signature) for signature in ( + "void MQTTBridge::cancelNtpRefresh(", + "void MQTTBridge::refreshNTP(", + "void MQTTBridge::pollNtpRefresh(", + )) + portable_source = PREAMBLE[:PREAMBLE.index("static constexpr uint32_t grace")] + portable_source = (portable_source.replace("#define ESP_PLATFORM 1", "") + .replace("@HELPERS@", helpers) + .replace("@METHODS@", portable_methods) + PORTABLE_MAIN) + portable_cpp = directory / "portable.cpp" + portable_cpp.write_text(portable_source, encoding="ascii") + cls.portable_binary = directory / "portable" + built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(portable_cpp), + "-o", str(cls.portable_binary)], + capture_output=True, text=True, timeout=60) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + + def run_case(self, name): + checked = subprocess.run([str(self.binary), name], capture_output=True, + text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("PASS: " + name, checked.stdout) + + def test_failed_ntp_and_mqtt_quiesce_before_wifi_shutdown_publication(self): + self.run_case("handoff") + + def test_failed_sdk_transport_stop_keeps_station_owned_and_awake(self): + self.run_case("failed-stop") + + def test_recovery_restarts_once_and_stops_after_bounded_probe_across_wrap(self): + self.run_case("recovery") + + def test_foreground_and_espnow_owners_preserve_driver_even_during_stop_race(self): + self.run_case("owners") + + def test_automatic_clock_estimate_cannot_bypass_background_outage_cooldown(self): + self.run_case("background-estimate") + + def test_failed_espnow_channel_restore_retries_without_wifi_toggles_or_owner_conflicts(self): + self.run_case("channel-repair") + + def test_stable_mqtt_counts_as_useful_and_flapping_mqtt_cannot_renew(self): + self.run_case("healthy") + + def test_publish_ack_proves_recovery_immediately_and_is_consumed_once(self): + self.run_case("puback") + + def test_quiet_broker_health_requires_exactly_thirty_seconds(self): + self.run_case("quiet-boundary") + + def test_sntp_failure_and_busy_lease_have_different_bounded_retries(self): + self.run_case("ntp-retry") + + def test_all_six_ntp_servers_lead_a_bounded_request_before_rotation_repeats(self): + self.run_case("ntp-rotation") + + def test_retained_clock_or_fresh_ntp_cannot_keep_failed_mqtt_wifi_active(self): + self.run_case("ntp-proof") + + def test_portable_non_esp_stack_preserves_settle_and_short_retry_policy(self): + checked = subprocess.run([str(self.portable_binary)], capture_output=True, + text=True, timeout=10) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("PASS: portable", checked.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_room_web_http.py b/test/test_room_web_http.py index a35e5694f..e3d944915 100644 --- a/test/test_room_web_http.py +++ b/test/test_room_web_http.py @@ -28,6 +28,7 @@ HARNESS = r''' #include #include #include +#include using String = std::string; #define PROGMEM @HTML@ @@ -153,9 +154,15 @@ struct WebConfigServer { static constexpr size_t MAX_BODY = @MAX_BODY@; static constexpr uint32_t STOP_WARN_MS = WebConfigBatch::kStopWarnMs; enum Mode { MODE_OFF, MODE_SETUP, MODE_LAN }; + enum { BATCH_IDLE, BATCH_PENDING }; Mode _mode = MODE_LAN; + unsigned _batch_state = BATCH_IDLE; + bool _setup_wifi_handoff_pending = false; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; bool _stopping = false, _stop_warned = false, _board_cmds_probed = true; uint32_t _handler_refs = 0, _stop_warn_at = 0, _last_activity = 0; + uint32_t _network_activity_until = 0; std::mutex mutex; SemaphoreHandle_t _mux = &mutex; mesh::RoomWebMailbox* _room_mailbox = nullptr; @@ -170,11 +177,16 @@ struct WebConfigServer { if (allocate) _room_mailbox = new mesh::RoomWebMailbox; registerRoutes(); _active = this; } - ~WebConfigServer() { if (_active == this) _active = nullptr; delete _room_mailbox; } + ~WebConfigServer() { + if (_active == this) _active = nullptr; + delete _room_mailbox; + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + } void finalizeTeardown() { // Dependency stub: production tick decides when reclaim is safe. assert(_handler_refs == 0); ++finalizations; delete _room_mailbox; _room_mailbox = nullptr; _server = nullptr; _stopping = false; _cb->onWebConfigStopped(); + _network_activity_until = 0; refreshNetworkPowerHold(millis()); } void probeBoardCommands() { _board_cmds_probed = true; } void serviceTerminal(uint32_t) {} @@ -183,6 +195,9 @@ struct WebConfigServer { void requestStop(); void detachRoutes(); uint32_t handlerRefCount() const; + void noteNetworkActivity(); + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); void registerRoutes(); static void dispatchRequest(AsyncWebServerRequest*, RequestHandler); static void collectBody(AsyncWebServerRequest*, uint8_t*, size_t, size_t, size_t); @@ -342,6 +357,8 @@ static void boundsAndFloors() { static void pagesAndAvailability() { Callbacks cb; WebConfigServer server(cb); AsyncWebServerRequest page; server.host.invoke("/room", HTTP_GET, page); assert(page.status == 200); + assert(server.isNetworkForegroundActive(ticks)); + assert(mesh::wifi::networkPower().foregroundActive()); assert(page.reply.find("MeshCore Room") != String::npos && page.response_type == "text/html; charset=utf-8"); assert(page.response_headers["Cache-Control"] == "no-store"); assert(page.response_headers["X-Content-Type-Options"] == "nosniff" && page.response_headers["Referrer-Policy"] == "no-referrer"); @@ -360,12 +377,14 @@ static void lifecycle() { assert(server.handlerRefCount() == 1); server.requestStop(); server.tick(ticks); assert(server._stopping && server._room_mailbox && server.finalizations == 0); + assert(mesh::wifi::networkPower().foregroundActive()); assert(cb.calls == 0); }; submit(server, pending); assert(pending.status == 202 && server.host.ended); assert(server.handlerRefCount() == 0 && server._stopping && cb.calls == 0); AsyncWebServerRequest unavailable; identity(unavailable); result(server, unavailable); assert(unavailable.status == 503); server.tick(ticks); assert(!server._room_mailbox && !server._stopping && cb.stops == 1 && server.finalizations == 1); + assert(!mesh::wifi::networkPower().foregroundActive()); assert(cb.calls == 0); // Pending work is canceled, never executed after stop. Callbacks off_cb; WebConfigServer off(off_cb); AsyncWebServerRequest post; identity(post); body(post, "{}"); submit(off, post); off._mode = WebConfigServer::MODE_OFF; off.tick(ticks); assert(off_cb.calls == 0); @@ -435,6 +454,9 @@ class RoomWebHttpTests(unittest.TestCase): "void WebConfigServer::handleRoomResult(", "void WebConfigServer::collectBody(", "void WebConfigServer::dispatchRequest(", "void WebConfigServer::detachRoutes(", "uint32_t WebConfigServer::handlerRefCount(", "void WebConfigServer::requestStop(", + "void WebConfigServer::noteNetworkActivity(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(", "void WebConfigServer::registerRoutes(")] tick = extract_braced(source, "void WebConfigServer::tick(") # Execute the exact stop/OFF guards and loop service calls; SDK Wi-Fi, diff --git a/test/test_rs232_mqtt_runtime.py b/test/test_rs232_mqtt_runtime.py index 8168a6c90..c56f49425 100644 --- a/test/test_rs232_mqtt_runtime.py +++ b/test/test_rs232_mqtt_runtime.py @@ -187,9 +187,14 @@ class RS232MQTTRuntimeTests(unittest.TestCase): MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh); expect_get(cli, "bridge.type", "> mqtt+rs232"); assert(mesh.isMqttBridgeRunning() && !mesh.isRs232BridgeRunning()); + mesh.mqtt_bridge->setNetworkSuspended(true); + assert(mesh.isMqttBridgeRunning() && mesh._prefs.bridge_enabled); + assert(!mesh.bridgesPreventSleep()); char reply[256] = {}; cli.set("rs232.enabled on", reply); assert(strcmp(reply, "OK") == 0); assert(mesh.isRs232BridgeRunning() && mesh.isMqttBridgeRunning()); + assert(mesh.bridgesPreventSleep()); // Paused MQTT cannot mask an active UART. + mesh.mqtt_bridge->setNetworkSuspended(false); const unsigned starts = mesh.mqtt_bridge->begins; cli.set("bridge.baud 57600", reply); assert(strcmp(reply, "OK") == 0); assert(mesh.isRs232BridgeRunning() && mesh.mqtt_bridge->begins == starts); diff --git a/test/test_tls_download_clock_gates.py b/test/test_tls_download_clock_gates.py index d73a9f850..76e36fdd9 100644 --- a/test/test_tls_download_clock_gates.py +++ b/test/test_tls_download_clock_gates.py @@ -190,7 +190,9 @@ class TlsDownloadClockGateTest(unittest.TestCase): ) self.assertIn("SntpOperationCoordinator.h", implementation) - self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 2) + # Background refresh has one ESP32 three-server branch and one portable + # single-server branch; the explicit administrator sync is separate. + self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 3) self.assertEqual( len(re.findall(r"(?m)^\s*sntp_set_sync_status\(", implementation)), 2, @@ -321,10 +323,15 @@ class TlsDownloadClockGateTest(unittest.TestCase): header, ) self.assertIn("MQTT_NTP_REFRESH_TIMEOUT_MS = 15000UL", implementation) - self.assertEqual(implementation.count("pollNtpRefresh("), 3) - self.assertGreaterEqual( - implementation.count("&& !_ntp_refresh_pending"), 2 + self.assertEqual(implementation.count("pollNtpRefresh("), 4) + background = section( + implementation, + "void MQTTBridge::serviceBackgroundNtp(", + "void MQTTBridge::mqttTaskLoop()", ) + self.assertIn("if (_ntp_refresh_pending) return;", background) + self.assertLess(background.index("if (_ntp_refresh_pending) return;"), + background.index("refreshNTP();")) reset = refresh.index("sntp_set_sync_status(SNTP_SYNC_STATUS_RESET)") configure = refresh.index( @@ -351,7 +358,7 @@ class TlsDownloadClockGateTest(unittest.TestCase): self.assertIn("_ntp_refresh_operation.owns()", poll) timeout = poll.index("elapsed >= MQTT_NTP_REFRESH_TIMEOUT_MS") - retry = poll.index("mqttNtpRetryAt(now)", timeout) + retry = poll.index("mqttNtpFailureRetryAt(now)", timeout) self.assertLess(timeout, retry) self.assertNotIn("_last_ntp_sync =", poll[timeout:]) diff --git a/test/test_webconfig_network_power.py b/test/test_webconfig_network_power.py new file mode 100644 index 000000000..1e01cf41f --- /dev/null +++ b/test/test_webconfig_network_power.py @@ -0,0 +1,151 @@ +"""Exercise the production portal's bounded foreground ownership policy.""" +from pathlib import Path +import unittest + +from test_replay_reset_integration import extract_braced +import test_wifi_ota_start as wifi_start + +ROOT = Path(__file__).resolve().parents[1] + + +class WebConfigNetworkPowerTest(unittest.TestCase): + compile_and_run = wifi_start.WiFiOtaStartTest.compile_and_run + + def fixture(self): + source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() + methods = "\n".join(extract_braced(source, signature) for signature in ( + "void WebConfigServer::noteNetworkActivity(", + "void WebConfigServer::requestNetworkAccess(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(")) + return r''' +#include +#include +#include +#include +#include +uint32_t now = 1000; +uint32_t millis() { return now; } +struct WCLock { explicit WCLock(void*) {} }; +struct WebConfigServer { + enum { MODE_OFF, MODE_SETUP, MODE_CONNECTING, MODE_LAN }; + enum { BATCH_IDLE, BATCH_PENDING, BATCH_DONE }; + int _mode = MODE_LAN, _batch_state = BATCH_IDLE; + bool _stopping = false, _setup_wifi_handoff_pending = false; + uint32_t _last_activity = 1000, _network_activity_until = 0; + unsigned _handler_refs = 0; + void* _mux = nullptr; + struct Terminal { bool running = false; struct { bool pending = false; } queue; }; + Terminal* _terminal = nullptr; + mesh::RoomWebMailbox* _room_mailbox = nullptr; + unsigned handlerRefCount() const { return _handler_refs; } + void noteNetworkActivity(); + void requestNetworkAccess(); + bool isNetworkForegroundActive(uint32_t) const; + void refreshNetworkPowerHold(uint32_t); +}; +@METHODS@ +int main() { + WebConfigServer portal; + auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + // Boot/connection timestamps and a listener alone never hold failed WiFi. + portal._last_activity = now; + assert(!portal.isNetworkForegroundActive(now)); + portal.refreshNetworkPowerHold(now); + assert(!power.foregroundActive()); + portal.requestNetworkAccess(); + assert(power.consumeProbeRequest() && power.foregroundActive()); + now += 119999; + assert(portal.isNetworkForegroundActive(now)); + ++now; portal.refreshNetworkPowerHold(now); + assert(!portal.isNetworkForegroundActive(now) && !power.foregroundActive()); + // The bounded lease works across millis() rollover. + now = UINT32_MAX - 100; + portal.noteNetworkActivity(); + now += 1000; + assert(portal.isNetworkForegroundActive(now)); + now += 119000; + assert(!portal.isNetworkForegroundActive(now)); + // Setup, teardown and in-flight handlers/batches survive lease expiry. + portal._mode = WebConfigServer::MODE_SETUP; + assert(portal.isNetworkForegroundActive(now)); + portal._mode = WebConfigServer::MODE_LAN; + portal._handler_refs = 1; + assert(portal.isNetworkForegroundActive(now)); + portal._handler_refs = 0; portal._batch_state = WebConfigServer::BATCH_PENDING; + assert(portal.isNetworkForegroundActive(now)); + portal._batch_state = WebConfigServer::BATCH_DONE; + assert(!portal.isNetworkForegroundActive(now)); + portal._setup_wifi_handoff_pending = true; + assert(portal.isNetworkForegroundActive(now)); + portal._setup_wifi_handoff_pending = false; + WebConfigServer::Terminal terminal; + portal._terminal = &terminal; terminal.running = true; + assert(portal.isNetworkForegroundActive(now)); + terminal.running = false; terminal.queue.pending = true; + assert(portal.isNetworkForegroundActive(now)); + terminal.queue.pending = false; + assert(!portal.isNetworkForegroundActive(now)); + mesh::RoomWebMailbox mailbox; uint8_t token[32] = {1}, digest[32] = {2}; + portal._room_mailbox = &mailbox; + assert(mailbox.submit(token, 1, digest, "{}", 2, now) + == mesh::RoomWebMailbox::Result::Accepted); + assert(portal.isNetworkForegroundActive(now)); + assert(mailbox.begin() && portal.isNetworkForegroundActive(now)); + mailbox.finish(now); + assert(!portal.isNetworkForegroundActive(now)); + portal._mode = WebConfigServer::MODE_OFF; portal._stopping = true; + assert(portal.isNetworkForegroundActive(now)); + portal._stopping = false; + portal.refreshNetworkPowerHold(now); + assert(!power.foregroundActive()); + power.setManaged(false); +} +'''.replace("@METHODS@", methods) + + def test_actual_activity_setup_and_inflight_work_hold_with_bounded_idle(self): + self.compile_and_run(self.fixture()) + + def test_status_reports_shared_cooldown_before_connecting_or_off(self): + source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() + formatter = extract_braced(source, "bool WebConfigServer::formatWiFiStatus(") + block = extract_braced(formatter, + " if (mesh::wifi::networkPower().backgroundSuspended()") + self.assertLess(formatter.index(block), + formatter.index("active->_mode == MODE_CONNECTING")) + fixture = r''' +#include +#include +#include +#include +void appendOtaSeederStatus(char* reply, size_t capacity) { + size_t used = strlen(reply); + snprintf(reply + used, capacity - used, "; seeder status"); +} +bool format(char* reply, size_t reply_len, bool configured) { + const char* ssid = "working-network"; +@BLOCK@ + snprintf(reply, reply_len, "> connecting"); + return true; +} +int main() { + char reply[160]; auto& power = mesh::wifi::networkPower(); + power.setManaged(true); power.setSuspended(true); + assert(format(reply, sizeof(reply), true)); + assert(strcmp(reply, "> suspended; recovery every 30 minutes, SSID: working-network; seeder status") == 0); + assert(format(reply, sizeof(reply), false)); + assert(strstr(reply, "SSID: (not configured); seeder status")); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, true); + format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0); + power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false); + power.setSuspended(false); + format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0); + power.setManaged(false); +} +'''.replace("@BLOCK@", block) + self.compile_and_run(fixture) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_webconfig_recovery_auth.py b/test/test_webconfig_recovery_auth.py index 0651d7b77..092de93e1 100644 --- a/test/test_webconfig_recovery_auth.py +++ b/test/test_webconfig_recovery_auth.py @@ -54,7 +54,7 @@ class WebConfigRecoveryAuthTest(unittest.TestCase): def test_old_interface_only_fallback_fails_executable_regression(self): fixture = self.fixture() - old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) return true;", "return true;", 1) + old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) { noteNetworkActivity(); return true; }", "return true;", 1) self.assertNotEqual(old, fixture) self.compile_run(old, expect_rejection=True) diff --git a/test/test_webconfig_station_ownership.py b/test/test_webconfig_station_ownership.py index ef0125815..f5cf0ea39 100644 --- a/test/test_webconfig_station_ownership.py +++ b/test/test_webconfig_station_ownership.py @@ -87,6 +87,10 @@ class WebConfigStationOwnershipTest(unittest.TestCase): def test_non_mqtt_portal_retains_saved_station_recovery(self): self.compile_and_run(self.station_fixture()) + def test_shared_outage_blocks_idle_portal_reconnect_and_allows_foreground(self): + self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1", + "-DTEST_BACKGROUND_SUSPEND=1") + def test_manual_companion_start_preserves_legacy_join_and_15s_fallback(self): self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1", "-DTEST_LEGACY_COMPANION=1") diff --git a/test/test_wifi_network_power_policy.py b/test/test_wifi_network_power_policy.py new file mode 100644 index 000000000..b3b2f7e51 --- /dev/null +++ b/test/test_wifi_network_power_policy.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Run the shared WiFi outage policy against deterministic clocks and owners. + +The production header is compiled directly, without Arduino or PlatformIO. +Independent translation units and concurrent service holds exercise the actual +handoff used between the network worker and foreground service owners. +""" + +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include + +using mesh::wifi::NetworkPowerControl; +using mesh::wifi::NetworkPowerHold; +using mesh::wifi::NetworkPowerPolicy; +using mesh::wifi::networkPower; +using Action = NetworkPowerPolicy::Action; + +extern NetworkPowerControl* otherControl(); +extern void otherSetWebHold(bool value); +extern bool otherSuspended(); + +static void require(bool value, const char* reason) { + if (!value) throw std::runtime_error(reason); +} + +static void expect(NetworkPowerPolicy& policy, uint32_t now, + bool useful, bool foreground, Action expected, + bool probe = false) { + require(policy.update(now, useful, foreground, probe) == expected, + "unexpected network power transition"); + require(policy.suspended() == (expected == Action::Suspend), + "transition and suspension state disagree"); +} + +// Keep expected timings independent of the production constants. Altering a +// timeout must be an intentional policy change, rather than a self-passing test. +static const uint32_t grace = 300000U; +static const uint32_t probe_budget = 120000U; +static const uint32_t cooldown = 1800000U; + +static void graceBoundary() { + for (uint32_t start : {0U, 1234U, UINT32_MAX - 10000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + expect(policy, start + grace - 1U, false, false, Action::Run); + expect(policy, start + grace, false, false, Action::Suspend); + expect(policy, start + grace + 1U, false, false, Action::Suspend); + } +} + +static void failedServicesCannotRenew() { + // Serial event stream: associations succeed, SNTP alternates success/failure, + // and MQTT handshakes keep failing. None means the configured broker works. + enum Event { GotIp, NtpTimeout, NtpReply, MqttConnectFailure, MqttRetry }; + const Event events[] = {GotIp, NtpTimeout, MqttConnectFailure, + NtpReply, MqttRetry}; + for (uint32_t start : {0U, UINT32_MAX - 70000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + for (uint32_t offset = 1000U; offset < grace; offset += 1000U) { + const Event event = events[(offset / 1000U) % 5U]; + require(event == GotIp || event == NtpTimeout || event == NtpReply + || event == MqttConnectFailure || event == MqttRetry, + "invalid simulated network event"); + expect(policy, start + offset, false, false, Action::Run); + } + expect(policy, start + grace, false, false, Action::Suspend); + // Both services keep wanting retries. Their failure notifications cannot + // restart either the grace period or the 30-minute suspension clock. + for (uint32_t offset = 1000U; offset < cooldown; offset += 1000U) { + expect(policy, start + grace + offset, false, false, Action::Suspend); + } + expect(policy, start + grace + cooldown, false, false, Action::Resume); + } +} + +static void successfulBrokerRenews() { + for (uint32_t start : {0U, UINT32_MAX - 200000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + expect(policy, start + grace - 1U, true, false, Action::Run); + const uint32_t last_healthy = start + 2U * grace; + expect(policy, last_healthy, true, false, Action::Run); + expect(policy, last_healthy + grace - 1U, false, false, Action::Run); + expect(policy, last_healthy + grace, false, false, Action::Suspend); + // A genuine healthy connection observed during suspension cancels it and + // receives the normal grace budget, rather than a short recovery probe. + const uint32_t recovered = last_healthy + grace + 20U; + expect(policy, recovered, true, false, Action::Resume); + expect(policy, recovered + probe_budget, false, false, Action::Run); + expect(policy, recovered + grace, false, false, Action::Suspend); + } +} + +static void boundedRecoveryCycles() { + for (uint32_t start : {0U, UINT32_MAX - 10000U}) { + NetworkPowerPolicy policy; + expect(policy, start, false, false, Action::Run); + uint32_t suspended_at = start + grace; + expect(policy, suspended_at, false, false, Action::Suspend); + for (unsigned cycle = 0; cycle < 20; ++cycle) { + expect(policy, suspended_at + cooldown - 1U, false, false, + Action::Suspend); + const uint32_t resumed_at = suspended_at + cooldown; + expect(policy, resumed_at, false, false, Action::Resume); + for (uint32_t offset = 1000U; offset < probe_budget; offset += 1000U) { + expect(policy, resumed_at + offset, false, false, Action::Run); + } + expect(policy, resumed_at + probe_budget - 1U, false, false, + Action::Run); + suspended_at = resumed_at + probe_budget; + expect(policy, suspended_at, false, false, Action::Suspend); + } + } +} + +static void foregroundPreservesAccess() { + NetworkPowerPolicy policy; + expect(policy, 0, false, true, Action::Run); + expect(policy, grace, false, true, Action::Run); + expect(policy, 3U * grace, false, true, Action::Run); + // Closing the foreground session does not grant a new background grace. + expect(policy, 3U * grace + 1U, false, false, Action::Suspend); + expect(policy, 3U * grace + 2U, false, true, Action::Resume); + expect(policy, 3U * grace + 2U + probe_budget, false, true, Action::Run); + expect(policy, 3U * grace + 3U + probe_budget, false, false, + Action::Suspend); +} + +static void manualProbeOnlyOnDemand() { + NetworkPowerPolicy policy; + expect(policy, 100U, false, false, Action::Run); + expect(policy, 100U + grace, false, false, Action::Suspend); + expect(policy, 101U + grace, false, false, Action::Suspend); + const uint32_t requested_at = 102U + grace; + expect(policy, requested_at, false, false, Action::Resume, true); + // Repeated manual requests while running cannot renew the probe deadline. + expect(policy, requested_at + probe_budget - 1U, false, false, + Action::Run, true); + expect(policy, requested_at + probe_budget, false, false, + Action::Suspend, true); + expect(policy, requested_at + probe_budget + 1U, false, false, + Action::Suspend); + expect(policy, requested_at + probe_budget + cooldown - 1U, false, false, + Action::Suspend); + expect(policy, requested_at + probe_budget + cooldown, false, false, + Action::Resume); +} + +static void usefulProbeBecomesNormalOperation() { + NetworkPowerPolicy policy; + expect(policy, 0, false, false, Action::Run); + expect(policy, grace, false, false, Action::Suspend); + const uint32_t resumed_at = grace + cooldown; + expect(policy, resumed_at, false, false, Action::Resume); + const uint32_t recovered_at = resumed_at + probe_budget - 1U; + expect(policy, recovered_at, true, false, Action::Run); + expect(policy, recovered_at + probe_budget, false, false, Action::Run); + expect(policy, recovered_at + grace - 1U, false, false, Action::Run); + expect(policy, recovered_at + grace, false, false, Action::Suspend); +} + +static void clearControl(NetworkPowerControl& control) { + control.setManaged(false); + control.setHold(NetworkPowerHold::CompanionClient, false); + control.setHold(NetworkPowerHold::WebConfig, false); + control.setHold(NetworkPowerHold::OtaSeeder, false); + control.consumeProbeRequest(); +} + +static void programWideRegistry() { + NetworkPowerControl& control = networkPower(); + clearControl(control); + require(&control == otherControl(), "service translation units split registry"); + control.setManaged(true); + control.setSuspended(true); + require(otherSuspended(), "worker suspension invisible to station owner"); + otherSetWebHold(true); + require(control.foregroundActive(), "foreground hold invisible to worker"); + control.setHold(NetworkPowerHold::OtaSeeder, true); + otherSetWebHold(false); + require(control.foregroundActive(), "closing web session cleared OTA hold"); + control.setHold(NetworkPowerHold::OtaSeeder, false); + require(!control.foregroundActive(), "closed services retained WiFi lease"); + control.requestProbe(); + require(otherControl()->consumeProbeRequest(), "probe invisible across tasks"); + require(!control.consumeProbeRequest(), "probe request was not one-shot"); + control.setManaged(false); + require(!otherSuspended(), "disabled manager retained suspension"); +} + +static void controlLifecycle() { + NetworkPowerControl control; + require(!control.managed() && !control.backgroundSuspended() + && !control.foregroundActive() && !control.consumeProbeRequest(), + "new control has active service state"); + control.setSuspended(true); + require(!control.backgroundSuspended(), "unmanaged owner imposed suspension"); + control.setManaged(true); + require(control.backgroundSuspended(), "managed suspension was lost"); + control.requestProbe(); + control.setManaged(false); + require(!control.backgroundSuspended() && !control.consumeProbeRequest(), + "stopping worker preserved stale suspension or probe"); + control.setHold(NetworkPowerHold::CompanionClient, true); + control.setManaged(true); + require(!control.backgroundSuspended() && control.foregroundActive(), + "restarting manager lost foreground owner or kept stale suspension"); + control.setHold(NetworkPowerHold::CompanionClient, false); + require(!control.foregroundActive(), "last foreground owner failed to release"); +} + +static void concurrentServiceHolds() { + NetworkPowerControl control; + const NetworkPowerHold owners[] = {NetworkPowerHold::CompanionClient, + NetworkPowerHold::WebConfig, + NetworkPowerHold::OtaSeeder}; + for (unsigned round = 0; round < 256; ++round) { + std::atomic ready{0}; + std::atomic go{false}; + std::atomic release[3] = {}; + std::atomic removed[3] = {}; + std::vector threads; + for (unsigned index = 0; index < 3; ++index) { + threads.emplace_back([&, index]() { + ready.fetch_add(1, std::memory_order_release); + while (!go.load(std::memory_order_acquire)) std::this_thread::yield(); + control.setHold(owners[index], true); + ready.fetch_add(1, std::memory_order_release); + while (!release[index].load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + control.setHold(owners[index], false); + removed[index].store(true, std::memory_order_release); + }); + } + while (ready.load(std::memory_order_acquire) != 3U) std::this_thread::yield(); + go.store(true, std::memory_order_release); + while (ready.load(std::memory_order_acquire) != 6U) std::this_thread::yield(); + const bool opens_preserved = control.foregroundActive(); + bool remaining_holds_preserved = true; + for (unsigned index = 0; index < 3; ++index) { + release[index].store(true, std::memory_order_release); + while (!removed[index].load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (index < 2 && !control.foregroundActive()) remaining_holds_preserved = false; + } + for (auto& thread : threads) thread.join(); + require(opens_preserved, "simultaneous opens lost all leases"); + require(remaining_holds_preserved, "one owner cleared another owner's lease"); + require(!control.foregroundActive(), "concurrent closes leaked WiFi hold"); + } +} + +static void serviceLeaseEventSequence() { + NetworkPowerPolicy policy; + NetworkPowerControl control; + control.setManaged(true); + auto update = [&](uint32_t now, bool useful = false) { + const Action action = policy.update(now, useful, control.foregroundActive(), + control.consumeProbeRequest()); + control.setSuspended(action == Action::Suspend); + return action; + }; + require(update(0) == Action::Run, "network failed to start"); + require(update(grace) == Action::Suspend, "outage left WiFi active"); + control.setHold(NetworkPowerHold::WebConfig, true); + require(update(grace + 1U) == Action::Resume, "admin did not wake network"); + control.setHold(NetworkPowerHold::OtaSeeder, true); + control.setHold(NetworkPowerHold::WebConfig, false); + require(update(grace + probe_budget + 1U) == Action::Run, + "closing admin page interrupted active OTA"); + control.setHold(NetworkPowerHold::OtaSeeder, false); + require(update(grace + probe_budget + 2U) == Action::Suspend, + "completed OTA kept failed background network on"); + control.requestProbe(); + require(update(grace + probe_budget + 3U) == Action::Resume, + "explicit retry could not recover network"); + require(!control.consumeProbeRequest(), "owner probe did not get consumed"); + require(update(grace + probe_budget + 4U, true) == Action::Run, + "healthy broker did not resume normal service"); + require(!control.backgroundSuspended(), "healthy broker left station suspended"); +} + +int main(int argc, char** argv) { + if (argc != 2) return 2; + try { + const char* name = argv[1]; + if (!strcmp(name, "grace")) graceBoundary(); + else if (!strcmp(name, "failures")) failedServicesCannotRenew(); + else if (!strcmp(name, "healthy")) successfulBrokerRenews(); + else if (!strcmp(name, "cycles")) boundedRecoveryCycles(); + else if (!strcmp(name, "foreground")) foregroundPreservesAccess(); + else if (!strcmp(name, "manual")) manualProbeOnlyOnDemand(); + else if (!strcmp(name, "recovery")) usefulProbeBecomesNormalOperation(); + else if (!strcmp(name, "registry")) programWideRegistry(); + else if (!strcmp(name, "lifecycle")) controlLifecycle(); + else if (!strcmp(name, "threads")) concurrentServiceHolds(); + else if (!strcmp(name, "leases")) serviceLeaseEventSequence(); + else return 2; + } catch (const std::exception& error) { + std::cerr << argv[1] << ": " << error.what() << '\n'; + return 1; + } + std::cout << "PASS: " << argv[1] << '\n'; + return 0; +} +''' + +OTHER_OWNER = r''' +#include + +mesh::wifi::NetworkPowerControl* otherControl() { + return &mesh::wifi::networkPower(); +} +void otherSetWebHold(bool value) { + mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, value); +} +bool otherSuspended() { + return mesh::wifi::networkPower().backgroundSuspended(); +} +''' + + +class WiFiNetworkPowerPolicyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + raise RuntimeError("a host C++ compiler is required") + cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-wifi-outage-") + cls.addClassCleanup(cls.temporary.cleanup) + work = Path(cls.temporary.name) + source = work / "policy.cpp" + owner = work / "owner.cpp" + source.write_text(HARNESS, encoding="ascii") + owner.write_text(OTHER_OWNER, encoding="ascii") + cls.binary = work / "policy" + built = subprocess.run( + [compiler, "-std=c++11", "-pthread", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(source), str(owner), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60, + ) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + + def run_case(self, name): + result = subprocess.run([str(self.binary), name], capture_output=True, + text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("PASS: " + name, result.stdout) + + def test_initial_five_minute_grace_and_millis_rollover(self): + self.run_case("grace") + + def test_ntp_replies_association_and_mqtt_failures_do_not_renew_outage(self): + self.run_case("failures") + + def test_healthy_broker_renews_grace_and_cancels_suspension(self): + self.run_case("healthy") + + def test_two_minute_probes_and_thirty_minute_cooldowns_repeat_bounded(self): + self.run_case("cycles") + + def test_foreground_access_resumes_and_protects_wifi_until_released(self): + self.run_case("foreground") + + def test_requested_probe_is_explicit_and_does_not_extend_active_probe(self): + self.run_case("manual") + + def test_successful_probe_returns_to_normal_grace(self): + self.run_case("recovery") + + def test_two_translation_units_share_owner_holds_and_probe_requests(self): + self.run_case("registry") + + def test_worker_lifecycle_clears_stale_suspension_and_probe(self): + self.run_case("lifecycle") + + def test_concurrent_foreground_owners_preserve_each_others_holds(self): + self.run_case("threads") + + def test_admin_and_ota_lease_sequence_during_combined_background_outage(self): + self.run_case("leases") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_wifi_ota_start.py b/test/test_wifi_ota_start.py index 67099d349..788783ced 100644 --- a/test/test_wifi_ota_start.py +++ b/test/test_wifi_ota_start.py @@ -343,7 +343,9 @@ int main() { source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text() methods = "\n".join(extract_braced(source, signature) for signature in ( "void WebConfigServer::requestStop(", "bool WebConfigServer::stopForOTA(", - "void WebConfigServer::finalizeTeardown(")) + "void WebConfigServer::finalizeTeardown(", + "bool WebConfigServer::isNetworkForegroundActive(", + "void WebConfigServer::refreshNetworkPowerHold(")) fixture = (ROOT / "test/fixtures/webconfig_ota_handoff.cpp").read_text() self.compile_and_run(fixture.replace("@METHODS@", methods))