diff --git a/docs/research/firmware_memory_budget.md b/docs/research/firmware_memory_budget.md index 9af18d9c2..595e934bc 100644 --- a/docs/research/firmware_memory_budget.md +++ b/docs/research/firmware_memory_budget.md @@ -58,7 +58,8 @@ single allocation. The `mqtt_startup_allowance` component keeps a 24,576-byte internal startup minimum for the default 8,192-byte MQTTBridge worker stack and two active -clients. Each SDK client task defaults to 6,144 bytes and its maximum configured +clients. Each SDK client task is explicitly configured with a 6,144-byte stack +and priority 5 on IDF 4 and 5; its maximum configured receive/transmit buffers total 1,792 bytes. These stacks and buffers total 24,064 bytes for two clients, leaving 512 bytes within the existing allowance. This is a minimum allowance, not a complete allocation bound for the clients, @@ -85,8 +86,16 @@ qualification of five simultaneous PSRAM connections or two internal TLS connections. The `mqtt` report also names the production runtime admission thresholds. -Each attempt requires at least 16 KiB of DMA-capable internal free heap and a -4 KiB contiguous DMA block. A TLS attempt without PSRAM requires at least +A cold SDK start requires at least 32 KiB of DMA-capable internal free heap +and an 8 KiB contiguous DMA block. This includes a separate 16 KiB allowance +for its SDK task stack/TCB, receive/transmit/reassembly buffers and transport +or configuration objects, alongside the 16 KiB AES reserve. Those ordinary +SDK allocations occur after admission and do not use the TLS allocation guard. +A retained SDK handle with a stopped task receives the same cold allowance. +A warm reconnect keeps its running task and buffers; it requires at least +16 KiB free and a 4 KiB contiguous DMA block. The per-client SDK stack is +independent of `MQTT_TASK_STACK_SIZE`, which sizes the existing bridge worker. +A TLS attempt without PSRAM requires at least 60 KiB free and a 17,408-byte contiguous block. PSRAM allocation fallback must leave the 16 KiB DMA reserve available. These thresholds describe free heap at the time of admission; they are not additional linked capacity and the TLS diff --git a/docs/test-results/mqtt_tls_v4_regression.md b/docs/test-results/mqtt_tls_v4_regression.md index 49452648c..4309a305e 100644 --- a/docs/test-results/mqtt_tls_v4_regression.md +++ b/docs/test-results/mqtt_tls_v4_regression.md @@ -42,8 +42,12 @@ room-server and companion images that enable it. - Schedule initial connections through the normal maintenance loop. Check the actual pending attempt across SDK tasks, rather than assuming a five-second startup delay or 15-second reconnect gap finished a handshake. -- Defer a new connection if DMA-capable internal memory has less than - 16 KiB free or a largest block below 4 KiB. TLS without working PSRAM +- Reserve 32 KiB free DMA-capable internal memory and an 8 KiB largest block + before starting an SDK client task. This includes a 16 KiB startup allowance + above the 16 KiB AES reserve. Retained, running clients need 16 KiB free and + a 4 KiB largest block to reconnect. Pin SDK client tasks to their existing + 6,144-byte default stack; the separate bridge worker remains 8 KiB. + TLS without working PSRAM additionally needs 60 KiB free and a 17 KiB largest block. These are conservative admission bounds, not a promise that future allocations succeed. - Prefer PSRAM for TLS allocations and reject an internal fallback that @@ -72,7 +76,7 @@ PSRAM and internal-only modes, fragmented heap, exhausted PSRAM, delayed connection callbacks, retry timing and bounded replies. They do not execute an ESP32 AES peripheral or qualify physical WiFi behavior. -All 42 focused Python MQTT tests, 1,794 native tests and 31 memory-budget tests +All 43 focused Python MQTT tests, 1,794 native tests and 31 memory-budget tests passed. The focused MQTT tests are also connected to the GitHub unit-test workflow. diff --git a/scripts/check_firmware_ram.py b/scripts/check_firmware_ram.py index 41cf6e41b..8777d9b3d 100644 --- a/scripts/check_firmware_ram.py +++ b/scripts/check_firmware_ram.py @@ -34,7 +34,7 @@ DISPLAY_HEAP = { "GxEPDDisplay": 0, "E213Display": 8192, "E290Display": 8192, } -# Match MQTTBridge's default worker and the pinned ESP-MQTT client task/buffers. +# Match MQTTBridge's default worker and its explicit SDK client task/buffers. # The legacy 24 KiB allowance covers two clients with 512 bytes beyond the known # stacks/buffers. It is a startup minimum, not a bound on TLS/WSS/outbox memory. MQTT_STARTUP_ALLOWANCE = 24576 @@ -77,8 +77,12 @@ def mqtt_requirements(defines): "maximum_active_slots_without_psram": min(slots, 2), "worker_stack_bytes": worker_stack, "client_task_stack_bytes": MQTT_CLIENT_STACK_BYTES, + "client_task_priority": 5, "client_rx_tx_buffer_bytes": MQTT_CLIENT_BUFFER_BYTES, "maximum_slot_stack_buffer_allowance_bytes": potential, + "cold_start_allocation_allowance_bytes": 16384, + "cold_start_dma_admission_free_bytes": 32768, + "cold_start_dma_admission_contiguous_bytes": 8192, "dma_admission_free_bytes": 16384, "dma_admission_contiguous_bytes": 4096, "tls_internal_admission_free_bytes": 61440, diff --git a/src/helpers/MQTTConnectionAdmission.h b/src/helpers/MQTTConnectionAdmission.h index 0625d2296..7e282af0c 100644 --- a/src/helpers/MQTTConnectionAdmission.h +++ b/src/helpers/MQTTConnectionAdmission.h @@ -11,12 +11,24 @@ namespace MQTTConnectionAdmission { static const size_t kDmaReserveBytes = 16384; static const size_t kDmaLargestBytes = 4096; +// A stopped SDK client needs a new internal task stack/TCB. First starts also +// allocate receive/transmit/reassembly buffers and SDK transport/config objects +// after admission. Budget that ordinary allocation separately from AES headroom. +// Pin the actual SDK stack/priority in the optimizer rather than relying on an +// SDK default or confusing this per-client task with the MQTTBridge worker. +static const size_t kClientTaskStackBytes = 6144; +static const int kClientTaskPriority = 5; +static const size_t kColdStartAllowanceBytes = 16384; +static const size_t kColdStartLargestBytes = 8192; static const size_t kTlsInternalFreeBytes = 61440; static const size_t kTlsRecordAllocationBytes = 17408; -static inline bool canStart(bool psram, bool tls, size_t dma_free, - size_t dma_largest) { - if (dma_free < kDmaReserveBytes || dma_largest < kDmaLargestBytes) { +static inline bool canStart(bool psram, bool tls, bool cold_start, + size_t dma_free, size_t dma_largest) { + const size_t free_required = kDmaReserveBytes + + (cold_start ? kColdStartAllowanceBytes : 0); + const size_t largest_required = cold_start ? kColdStartLargestBytes : kDmaLargestBytes; + if (dma_free < free_required || dma_largest < largest_required) { return false; } return !tls || psram || diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 41c21cfc6..cd3c82f17 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2088,9 +2088,14 @@ bool MQTTBridge::canStartSlotConnection(int index) const { const char* uri = slot.preset ? slot.preset->server_url : (slot.broker_uri[0] ? slot.broker_uri : slot.host); const bool tls = MQTTConnectionAdmission::requiresTls(uri, slot.port); + // A never-created or stopped SDK client still allocates a task and ordinary + // SDK buffers after this check. Retained handles with a stopped task are + // conservatively charged the same allowance as first starts. A started + // reconnect keeps that task/buffer memory and needs only handshake headroom. + const bool cold_start = slot.client == nullptr || !slot.client->isStarted(); const uint32_t caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA; return MQTTConnectionAdmission::canStart( - ESP.getPsramSize() > 0, tls, heap_caps_get_free_size(caps), + ESP.getPsramSize() > 0, tls, cold_start, heap_caps_get_free_size(caps), heap_caps_get_largest_free_block(caps)); #else return true; @@ -5079,6 +5084,12 @@ void MQTTBridge::getClientVersion(char* buffer, size_t buffer_size) const { void MQTTBridge::optimizeMqttClientConfig(PsychicMqttClient* client, bool needs_large_buffer) { if (!client) return; + // Admission reserves this SDK client's internal stack and startup allocations + // separately from the long-lived MQTTBridge worker stack. Keep IDF4 and IDF5 + // on the same explicit settings so a changed SDK default cannot bypass it. + client->setTaskStackAndPriority(MQTTConnectionAdmission::kClientTaskStackBytes, + MQTTConnectionAdmission::kClientTaskPriority); + // Cloudflare closes WebSocket connections after 100s idle (non-configurable). #if defined(BOARD_HAS_PSRAM) client->setKeepAlive(45); diff --git a/test/test_firmware_ram.py b/test/test_firmware_ram.py index 21c03d737..16a97da25 100644 --- a/test/test_firmware_ram.py +++ b/test/test_firmware_ram.py @@ -222,6 +222,9 @@ class FirmwareRamTest(unittest.TestCase): self.assertEqual(profile["startup_active_slot_allowance"], 2) self.assertEqual(profile["maximum_active_slots_without_psram"], 2) self.assertEqual(profile["maximum_slot_stack_buffer_allowance_bytes"], potential) + self.assertEqual(profile["cold_start_allocation_allowance_bytes"], 16384) + self.assertEqual(profile["cold_start_dma_admission_free_bytes"], 32768) + self.assertEqual(profile["cold_start_dma_admission_contiguous_bytes"], 8192) self.assertFalse(profile["maximum_runtime_load_qualified"]) self.assertFalse(profile["physical_validation_performed"]) ordinary = ram.requirements("ESP32_PLATFORM", {"BOARD_HAS_PSRAM": 1}, "v4_companion") @@ -273,6 +276,10 @@ class FirmwareRamTest(unittest.TestCase): source = (ROOT / "src/helpers/MQTTConnectionAdmission.h").read_text() profile = ram.requirements("ESP32_PLATFORM", {"WITH_MQTT_BRIDGE": 1}, "v4_companion")["mqtt"] for key, name in ( + ("client_task_stack_bytes", "kClientTaskStackBytes"), + ("client_task_priority", "kClientTaskPriority"), + ("cold_start_allocation_allowance_bytes", "kColdStartAllowanceBytes"), + ("cold_start_dma_admission_contiguous_bytes", "kColdStartLargestBytes"), ("dma_admission_free_bytes", "kDmaReserveBytes"), ("dma_admission_contiguous_bytes", "kDmaLargestBytes"), ("tls_internal_admission_free_bytes", "kTlsInternalFreeBytes"), @@ -280,6 +287,8 @@ class FirmwareRamTest(unittest.TestCase): ): with self.subTest(key=key): self.assertIn(f"{name} = {profile[key]};", source) + self.assertEqual(profile["cold_start_dma_admission_free_bytes"], + profile["dma_admission_free_bytes"] + profile["cold_start_allocation_allowance_bytes"]) def test_combined_rak_budgets_uart_beside_exclusive_ethernet_and_ota(self): defines = { diff --git a/test/test_mqtt_connection_admission.py b/test/test_mqtt_connection_admission.py index 56c9e35a1..177f03632 100644 --- a/test/test_mqtt_connection_admission.py +++ b/test/test_mqtt_connection_admission.py @@ -53,8 +53,13 @@ CONNECTION_PREAMBLE = HEAP_PREAMBLE + r''' constexpr int RUNTIME_MQTT_SLOTS = 6; struct { size_t size = 0; size_t getPsramSize() const { return size; } } ESP; struct Preset { const char* server_url; }; +struct Client { + bool started = false; + bool isStarted() const { return started; } +}; struct MQTTBridge { struct MQTTSlot { + Client* client = nullptr; const Preset* preset = nullptr; char broker_uri[128] = {}, host[64] = "broker"; uint16_t port = 8883; @@ -87,18 +92,35 @@ class MqttConnectionAdmissionTest(unittest.TestCase): self.compile_run(POLICY_PREAMBLE + r''' int main() { // A large aggregate heap cannot compensate for a fragmented DMA region. - assert(!Admission::canStart(true, true, 100000, 4095)); - assert(!Admission::canStart(true, true, 16383, 20000)); - assert(Admission::canStart(true, true, 16384, 4096)); - assert(Admission::canStart(false, false, 16384, 4096)); - assert(!Admission::canStart(false, false, 16383, 4096)); - assert(!Admission::canStart(false, false, 16384, 4095)); + assert(!Admission::canStart(true, true, false, 100000, 4095)); + assert(!Admission::canStart(true, true, false, 16383, 20000)); + assert(Admission::canStart(true, true, false, 16384, 4096)); + assert(Admission::canStart(false, false, false, 16384, 4096)); + assert(!Admission::canStart(false, false, false, 16383, 4096)); + assert(!Admission::canStart(false, false, false, 16384, 4095)); // Without PSRAM, TLS also needs a full record allocation and session budget. - assert(!Admission::canStart(false, true, 61439, 20000)); - assert(!Admission::canStart(false, true, 100000, 17407)); - assert(Admission::canStart(false, true, 61440, 17408)); - assert(Admission::canStart(true, false, 16384, 4096)); + for (int cold = 0; cold <= 1; ++cold) { + assert(!Admission::canStart(false, true, cold, 61439, 20000)); + assert(!Admission::canStart(false, true, cold, 100000, 17407)); + assert(Admission::canStart(false, true, cold, 61440, 17408)); + } + assert(Admission::canStart(true, false, false, 16384, 4096)); + + // A new SDK task needs its stack and ordinary startup allocations in + // addition to the DMA reserve, including plaintext and PSRAM-backed TLS. + assert(Admission::kClientTaskStackBytes == 6144); + assert(Admission::kClientTaskPriority == 5); + assert(Admission::kColdStartAllowanceBytes == 16384); + assert(Admission::kColdStartLargestBytes == 8192); + for (int psram = 0; psram <= 1; ++psram) { + for (int tls = 0; tls <= 1; ++tls) { + if (!psram && tls) continue; // Larger TLS thresholds checked above. + assert(!Admission::canStart(psram, tls, true, 32767, 20000)); + assert(!Admission::canStart(psram, tls, true, 100000, 8191)); + assert(Admission::canStart(psram, tls, true, 32768, 8192)); + } + } } ''') @@ -256,6 +278,19 @@ int main() { dma_free = 100000; dma_largest = 17407; assert(!bridge.canStartSlotConnection(0)); ESP.size = 2097152; dma_free = 16384; dma_largest = 4096; + assert(!bridge.canStartSlotConnection(0)); + Client client; + bridge._slots[0].client = &client; + assert(!bridge.canStartSlotConnection(0)); // Allocated, but task not started. + dma_free = 32767; dma_largest = 20000; + assert(!bridge.canStartSlotConnection(0)); + dma_free = 32768; dma_largest = 8191; + assert(!bridge.canStartSlotConnection(0)); + dma_largest = 8192; + assert(bridge.canStartSlotConnection(0)); + + // A warm reconnect retains the SDK task/buffers and uses the smaller floor. + client.started = true; dma_free = 16384; dma_largest = 4096; assert(bridge.canStartSlotConnection(0)); dma_largest = 4095; assert(!bridge.canStartSlotConnection(0)); diff --git a/test/test_mqtt_handshake_scheduler.py b/test/test_mqtt_handshake_scheduler.py index 5bca0ab50..0f7736dee 100644 --- a/test/test_mqtt_handshake_scheduler.py +++ b/test/test_mqtt_handshake_scheduler.py @@ -241,16 +241,56 @@ int main() { self.compile_run(r''' int main() { MQTTBridge bridge; - bridge.only(0); clock_ms = 1000; dma_free = 16383; + bridge.only(0); clock_ms = 1000; dma_free = 32767; bridge.maintainSlotConnections(); assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); assert(bridge._slots[0].enabled && bridge.activatedSlotCount() == 0); - dma_free = 100000; dma_largest = 4095; + dma_free = 32768; dma_largest = 8191; bridge.maintainSlotConnections(); - assert(bridge.setups.empty()); - dma_largest = 4096; + assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); + assert(!bridge.hasPendingSlotConnection() && bridge.activatedSlotCount() == 0); + dma_largest = 8192; bridge.maintainSlotConnections(); assert(bridge.setups == std::vector{0} && bridge.activatedSlotCount() == 1); + assert(bridge.clients[0].starts == 1 && bridge.clients[0].reconnects == 0); +} +''') + + def test_stopped_client_defers_while_retained_task_reconnects_at_warm_budget(self): + self.compile_run(r''' +int main() { + MQTTBridge bridge; + for (int i = 2; i < RUNTIME_MQTT_SLOTS; ++i) bridge._slots[i].enabled = false; + clock_ms = 70000; + dma_free = 16384; dma_largest = 4096; + auto& stopped = bridge._slots[0]; + auto& retained = bridge._slots[1]; + stopped.client = &bridge.clients[0]; + retained.client = &bridge.clients[1]; + stopped.initial_connect_done = retained.initial_connect_done = true; + retained.client->started = true; + + // An existing handle with a stopped task needs the cold-start allowance. + // Its deferral must leave this cycle available for the retained SDK task. + assert(!bridge.canStartSlotConnection(0)); + assert(bridge.canStartSlotConnection(1)); + bridge.maintainSlotConnections(); + assert(stopped.client->starts == 0 && stopped.client->reconnects == 0); + assert(stopped.last_reconnect_attempt == 0 && stopped.reconnect_backoff == 0); + assert(stopped.start_failures == 0 && !bridge._slot_attempt_pending[0]); + assert(retained.client->starts == 0 && retained.client->reconnects == 1); + assert(bridge._slot_attempt_pending[1]); + + // Completion and newly available cold-start headroom allow a real SDK start, + // with no extra delay introduced by the earlier memory-only deferral. + bridge.complete(1); + clock_ms += 15000; + dma_free = 32768; dma_largest = 8192; + bridge.maintainSlotConnections(); + assert(stopped.client->starts == 1 && stopped.client->reconnects == 0); + assert(stopped.client->isStarted() && bridge._slot_attempt_pending[0]); + assert(stopped.last_reconnect_attempt == clock_ms); + assert(retained.client->reconnects == 1 && bridge.setups.empty()); } ''') diff --git a/test/test_mqtt_network_timeout.py b/test/test_mqtt_network_timeout.py index 174bd99f8..dc26d962a 100644 --- a/test/test_mqtt_network_timeout.py +++ b/test/test_mqtt_network_timeout.py @@ -17,15 +17,18 @@ ROOT = Path(__file__).resolve().parents[1] PREAMBLE = r''' #include #include +#include "helpers/MQTTConnectionAdmission.h" #include "helpers/MQTTConnectionPolicy.h" struct esp_mqtt_client_config_t { #if ESP_IDF_VERSION_MAJOR == 5 struct { int keepalive = 0, message_retransmit_timeout = 0; } session; struct { int timeout_ms = 0; } network; struct { int size = 0, out_size = 0; } buffer; + struct { int stack_size = 0, priority = 0; } task; #else int keepalive = 0, message_retransmit_timeout = 0; int network_timeout_ms = 0, buffer_size = 0; + int task_stack = 0, task_prio = 0; #endif }; struct PsychicMqttClient { @@ -37,6 +40,7 @@ struct PsychicMqttClient { PsychicMqttClient& setBufferSize(int); PsychicMqttClient& setNetworkTimeout(int); PsychicMqttClient& setOutboxLimit(size_t); + PsychicMqttClient& setTaskStackAndPriority(int, int); esp_mqtt_client_config_t* getMqttConfig(); }; struct MQTTBridge { @@ -53,6 +57,7 @@ int main() { // for both small user/password and large JWT CONNECT buffers. for (bool needs_large : {false, true, false}) { client.setNetworkTimeout(2500); + client.setTaskStackAndPriority(2048, 1); client._config_dirty = false; bridge.optimizeMqttClientConfig(&client, needs_large); assert(client._config_dirty); @@ -60,16 +65,23 @@ int main() { const int timeout = client._mqtt_cfg.network.timeout_ms; const int buffer_size = client._mqtt_cfg.buffer.size; const int keepalive = client._mqtt_cfg.session.keepalive; + const int stack = client._mqtt_cfg.task.stack_size; + const int priority = client._mqtt_cfg.task.priority; assert(client._mqtt_cfg.buffer.out_size == buffer_size); assert(client._mqtt_cfg.session.message_retransmit_timeout == 15000); #else const int timeout = client._mqtt_cfg.network_timeout_ms; const int buffer_size = client._mqtt_cfg.buffer_size; const int keepalive = client._mqtt_cfg.keepalive; + const int stack = client._mqtt_cfg.task_stack; + const int priority = client._mqtt_cfg.task_prio; assert(client._mqtt_cfg.message_retransmit_timeout == 15000); #endif assert(timeout == 10000); assert(timeout == MQTTConnectionPolicy::kNetworkTimeoutMs); + assert(stack == 6144 && priority == 5); + assert(stack == MQTTConnectionAdmission::kClientTaskStackBytes); + assert(priority == MQTTConnectionAdmission::kClientTaskPriority); #ifdef BOARD_HAS_PSRAM assert(buffer_size == 896 && keepalive == 45); assert(client._outbox_limit == 16384); @@ -92,6 +104,7 @@ class MqttNetworkTimeoutTest(unittest.TestCase): "PsychicMqttClient &PsychicMqttClient::setBufferSize(", "PsychicMqttClient &PsychicMqttClient::setNetworkTimeout(", "PsychicMqttClient &PsychicMqttClient::setOutboxLimit(", + "PsychicMqttClient &PsychicMqttClient::setTaskStackAndPriority(", "esp_mqtt_client_config_t *PsychicMqttClient::getMqttConfig(")) source += method("src/helpers/bridges/MQTTBridge.cpp", "void MQTTBridge::optimizeMqttClientConfig(")