From 142ff464e8d8b061d7a528f3c409ee777fecd932 Mon Sep 17 00:00:00 2001 From: "torlando-agent[bot]" <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:58:45 -0400 Subject: [PATCH] fix: populate announces end-to-end (ratchet name, blob leak, path-table, safe refresh) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins microReticulum to the fork's pyxis-fixes-on-0.4.1 @ b582986 (three 0.4.1-graft regression fixes) and adds the pyxis-side pieces. microReticulum fixes (via the pin): - Identity::recall now accounts for the announce ratchet (matches validate_announce) — it read app_data 32 bytes early for ratchet-bearing announces (Sideband/Columba), garbling peer display names. - DestinationEntry caps persisted random_blobs to PERSIST_RANDOM_BLOBS (16); the set was never trimmed, growing path entries ~12B/re-announce past microStore's 1024B ceiling so put() rejected them (empty path table) — and under a raised ceiling bloated the FS until compaction crashed. - Transport mirrors learned paths into _path_table so path_table() (the UI announce list + T:PATHS) reflects them; the microStore migration stopped populating it. pyxis pieces: - -DUSTORE_DEFAULT_MAX_RECS=400 caps the live path-store record count (defense-in-depth). - AnnounceListScreen::refresh() defers to a main-loop tick(): the gather (path-table iteration + per-entry recall/app_data) runs off the LVGL lock, serialized with the main-loop path-table writes, instead of racing them on the LVGL task and blocking past the 5s lock timeout — which hung/crashed on open once the table was non-empty. Verified on device: display names correct, T:PATHS 0 -> 11, no rejections, no crashes, announce list opens cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01UWZuYkHBRqNb6BZHV8sTG5 --- lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp | 69 ++++++++++----------- lib/tdeck_ui/UI/LXMF/AnnounceListScreen.h | 9 +++ lib/tdeck_ui/UI/LXMF/UIManager.cpp | 7 +++ platformio.ini | 12 +++- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp index ea7064e8..b41ef911 100644 --- a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp @@ -128,41 +128,39 @@ void AnnounceListScreen::create_list() { } void AnnounceListScreen::refresh() { - LVGL_LOCK(); - INFO("Refreshing announce list"); + // Defer the real work to tick() on the main loop (UIManager::update). The + // gather (iterate the live path table + per-entry Identity::recall / + // recall_app_data) must NOT run on the LVGL task or hold the render lock: + // once the path table became non-empty it raced the main-loop path-table + // writes and blocked past LVGLLock's 5s timeout — hanging, then crashing, on + // open. refresh() is safe to call from any task; it just arms the tick. + _refresh_pending.store(true); +} - // Clear existing items (also removes from focus group when deleted) - lv_obj_clean(_list); - _announces.clear(); - _announce_containers.clear(); - _dest_hash_pool.clear(); - _empty_label = nullptr; +void AnnounceListScreen::tick() { + // Runs on the main loop (UIManager::update), so iterating Transport::path_table() + // and reading the identity cache is serialized with — never concurrent to — the + // main-loop announce processing that writes them (both run in the Arduino loop). + // No LVGL lock is held during the gather; only the final render briefly takes it. + if (!_refresh_pending.exchange(false)) { + return; + } - // Get path table from Transport. Pre-graft the fork called this - // get_destination_table(); upstream microReticulum @ 0.3.0 renamed it - // to get_path_table() (the same map of dest_hash → DestinationEntry). + // GATHER (no LVGL lock) — build the item list off the render lock. + std::vector items; const auto& dest_table = Transport::path_table(); - - // Compute name_hash for lxmf.delivery to filter announces - Bytes lxmf_delivery_name_hash = Destination::name_hash("lxmf", "delivery"); - for (auto it = dest_table.begin(); it != dest_table.end(); ++it) { const Bytes& dest_hash = it->first; - // Pre-graft: nested Transport::DestinationEntry. Upstream moved - // this to RNS::Persistence::DestinationEntry in - // Persistence/DestinationEntry.h. const RNS::Persistence::DestinationEntry& dest_entry = it->second; - // Check if this destination has a known identity (was announced properly) + // Only lxmf.delivery destinations with a known identity belong in the list. Identity identity = Identity::recall(dest_hash); if (!identity) { - continue; // Skip destinations without known identity + continue; } - - // Verify this is an lxmf.delivery destination by computing expected hash Bytes expected_hash = Destination::hash(identity, "lxmf", "delivery"); if (dest_hash != expected_hash) { - continue; // Not an lxmf.delivery destination + continue; } AnnounceItem item; @@ -171,40 +169,41 @@ void AnnounceListScreen::refresh() { item.hops = dest_entry._hops; item.timestamp = dest_entry._timestamp; item.timestamp_str = format_timestamp(dest_entry._timestamp); - item.has_path = Transport::has_path(dest_hash); - - // Try to get display name from app_data + item.has_path = true; // it's in the path table we're iterating — skip the store probe Bytes app_data = Identity::recall_app_data(dest_hash); if (app_data && app_data.size() > 0) { item.display_name = parse_display_name(app_data); } - - _announces.push_back(item); + items.push_back(item); + if (items.size() >= 64) break; // bound the gather } - // Sort by timestamp (newest first) - std::sort(_announces.begin(), _announces.end(), + std::sort(items.begin(), items.end(), [](const AnnounceItem& a, const AnnounceItem& b) { return a.timestamp > b.timestamp; }); { char log_buf[64]; - snprintf(log_buf, sizeof(log_buf), " Found %zu announced destinations", _announces.size()); + snprintf(log_buf, sizeof(log_buf), "Announce list: %zu lxmf.delivery destinations", items.size()); INFO(log_buf); } + // RENDER (brief LVGL lock) — no store access here, capped item count. + LVGL_LOCK(); + lv_obj_clean(_list); + _announce_containers.clear(); + _dest_hash_pool.clear(); + _empty_label = nullptr; + _announces = std::move(items); + if (_announces.empty()) { show_empty_state(); } else { - // Limit to 20 most recent to prevent memory exhaustion const size_t MAX_DISPLAY = 20; size_t display_count = std::min(_announces.size(), MAX_DISPLAY); - - // Reserve capacity to avoid reallocations during population _dest_hash_pool.reserve(display_count); _announce_containers.reserve(display_count); - size_t count = 0; for (const auto& item : _announces) { if (count >= MAX_DISPLAY) break; diff --git a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.h b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.h index fc0843ea..b4e452c4 100644 --- a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace UI { @@ -77,6 +78,13 @@ public: */ void refresh(); + /** + * Main-loop tick: if a refresh was requested, gather the announce items off + * the LVGL lock (race-free on the main loop) and render them. Call from + * UIManager::update() while the announces screen is shown. + */ + void tick(); + /** * Set callback for announce selection (to start conversation) * @param callback Function to call when announce is selected @@ -121,6 +129,7 @@ private: lv_obj_t* _empty_label; std::vector _announces; + std::atomic _refresh_pending{false}; // armed by refresh(), consumed by tick() on the main loop std::vector _announce_containers; // For focus group management std::vector _dest_hash_pool; // Object pool to avoid per-item allocations diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index eb5e18cf..8cbbb613 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -334,6 +334,13 @@ void UIManager::update() { if (_current_screen == SCREEN_CHAT && _chat_screen) { _chat_screen->tick_background_fill(); } + // Gather + render the announce list off the LVGL lock on the main loop (its + // refresh() just arms a flag). Iterating the path table on the LVGL task raced + // the main-loop path-table writes and blocked past the lock timeout once the + // table was non-empty — this serializes the gather with the writes instead. + if (_current_screen == SCREEN_ANNOUNCES && _announce_list_screen) { + _announce_list_screen->tick(); + } LVGL_LOCK(); // Process outbound LXMF messages _router.process_outbound(); diff --git a/platformio.ini b/platformio.ini index 9dbcecc6..865bf054 100644 --- a/platformio.ini +++ b/platformio.ini @@ -76,7 +76,11 @@ lib_deps = ; racy 32-bit millis() rollover counter (static low32/high32 raced across ; FreeRTOS tasks, inflating OS::time() by N*49.7 days — garbage convo ; timestamps, e.g. "Future"/"49w ago"/year-2046 now). - https://github.com/torlando-tech/microReticulum.git#cb71aceb8a9c58444521d5ce24ea2db10191d92d + ; 97fecfe: Identity::recall now accounts for the optional ratchet when + ; extracting announce app_data (matches validate_announce). Without it, + ; ratchet-bearing announces (Sideband/Columba) had app_data read 32 bytes + ; too early, garbling peer display names ("ySId!M@pB`Z:ev"-style prefix). + https://github.com/torlando-tech/microReticulum.git#b582986029ff3272e0cfd99a958b02d9657da33d ; microLXMF: chore/microreticulum-0.4.1-layout — includes namespaced to ; for the 0.4.x src/microReticulum/ layout. ; 3cdde79: faster load_message_metadata — single LittleFS open (read_file @@ -189,6 +193,12 @@ build_flags = ; caused FileStore::put → flush_buffer() to fail under sustained ; announce writes (~3/sec), surfacing as path-table-add error spam. -DUSTORE_USE_LITTLEFS + ; Cap the path-store live record count (defense-in-depth). microReticulum now + ; bounds each entry's random_blobs (the actual leak that filled the FS), but this + ; also bounds the TOTAL live set so the compaction output segment (path_store_0) + ; cannot bloat the 1.875MB partition over long uptime — prune_index runs inside + ; put() and compact(). 400 records x ~560B ~= 225KB, far from full. + -DUSTORE_DEFAULT_MAX_RECS=400 -DUSE_NIMBLE -DCONFIG_BT_NIMBLE_MEM_ALLOC_MODE_EXTERNAL=1 -Os