From 8ed9551a02e60c0da63150732f2b69c3af3690ef Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:38:03 +0000 Subject: [PATCH 01/13] perf(messages): fast conversation-list load + real unread badges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refresh() did heavy unnecessary work per conversation, on the LVGL render task under the render lock: - get_messages_for_conversation() copied the full 256-slot hash array (8KB) just to read the newest hash (messages.back()); - load_message() on that hash read the payload file ~3x, JSON-parsed it twice (including the large hex 'packed' blob), hex-decoded and msgpack-unpacked the whole message — to extract a 30-char preview and a timestamp; - the unread badge was rendered from a hardwired 0 even though the store maintains and persists unread_count. Now: - newest hash via the new O(1) MessageStore::get_last_message_hash (index tail — always hot-tier, no I/O, no array copy); - preview/timestamp via load_message_metadata (single open + filtered parse, the fast path ChatScreen already uses for the same fields); - unread badge from MessageStore::get_conversation_unread_count; - badge cleared + mark-read on open (click) and when a message lands in the currently-viewed chat, with the LittleFS index commit deferred out of the LVGL lock (UIManager::update), matching the existing deferred display-name write-through pattern. Pins microLXMF 59ca70a (PR #10, temporary branch head) for the two new accessors. --- .../UI/LXMF/ConversationListScreen.cpp | 107 ++++++++++++++---- lib/tdeck_ui/UI/LXMF/ConversationListScreen.h | 26 ++++- lib/tdeck_ui/UI/LXMF/UIManager.cpp | 9 ++ platformio.ini | 8 +- 4 files changed, 123 insertions(+), 27 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 20ef5b4e..a44b5948 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -204,6 +204,7 @@ void ConversationListScreen::refresh() { lv_obj_clean(_list); _conversations.clear(); _conversation_containers.clear(); + _badge_pool.clear(); _peer_hash_pool.clear(); _has_unresolved_names = false; @@ -214,6 +215,7 @@ void ConversationListScreen::refresh() { _peer_hash_pool.reserve(peer_hashes.size()); _conversations.reserve(peer_hashes.size()); _conversation_containers.reserve(peer_hashes.size()); + _badge_pool.reserve(peer_hashes.size()); { char log_buf[48]; @@ -222,15 +224,28 @@ void ConversationListScreen::refresh() { } for (const auto& peer_hash : peer_hashes) { - std::vector messages = _message_store->get_messages_for_conversation(peer_hash); - - if (messages.empty()) { + // Last-message hash comes from the conversation index cache + // (O(1) in-memory lookup, no 8KB hash-array copy and no I/O). + Bytes last_msg_hash = _message_store->get_last_message_hash(peer_hash); + if (!last_msg_hash) { continue; } - // Load last message for preview - Bytes last_msg_hash = messages.back(); - ::LXMF::LXMessage last_msg = _message_store->load_message(last_msg_hash); + // Load last message metadata for the preview. load_message() would + // also read the payload file twice (recovery validation + load), + // parse the full JSON twice (including the large "packed" hex + // blob), hex-decode it, and msgpack-unpack the whole message — + // all for a 30-char preview and a timestamp. The metadata path + // does one open + one filtered parse per conversation, which is + // what ChatScreen already uses for the same fields. + ::LXMF::MessageStore::MessageMetadata last_meta = + _message_store->load_message_metadata(last_msg_hash); + + if (!last_meta.valid) { + // Corrupt/unreadable last message — skip the row rather than + // rendering an empty preview with a bogus timestamp. + continue; + } // Create conversation item ConversationItem item; @@ -270,16 +285,21 @@ void ConversationListScreen::refresh() { _has_unresolved_names = true; } - // Get message content for preview - String content((const char*)last_msg.content().data(), last_msg.content().size()); + // Get message content for preview (metadata path returns the + // pre-extracted UTF-8 content — no msgpack unpacking) + String content(last_meta.content.c_str()); item.last_message = content.substring(0, 30); // Truncate to 30 chars if (content.length() > 30) { item.last_message += "..."; } - item.timestamp = (uint32_t)last_msg.timestamp(); + item.timestamp = (uint32_t)last_meta.timestamp; item.timestamp_str = format_timestamp(item.timestamp); - item.unread_count = 0; // TODO: Track unread count + // Unread count straight from the conversation index (persisted, + // incremented on incoming save, cleared when the user opens the + // chat via request_mark_read()). + item.unread_count = + (uint16_t)_message_store->get_conversation_unread_count(peer_hash); _conversations.push_back(item); create_conversation_item(item); @@ -356,12 +376,17 @@ void ConversationListScreen::create_conversation_item(const ConversationItem& it lv_obj_t* label_time = lv_label_create(container); lv_label_set_text(label_time, item.timestamp_str.c_str()); - lv_obj_align(label_time, LV_ALIGN_BOTTOM_RIGHT, -6, -4); + // Leave room for the unread badge (20px + 6px pad) when one is + // present; the badge occupies the bottom-right corner. + lv_obj_align(label_time, LV_ALIGN_BOTTOM_RIGHT, + item.unread_count > 0 ? -30 : -6, -4); lv_obj_set_style_text_color(label_time, Theme::textMuted(), 0); - // Unread count badge + // Unread count badge. Always append to _badge_pool (nullptr when no + // badge) so the pool stays index-aligned with _conversation_containers. + lv_obj_t* badge = nullptr; if (item.unread_count > 0) { - lv_obj_t* badge = lv_obj_create(container); + badge = lv_obj_create(container); lv_obj_set_size(badge, 20, 20); lv_obj_align(badge, LV_ALIGN_BOTTOM_RIGHT, -6, -4); lv_obj_set_style_bg_color(badge, Theme::error(), 0); @@ -370,20 +395,53 @@ void ConversationListScreen::create_conversation_item(const ConversationItem& it lv_obj_set_style_pad_all(badge, 0, 0); lv_obj_t* label_count = lv_label_create(badge); - lv_label_set_text_fmt(label_count, "%d", item.unread_count); + // Badge is a 20px circle; cap the digit count so it always fits. + int shown = item.unread_count; + if (shown > 9) shown = 9; + lv_label_set_text_fmt(label_count, "%d", shown); lv_obj_center(label_count); lv_obj_set_style_text_color(label_count, lv_color_white(), 0); } + _badge_pool.push_back(badge); } -void ConversationListScreen::update_unread_count(const Bytes& peer_hash, uint16_t unread_count) { - LVGL_LOCK(); - // Find conversation and update - for (auto& conv : _conversations) { - if (conv.peer_hash == peer_hash) { - conv.unread_count = unread_count; - refresh(); // Redraw list - break; +void ConversationListScreen::request_mark_read(const Bytes& peer_hash) { + // Defers the actual store mutation. mark_conversation_read() commits + // the index to LittleFS, so it runs from UIManager::update() BEFORE + // the LVGL lock (same pattern as flush_pending_name_writes) — not + // from this LVGL event callback. + for (const auto& h : _pending_mark_reads) { + if (h == peer_hash) return; + } + _pending_mark_reads.push_back(peer_hash); +} + +void ConversationListScreen::flush_pending_mark_reads() { + // Called from UIManager::update() BEFORE it takes the LVGL lock. + // mark_conversation_read() hits microStore/LittleFS; the LVGL event + // that requested the mark-read only set the flag. + if (!_message_store || _pending_mark_reads.empty()) { + return; + } + for (const auto& h : _pending_mark_reads) { + _message_store->mark_conversation_read(h); + } + _pending_mark_reads.clear(); +} + +void ConversationListScreen::clear_unread_badge(lv_obj_t* container) { + // Remove the unread badge from a rendered conversation row. The badge + // is tracked in _badge_pool in lockstep with _conversation_containers, + // so locate the row by container and delete its stored badge object. + // (LVGL 8.4's lv_obj_t is opaque — no public child iteration — so a + // side table is the portable way to reach the badge on click.) + for (size_t i = 0; i < _conversation_containers.size(); ++i) { + if (_conversation_containers[i] == container) { + if (i < _badge_pool.size() && _badge_pool[i]) { + lv_obj_del(_badge_pool[i]); + _badge_pool[i] = nullptr; + } + return; } } } @@ -655,6 +713,11 @@ void ConversationListScreen::on_conversation_clicked(lv_event_t* event) { Bytes* peer_hash = (Bytes*)lv_obj_get_user_data(target); if (peer_hash && screen->_conversation_selected_callback) { + // Clear the badge in the UI now; the store commit is deferred to + // UIManager::update() (mark_conversation_read() writes the + // LittleFS index and must not run under the LVGL lock). + screen->clear_unread_badge(target); + screen->request_mark_read(*peer_hash); screen->_conversation_selected_callback(*peer_hash); } } diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h index efc2a976..884a8e93 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h @@ -102,11 +102,24 @@ public: void flush_pending_name_writes(); /** - * Update unread count for a specific conversation - * @param peer_hash Peer hash - * @param unread_count New unread count + * Request that a conversation be marked read. Defers the store + * mutation (LittleFS index commit) out of the LVGL event callback; + * MUST be drained by flush_pending_mark_reads() OUTSIDE the LVGL + * lock (UIManager::update() calls it before taking the lock), the + * same pattern as the deferred display-name write-throughs. */ - void update_unread_count(const RNS::Bytes& peer_hash, uint16_t unread_count); + void request_mark_read(const RNS::Bytes& peer_hash); + + /** + * Commit pending mark-read requests to the store. MUST be called + * OUTSIDE the LVGL lock. + */ + void flush_pending_mark_reads(); + + /** + * Remove the unread badge from a rendered conversation row. + */ + void clear_unread_badge(lv_obj_t* container); /** * Set callback for conversation selection @@ -213,6 +226,7 @@ private: ::LXMF::MessageStore* _message_store; std::vector _conversations; std::vector _conversation_containers; // For focus group management + std::vector _badge_pool; // Unread badge per row (nullptr when none); index-aligned with _conversation_containers std::vector _peer_hash_pool; // Object pool to avoid per-item allocations RNS::Bytes _pending_delete_hash; // Hash of conversation pending deletion bool _has_unresolved_names = false; // True if any conversation shows hash instead of name @@ -221,6 +235,10 @@ private: // under the lock stalls the render task on a cold-boot announce burst. // Drained by UIManager::update() before it takes the lock. std::vector> _pending_name_writes; + // Mark-read requests deferred the same way: the LVGL click handler + // only queues the hash; UIManager::update() commits it (LittleFS + // index write) before taking the lock. + std::vector _pending_mark_reads; ConversationSelectedCallback _conversation_selected_callback; ComposeCallback _compose_callback; diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index a492da09..c6bb5765 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -719,6 +719,9 @@ void UIManager::update() { // never runs under the render lock (same reason as on_message_received). if (_conversation_list_screen) { _conversation_list_screen->flush_pending_name_writes(); + // Commit queued mark-read index writes the same way — the LVGL + // click handler only recorded the peer hash. + _conversation_list_screen->flush_pending_mark_reads(); } // SERVICE ORDER CONTRACT: consume peer consent commands and complete the @@ -1783,6 +1786,12 @@ void UIManager::on_message_received(::LXMF::LXMessage& message) { bool viewing_this_chat = (_navigation.current() == Route::CHAT && _current_peer_hash == message.source_hash()); if (viewing_this_chat) { _chat_screen->add_message(message, false); + // The user is watching this conversation land, so it's read. + // Defers the index commit to UIManager::update() like the + // click-path badge clear. + if (_conversation_list_screen) { + _conversation_list_screen->request_mark_read(message.source_hash()); + } } // Play notification sound if enabled and not viewing this conversation diff --git a/platformio.ini b/platformio.ini index a0812c0f..d0846420 100644 --- a/platformio.ini +++ b/platformio.ini @@ -117,7 +117,13 @@ lib_deps = ; 60fb7d6: merged non-evicting guarded outbound admission, ; persist-only-after-preparation semantics, and serialized native bridge ; transport callbacks/conformance closure from microLXMF PR #9. - https://github.com/torlando-tech/microLXMF.git#60fb7d6951bd17275da83fdd9581900d55b0a2ab + ; 59ca70a (feat/conversation-last-message-accessors, microLXMF PR #10): + ; O(1) per-conversation get_last_message_hash + get_conversation_unread_count + ; — the conversation list reads the newest hash from the index tail + ; instead of copying the full 8KB hash array, and renders the real + ; persisted unread badge. TEMPORARY branch-head pin: re-point to the + ; merge commit on main after PR #10 merges. + https://github.com/torlando-tech/microLXMF.git#59ca70a34abd5419f3e181d69e2262817e2548e7 lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 From 15239e67c4fcb35bf35397783bf53b2493c00163 Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:50:32 +0000 Subject: [PATCH 02/13] chore(release): bump microLXMF pin assertion to 59ca70a (PR #10 head) --- tools/audit_release_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index 66743978..122c9222 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "60fb7d6951bd17275da83fdd9581900d55b0a2ab", + "microLXMF": "59ca70a34abd5419f3e181d69e2262817e2548e7", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From 48ee2d4d71f717874e21344b7caa6b0dbd703eaf Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:24:51 +0000 Subject: [PATCH 03/13] chore(deps): re-point microLXMF pin to merge commit df1b01a PR #10 merged to main; swap the temporary branch-head pin (59ca70a) for the merge commit so the consumer tracks a stable main revision. Update the release-audit allowlist in the same commit. Firmware source is byte-identical to 59ca70a (the merge added only the CI workflow commit). --- platformio.ini | 7 +++---- tools/audit_release_build.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/platformio.ini b/platformio.ini index d0846420..02b4a4d3 100644 --- a/platformio.ini +++ b/platformio.ini @@ -117,13 +117,12 @@ lib_deps = ; 60fb7d6: merged non-evicting guarded outbound admission, ; persist-only-after-preparation semantics, and serialized native bridge ; transport callbacks/conformance closure from microLXMF PR #9. - ; 59ca70a (feat/conversation-last-message-accessors, microLXMF PR #10): + ; df1b01a (merged feat/conversation-last-message-accessors, microLXMF PR #10): ; O(1) per-conversation get_last_message_hash + get_conversation_unread_count ; — the conversation list reads the newest hash from the index tail ; instead of copying the full 8KB hash array, and renders the real - ; persisted unread badge. TEMPORARY branch-head pin: re-point to the - ; merge commit on main after PR #10 merges. - https://github.com/torlando-tech/microLXMF.git#59ca70a34abd5419f3e181d69e2262817e2548e7 + ; persisted unread badge. + https://github.com/torlando-tech/microLXMF.git#df1b01a21c27b11b2cbd1674c7f8b648b76345ae lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index 122c9222..f5e90750 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "59ca70a34abd5419f3e181d69e2262817e2548e7", + "microLXMF": "df1b01a21c27b11b2cbd1674c7f8b648b76345ae", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From 918f89d63ef1809023a9a706d3e67922a60ec956 Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 04:14:52 +0000 Subject: [PATCH 04/13] fix(messages): drop corrupt last message instead of hiding the row When a conversation's newest message is unreadable, walk the index newest-to-oldest for the newest readable preview and queue the unreadable messages for deletion (deferred out of the LVGL lock, drained by UIManager::update()). The store's delete_message() commits the index and updates last_message_hash, so the next refresh converges to the same preview and the row never disappears over one bad message. --- .../UI/LXMF/ConversationListScreen.cpp | 69 ++++++++++++++++++- lib/tdeck_ui/UI/LXMF/ConversationListScreen.h | 18 +++++ lib/tdeck_ui/UI/LXMF/UIManager.cpp | 5 ++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index a44b5948..eab42e68 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -242,9 +242,41 @@ void ConversationListScreen::refresh() { _message_store->load_message_metadata(last_msg_hash); if (!last_meta.valid) { - // Corrupt/unreadable last message — skip the row rather than - // rendering an empty preview with a bogus timestamp. - continue; + // The newest message is corrupt/unreadable. Don't hide the + // conversation over one bad message: drop the unreadable + // message(s) and preview the newest readable one instead. + // + // This only costs the get_messages_for_conversation() array + // copy and the extra metadata reads on the corrupt path — the + // common (all-readable) path above stays O(1) + one read. + // + // delete_message() commits the index and removes files, so it + // must not run under this LVGL lock: we only queue the hashes + // here, and UIManager::update() drains them (before taking the + // lock) — same deferred pattern as mark-read / name writes. + // The store's index updates last_message_hash on delete, so + // the next refresh converges to the same preview. + std::vector hashes = + _message_store->get_messages_for_conversation(peer_hash); + request_drop_message(last_msg_hash); // tail is the known-bad one + // Walk newest→oldest over all but the tail (array is + // chronological; hashes.back() is the tail we just queued) to + // find the newest readable preview, queuing any unreadable + // messages along the way. + for (int i = (int)hashes.size() - 2; i >= 0; --i) { + ::LXMF::MessageStore::MessageMetadata m = + _message_store->load_message_metadata(hashes[i]); + if (m.valid) { + last_meta = m; // newest readable preview + break; + } + request_drop_message(hashes[i]); + } + if (!last_meta.valid) { + // Every message was unreadable; all queued for drop. Skip + // the row this refresh — the store converges to none. + continue; + } } // Create conversation item @@ -429,6 +461,37 @@ void ConversationListScreen::flush_pending_mark_reads() { _pending_mark_reads.clear(); } +void ConversationListScreen::request_drop_message(const Bytes& message_hash) { + // Queues a corrupt/unreadable message for deletion. The actual + // delete_message() (index commit + payload file removal, and the + // last_message_hash update) runs from flush_pending_drops() OUTSIDE + // the LVGL lock. A bounded number of unreadable messages can be + // queued per refresh (one per conversation, walking the tail), so the + // queue cannot grow without bound. + for (const auto& h : _pending_drops) { + if (h == message_hash) return; + } + _pending_drops.push_back(message_hash); +} + +void ConversationListScreen::flush_pending_drops() { + // Called from UIManager::update() BEFORE it takes the LVGL lock. + // Drop the queued unreadable messages so the conversation index + // converges (deletion updates last_message_hash to the new tail); the + // next list refresh then previews the newest readable message. A + // delete that fails (store not ready) is left for a later drain. + if (!_message_store || _pending_drops.empty()) { + return; + } + std::vector remaining; + for (const auto& h : _pending_drops) { + if (!_message_store->delete_message(h)) { + remaining.push_back(h); + } + } + _pending_drops.swap(remaining); +} + void ConversationListScreen::clear_unread_badge(lv_obj_t* container) { // Remove the unread badge from a rendered conversation row. The badge // is tracked in _badge_pool in lockstep with _conversation_containers, diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h index 884a8e93..541f4ed3 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h @@ -121,6 +121,19 @@ public: */ void clear_unread_badge(lv_obj_t* container); + /** + * Queue a corrupt/unreadable message for deletion. The actual + * store mutation (index commit + payload removal) is drained by + * flush_pending_drops() OUTSIDE the LVGL lock, like mark-read. + */ + void request_drop_message(const RNS::Bytes& message_hash); + + /** + * Drop queued unreadable messages from the store. MUST be called + * OUTSIDE the LVGL lock. + */ + void flush_pending_drops(); + /** * Set callback for conversation selection * @param callback Function to call when conversation is selected @@ -239,6 +252,11 @@ private: // only queues the hash; UIManager::update() commits it (LittleFS // index write) before taking the lock. std::vector _pending_mark_reads; + // Corrupt/unreadable messages queued for deletion (one per unreadable + // tail walk, bounded per refresh); UIManager::update() drops them + // before taking the lock so the index converges to the newest + // readable message. + std::vector _pending_drops; ConversationSelectedCallback _conversation_selected_callback; ComposeCallback _compose_callback; diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index c6bb5765..f37645c5 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -719,6 +719,11 @@ void UIManager::update() { // never runs under the render lock (same reason as on_message_received). if (_conversation_list_screen) { _conversation_list_screen->flush_pending_name_writes(); + // Drop queued corrupt messages (index commit + payload removal). + // Drained before mark-read so both index commits land in a + // sensible order — each save_index() rewrites the whole index, + // and mark-read's final write carries the cleared unread count. + _conversation_list_screen->flush_pending_drops(); // Commit queued mark-read index writes the same way — the LVGL // click handler only recorded the peer hash. _conversation_list_screen->flush_pending_mark_reads(); From cb6a9d38e6ad1399a6eea309f20862e0964f1dcd Mon Sep 17 00:00:00 2001 From: Pike Date: Thu, 3 Sep 2026 14:30:03 +0000 Subject: [PATCH 05/13] perf(messages): read conversation preview from store index cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refresh() previously opened + parsed each conversation's newest message file (load_message_metadata) on every list refresh, which dominated list-load time on SPI LittleFS. It now reads the per-conversation last-message preview + timestamp from the store's in-memory index (O(1), zero I/O) and falls back to load_message_metadata only when the index has no cached preview — the first refresh after a firmware upgrade or after a corrupt-tail drop — then writes the preview back through so the fallback happens at most once per conversation per firmware generation. The write-through is skipped when drops were queued (the drained deletes move the tail, so a preview written for the old tail would be stale for one refresh). Host benchmark (x86 + POSIX fs, 24 msgs/conv): the per-conversation store-load work drops from ~0.149ms (9 convs) / ~0.342ms (20 convs) to ~0.002ms / ~0.005ms, and the cold-boot path (store reconstructed from disk) matches the warm path because the preview now persists in the index. Bumps the microLXMF pin to c8d3156 (feat/conversation-preview-cache) in platformio.ini, the release audit, and the native reference test. Adds tests/microlxmf/bench_conversation_list_load.cpp (baseline vs index warm/cold) and the bench target in tests/microlxmf/CMakeLists.txt. --- .../UI/LXMF/ConversationListScreen.cpp | 60 +++- platformio.ini | 6 +- tests/microlxmf/CMakeLists.txt | 26 ++ .../bench_conversation_list_load.cpp | 328 ++++++++++++++++++ .../test_microlxmf_telemetry_field.py | 2 +- tools/audit_release_build.py | 2 +- 6 files changed, 406 insertions(+), 18 deletions(-) create mode 100644 tests/microlxmf/bench_conversation_list_load.cpp diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index eab42e68..05d27603 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -231,24 +231,39 @@ void ConversationListScreen::refresh() { continue; } - // Load last message metadata for the preview. load_message() would - // also read the payload file twice (recovery validation + load), - // parse the full JSON twice (including the large "packed" hex - // blob), hex-decode it, and msgpack-unpack the whole message — - // all for a 30-char preview and a timestamp. The metadata path - // does one open + one filtered parse per conversation, which is - // what ChatScreen already uses for the same fields. - ::LXMF::MessageStore::MessageMetadata last_meta = - _message_store->load_message_metadata(last_msg_hash); + // Preview + timestamp come from the conversation index cache when + // available: O(1), zero I/O. Only fall back to load_message_metadata() + // when the cache is empty — the first refresh after a firmware + // upgrade (index generation predates the field) or after a + // corrupt-message drop. The write-through below re-pops the cache, + // so the fallback happens at most once per conversation per + // firmware generation. + // + // The metadata path does one open + one filtered parse per + // conversation (the old unconditional path opened + parsed the + // newest message file on EVERY refresh, which dominated list-load + // time on SPI LittleFS). + std::string preview; + double preview_ts = 0.0; + bool have_preview = + _message_store->get_last_message_preview(peer_hash, preview, preview_ts); + bool need_metadata = !have_preview; + bool queued_drops = false; - if (!last_meta.valid) { + ::LXMF::MessageStore::MessageMetadata last_meta; + last_meta.valid = false; + if (need_metadata) { + last_meta = _message_store->load_message_metadata(last_msg_hash); + } + + if (need_metadata && !last_meta.valid) { // The newest message is corrupt/unreadable. Don't hide the // conversation over one bad message: drop the unreadable // message(s) and preview the newest readable one instead. // // This only costs the get_messages_for_conversation() array // copy and the extra metadata reads on the corrupt path — the - // common (all-readable) path above stays O(1) + one read. + // common (cached) path above stays O(1) + no I/O. // // delete_message() commits the index and removes files, so it // must not run under this LVGL lock: we only queue the hashes @@ -259,6 +274,7 @@ void ConversationListScreen::refresh() { std::vector hashes = _message_store->get_messages_for_conversation(peer_hash); request_drop_message(last_msg_hash); // tail is the known-bad one + queued_drops = true; // Walk newest→oldest over all but the tail (array is // chronological; hashes.back() is the tail we just queued) to // find the newest readable preview, queuing any unreadable @@ -271,6 +287,7 @@ void ConversationListScreen::refresh() { break; } request_drop_message(hashes[i]); + queued_drops = true; } if (!last_meta.valid) { // Every message was unreadable; all queued for drop. Skip @@ -279,6 +296,19 @@ void ConversationListScreen::refresh() { } } + // Re-pop the preview cache from the metadata we just read (the + // next refresh becomes O(1) with no I/O). Only meaningful on the + // fallback path — when the cache was already serving us there is + // nothing new to write. Skipped when drops were queued: the + // drained deletes update last_message_hash, so a preview written + // for the old tail would be stale for one refresh — let the next + // fallback re-read the converged tail instead. + if (need_metadata && last_meta.valid && !last_meta.content.empty() + && !queued_drops) { + _message_store->set_last_message_preview(peer_hash, + last_meta.content.substr(0, 47)); + } + // Create conversation item ConversationItem item; item.peer_hash = peer_hash; @@ -317,15 +347,15 @@ void ConversationListScreen::refresh() { _has_unresolved_names = true; } - // Get message content for preview (metadata path returns the - // pre-extracted UTF-8 content — no msgpack unpacking) - String content(last_meta.content.c_str()); + // Get message content for preview (index cache, or pre-extracted + // metadata on the fallback path — no msgpack unpacking) + String content(have_preview ? preview.c_str() : last_meta.content.c_str()); item.last_message = content.substring(0, 30); // Truncate to 30 chars if (content.length() > 30) { item.last_message += "..."; } - item.timestamp = (uint32_t)last_meta.timestamp; + item.timestamp = (uint32_t)(have_preview ? preview_ts : last_meta.timestamp); item.timestamp_str = format_timestamp(item.timestamp); // Unread count straight from the conversation index (persisted, // incremented on incoming save, cleared when the user opens the diff --git a/platformio.ini b/platformio.ini index 02b4a4d3..ad5702b1 100644 --- a/platformio.ini +++ b/platformio.ini @@ -122,7 +122,11 @@ lib_deps = ; — the conversation list reads the newest hash from the index tail ; instead of copying the full 8KB hash array, and renders the real ; persisted unread badge. - https://github.com/torlando-tech/microLXMF.git#df1b01a21c27b11b2cbd1674c7f8b648b76345ae + ; c8d3156 (feat/conversation-preview-cache, microLXMF PR #11): last-message + ; preview + timestamp cached per conversation in the index — list refresh + ; becomes O(1) in-memory with zero message-file reads (one fallback + ; write-through per conversation when the index predates the field). + https://github.com/torlando-tech/microLXMF.git#c8d3156907287052ded8574dcce6e8a87bbb3411 lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 diff --git a/tests/microlxmf/CMakeLists.txt b/tests/microlxmf/CMakeLists.txt index c15abfaf..ac1c0c73 100644 --- a/tests/microlxmf/CMakeLists.txt +++ b/tests/microlxmf/CMakeLists.txt @@ -41,3 +41,29 @@ target_compile_options(test_pyxis_telemetry_field PRIVATE target_link_options(test_pyxis_telemetry_field PRIVATE -fsanitize=address,undefined ) + +# Host-side benchmark of the conversation-list store load path (the per- +# conversation work ConversationListScreen::refresh() performs). Builds +# against the exact pinned microLXMF source, so before/after numbers are +# comparable across pin bumps. BENCH_INDEX_ACCESSORS is added by the +# harness only when the pinned store supports in-memory preview accessors. +add_executable(bench_conversation_list_load + bench_conversation_list_load.cpp +) +find_package(Threads REQUIRED) +target_link_libraries(bench_conversation_list_load PRIVATE MicroLXMFLib Threads::Threads) +target_compile_options(bench_conversation_list_load PRIVATE + -fexceptions + -Wall -Wno-unused-parameter -Wno-missing-field-initializers +) +target_compile_definitions(bench_conversation_list_load PRIVATE + MICROLXMF_NATIVE=1 + NATIVE=1 +) +# BENCH_INDEX_ACCESSORS=1 only when the pinned store exposes the in-memory +# preview accessors (get_last_message_preview); otherwise the binary reports +# that it is baseline-only. +option(BENCH_INDEX_ACCESSORS "pin supports in-memory preview accessors" OFF) +if(BENCH_INDEX_ACCESSORS) + target_compile_definitions(bench_conversation_list_load PRIVATE BENCH_INDEX_ACCESSORS=1) +endif() diff --git a/tests/microlxmf/bench_conversation_list_load.cpp b/tests/microlxmf/bench_conversation_list_load.cpp new file mode 100644 index 00000000..c415e70e --- /dev/null +++ b/tests/microlxmf/bench_conversation_list_load.cpp @@ -0,0 +1,328 @@ +// Benchmark: conversation-list store load path (the work ConversationListScreen::refresh() +// does per conversation), compiled against the EXACT pinned microLXMF source in .pio/libdeps. +// +// Modes: +// baseline : get_conversations() + per-conversation get_last_message_hash() + +// load_message_metadata(last_hash) + get_display_name() + +// get_conversation_unread_count() (== PR #95 refresh() I/O) +// index : same, but preview/timestamp come from in-memory index accessors when +// available (compiled only when the pinned store supports them) +// +// Prints median-of-iterations wall time per mode so a before/after table can be built. + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using LXMF::MessageStore; +using LXMF::LXMessage; +using RNS::Bytes; + +// ---------- prefixing filesystem (mirror of test_messagestore_tiers) ---------- +namespace bench_fs { + +class PrefixedFileImpl : public microStore::FileImpl { +private: + int _fd; + bool _closed; +public: + explicit PrefixedFileImpl(int fd) : microStore::FileImpl(), _fd(fd), _closed(false) {} + ~PrefixedFileImpl() override { if (!_closed) close(); } + const char* name() const override { return ""; } + size_t size() const override { struct stat st; ::fstat(_fd, &st); return st.st_size; } + void close() override { if (!_closed) { ::close(_fd); _closed = true; } } + int read() override { uint8_t b; if (::read(_fd, &b, 1) != 1) return -1; return b; } + int peek() override { return -1; } + size_t read(uint8_t* buf, size_t sz) override { + ssize_t n = ::read(_fd, buf, sz); return n < 0 ? 0 : (size_t)n; + } + size_t write(uint8_t b) override { return ::write(_fd, &b, 1) == 1 ? 1 : 0; } + size_t write(const uint8_t* buf, size_t sz) override { + ssize_t n = ::write(_fd, buf, sz); return n < 0 ? 0 : (size_t)n; + } + int available() override { return 0; } + size_t tell() override { return ::lseek(_fd, 0, SEEK_CUR); } + long seek(uint32_t pos, microStore::SeekMode m) override { + int wh = SEEK_SET; + if (m == microStore::SeekMode::SeekModeCur) wh = SEEK_CUR; + else if (m == microStore::SeekMode::SeekModeEnd) wh = SEEK_END; + return ::lseek(_fd, pos, wh); + } + void flush() override {} + bool isValid() const override { return !_closed && _fd >= 0; } +}; + +class PrefixedFSImpl : public microStore::FileSystemImpl { +private: + std::string _prefix; + std::string fp(const char* path) const { return _prefix + path; } + void mkparents(const std::string& path) const { + size_t p = 0; + while ((p = path.find('/', p + 1)) != std::string::npos) { + std::string dir = path.substr(0, p); + ::mkdir(dir.c_str(), 0755); + } + } +public: + explicit PrefixedFSImpl(const std::string& prefix) : _prefix(prefix) { + ::mkdir(prefix.c_str(), 0755); + } + bool init(bool) override { return true; } + bool format() override { return false; } + microStore::File open(const char* path, microStore::File::Mode mode, + const bool create = false) override { + std::string full = fp(path); + int flags; + switch (mode) { + case microStore::File::ModeRead: flags = O_RDONLY; break; + case microStore::File::ModeWrite: flags = O_WRONLY|O_CREAT|O_TRUNC; break; + case microStore::File::ModeAppend: flags = O_WRONLY|O_CREAT|O_APPEND; break; + case microStore::File::ModeReadWrite: flags = O_RDWR|O_CREAT|O_TRUNC; break; + case microStore::File::ModeReadAppend: flags = O_RDWR|O_CREAT|O_APPEND; break; + default: return {}; + } + if (flags & O_CREAT) mkparents(full); + int fd = ::open(full.c_str(), flags, 0644); + if (fd == -1) return {}; + return microStore::File(new PrefixedFileImpl(fd)); + } + bool exists(const char* path) override { struct stat st; return ::stat(fp(path).c_str(), &st) == 0; } + bool remove(const char* path) override { return ::unlink(fp(path).c_str()) == 0; } + bool rename(const char* a, const char* b) override { return ::rename(fp(a).c_str(), fp(b).c_str()) == 0; } + bool mkdir(const char* path) override { + std::string full = fp(path); mkparents(full); + return ::mkdir(full.c_str(), 0755) == 0 || errno == EEXIST; + } + bool rmdir(const char* path) override { return ::rmdir(fp(path).c_str()) == 0; } + size_t size(const char* path) override { struct stat st; if (::stat(fp(path).c_str(), &st) != 0) return 0; return st.st_size; } + bool isDirectory(const char* path) override { + struct stat st; if (::stat(fp(path).c_str(), &st) != 0) return false; + return S_ISDIR(st.st_mode); + } + std::list listDirectory(const char* path, + Callbacks::DirectoryListing cb = nullptr) override { + std::list out; + DIR* d = ::opendir(fp(path).c_str()); + if (!d) return out; + while (auto* ent = ::readdir(d)) { + if (ent->d_name[0] == '.') continue; + if (cb) cb(ent->d_name); else out.push_back(ent->d_name); + } + ::closedir(d); + return out; + } + size_t storageSize() override { return 0; } + size_t storageAvailable() override { return 0; } +}; + +class PrefixedFS : public microStore::FileSystem { +public: + explicit PrefixedFS(const std::string& prefix) + : microStore::FileSystem(new PrefixedFSImpl(prefix)) {} +}; + +} // namespace bench_fs + +// ---------- synthetic messages (mirror of test_messagestore_tiers) ---------- +static Bytes make_msgpack_payload(double timestamp, + const std::string& title, + const std::string& content) { + Bytes p; + p.append((uint8_t)0x94); // arr_size 4 + p.append((uint8_t)0xcb); + union { double d; uint64_t u; } cv; + cv.d = timestamp; + for (int i = 7; i >= 0; --i) p.append((uint8_t)((cv.u >> (i * 8)) & 0xff)); + p.append((uint8_t)0xc4); + p.append((uint8_t)title.size()); + p.append((const uint8_t*)title.data(), title.size()); + if (content.size() < 256) { + p.append((uint8_t)0xc4); + p.append((uint8_t)content.size()); + } else { + p.append((uint8_t)0xc5); + p.append((uint8_t)((content.size() >> 8) & 0xff)); + p.append((uint8_t)(content.size() & 0xff)); + } + p.append((const uint8_t*)content.data(), content.size()); + p.append((uint8_t)0x80); // empty fields map + return p; +} + +static LXMessage make_message(const Bytes& peer_hash, const Bytes& self_hash, + double timestamp, const std::string& content, + bool incoming) { + // save_message() keys a conversation by source for incoming, + // destination for outgoing — set the endpoints so the conversation + // always lands under peer_hash regardless of direction. + const Bytes& dest = incoming ? self_hash : peer_hash; + const Bytes& src = incoming ? peer_hash : self_hash; + Bytes raw; + raw.append(dest.data(), 16); + raw.append(src.data(), 16); + for (int i = 0; i < 64; ++i) raw.append((uint8_t)0); + Bytes payload = make_msgpack_payload(timestamp, "t", content); + raw.append(payload.data(), payload.size()); + LXMessage m = LXMessage::unpack_from_bytes(raw, LXMF::Type::Message::DIRECT, true); + m.incoming(incoming); + return m; +} + +// ---------- seed a store shaped like a real T-Deck: N conversations ---------- +static void seed_store(MessageStore& store, int convs, int msgs_per_conv) { + double base_ts = 1750000000.0; + for (int c = 0; c < convs; ++c) { + Bytes peer(16), self(16); + for (int i = 0; i < 16; ++i) { + peer.append((uint8_t)(0xa0 + c * 7 + i)); + self.append((uint8_t)(0x50 + i)); + } + store.set_display_name(peer, "peer_" + std::to_string(c)); + for (int m = 0; m < msgs_per_conv; ++m) { + // ~240-char content: realistic chat text plus a tail so previews + // exercise truncation the same way the list does. + std::string content = + "Check in when you land, the uplink window was tight today "; + while (content.size() < 200) content += "and the router was hopping paths. "; + content = content.substr(0, 220); + double ts = base_ts + (c * 5000 + m * 3600); + LXMessage msg = make_message(peer, self, ts, content, m % 3 != 0); + if (!store.save_message(msg)) { + std::cerr << "seed save failed c=" << c << " m=" << m << "\n"; + std::exit(2); + } + } + } +} + +// ---------- the two load paths ---------- +// baseline == exactly the per-conversation work of PR #95 refresh() +static double run_baseline(MessageStore& store) { + using Clock = std::chrono::steady_clock; + auto t0 = Clock::now(); + std::vector peers = store.get_conversations(); + for (const auto& peer : peers) { + Bytes last = store.get_last_message_hash(peer); + if (!last) continue; + MessageStore::MessageMetadata meta = store.load_message_metadata(last); + if (!meta.valid) continue; + std::string preview = meta.content.substr(0, 30); // the 30-char preview + double ts = meta.timestamp; + std::string name = store.get_display_name(peer); + size_t unread = store.get_conversation_unread_count(peer); + (void)preview; (void)ts; (void)name; (void)unread; + } + auto t1 = Clock::now(); + return std::chrono::duration(t1 - t0).count(); +} + +#ifdef BENCH_INDEX_ACCESSORS +// index == store provides in-memory per-conversation preview/timestamp; no +// message-file I/O on the hot path. +static double run_index(MessageStore& store) { + using Clock = std::chrono::steady_clock; + auto t0 = Clock::now(); + std::vector peers = store.get_conversations(); + for (const auto& peer : peers) { + Bytes last = store.get_last_message_hash(peer); + if (!last) continue; + std::string preview; + double ts = 0.0; + bool from_index = store.get_last_message_preview(peer, preview, ts); + MessageStore::MessageMetadata meta; + meta.valid = false; + if (!from_index) { + meta = store.load_message_metadata(last); // fallback (no cached preview) + if (!meta.valid) continue; + preview = meta.content.substr(0, 30); + ts = meta.timestamp; + } + std::string name = store.get_display_name(peer); + size_t unread = store.get_conversation_unread_count(peer); + (void)preview; (void)ts; (void)name; (void)unread; + } + auto t1 = Clock::now(); + return std::chrono::duration(t1 - t0).count(); +} +#endif + +static double median(std::vector v) { + std::sort(v.begin(), v.end()); + if (v.empty()) return 0.0; + return v[v.size() / 2]; +} + +int main(int argc, char** argv) { + std::string root = argc > 1 ? argv[1] : "/tmp/bench-convlist"; + int convs = argc > 2 ? std::atoi(argv[2]) : 9; + int msgs = argc > 3 ? std::atoi(argv[3]) : 24; + int iters = argc > 4 ? std::atoi(argv[4]) : 25; + + ::system(("rm -rf " + root + " && mkdir -p " + root + "/hot").c_str()); + + bench_fs::PrefixedFS hot_fs(root + "/hot"); + RNS::Utilities::OS::register_filesystem(hot_fs); + + std::vector baseline_ms; + std::vector index_warm_ms; + std::vector index_cold_ms; + + { + MessageStore store("/lxmf"); + seed_store(store, convs, msgs); + for (int i = 0; i < iters; ++i) baseline_ms.push_back(run_baseline(store)); +#ifdef BENCH_INDEX_ACCESSORS + // Warm path: the store the app holds right now (in-memory index + // includes the preview cache). + for (int i = 0; i < iters; ++i) index_warm_ms.push_back(run_index(store)); +#endif + } // drop the in-memory store (reboot shape) + +#ifdef BENCH_INDEX_ACCESSORS + { + // Cold-boot path: reconstruct the store from disk (reboot / fresh + // firmware boot). The index on disk now carries last_preview (new + // pin), so the first refresh still hits the cache with zero + // message-file reads — that is the whole point of persisting it. + MessageStore store("/lxmf"); + for (int i = 0; i < iters; ++i) index_cold_ms.push_back(run_index(store)); + } +#endif + + std::cout << "convlist-bench convs=" << convs << " msgs_per_conv=" << msgs + << " iters=" << iters << "\n"; + std::cout << "baseline_ms=" << median(baseline_ms) + << " (min=" << *std::min_element(baseline_ms.begin(), baseline_ms.end()) + << " max=" << *std::max_element(baseline_ms.begin(), baseline_ms.end()) << ")\n"; +#ifdef BENCH_INDEX_ACCESSORS + std::cout << "index_warm_ms=" << median(index_warm_ms) + << " (min=" << *std::min_element(index_warm_ms.begin(), index_warm_ms.end()) + << " max=" << *std::max_element(index_warm_ms.begin(), index_warm_ms.end()) << ")\n"; + std::cout << "index_cold_ms=" << median(index_cold_ms) + << " (min=" << *std::min_element(index_cold_ms.begin(), index_cold_ms.end()) + << " max=" << *std::max_element(index_cold_ms.begin(), index_cold_ms.end()) << ")\n"; +#endif + std::cout << "convlist-bench: done\n"; + return 0; +} diff --git a/tests/reference/test_microlxmf_telemetry_field.py b/tests/reference/test_microlxmf_telemetry_field.py index dbdc5b28..1288d785 100644 --- a/tests/reference/test_microlxmf_telemetry_field.py +++ b/tests/reference/test_microlxmf_telemetry_field.py @@ -22,7 +22,7 @@ def test_pinned_microlxmf_telemetry_field_roundtrip(tmp_path): if not (source / "LXMF" / "LXMessage.cpp").is_file(): pytest.fail(f"pinned microLXMF source is unavailable at {microlxmf}") - expected = "5a44862b85e902f351b756ee00482f3258b244f5" + expected = "c8d3156907287052ded8574dcce6e8a87bbb3411" actual = subprocess.run( ["git", "-C", str(microlxmf), "rev-parse", "HEAD"], capture_output=True, diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index f5e90750..3035048f 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "df1b01a21c27b11b2cbd1674c7f8b648b76345ae", + "microLXMF": "c8d3156907287052ded8574dcce6e8a87bbb3411", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From ff79a87e425b9e8a147a4654f8bff42aeba88e34 Mon Sep 17 00:00:00 2001 From: Pike Date: Thu, 3 Sep 2026 15:20:24 +0000 Subject: [PATCH 06/13] perf(messages): revalidate conversation rows, skip LVGL rebuild when unchanged refresh() unconditionally ran lv_obj_clean(_list) and recreated 5-7 LVGL objects per row on every call - every navigation back to Messages, every 750ms coalesced inbound batch, and the periodic name-resolution sweep - even when nothing had changed. With the store read now O(1) (index preview cache), that widget churn was the remaining gap vs NomadNet / Network / Maps, which build once and just unhide. refresh() now gathers row data first (index preview + hash + unread, no message-file I/O) and diffs it against the rendered rows; the rebuild only happens when data actually changes. Rows keep focus-group membership and the screen keeps its scroll position on the no-change path. The per-refresh 'Found N conversations' log drops from INFO to DEBUG (serial output under the LVGL lock stalls the render task). get_conversations() ordering is deterministic (last_activity desc with peer-hash tie-break), so the index-aligned comparison is stable. --- .../UI/LXMF/ConversationListScreen.cpp | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 05d27603..171d8aa9 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -198,29 +198,29 @@ void ConversationListScreen::refresh() { return; } - INFO("Refreshing conversation list"); + // Revalidate-then-render: gather the row data first (now O(1) per + // conversation — index preview + hash + unread, no message-file I/O) + // and compare it against what is already on screen. The old path + // unconditionally ran lv_obj_clean(_list) + recreated 5-7 LVGL objects + // per row on EVERY refresh (navigation back to Messages, every + // 750ms coalesced inbound batch, name-resolution sweep) — that + // widget churn was what still made Messages feel slow vs NomadNet / + // Network / Maps, which build once and just unhide. + std::vector gathered; - // Clear existing items (also removes from focus group when deleted) - lv_obj_clean(_list); - _conversations.clear(); - _conversation_containers.clear(); - _badge_pool.clear(); - _peer_hash_pool.clear(); _has_unresolved_names = false; // Load conversations from store std::vector peer_hashes = _message_store->get_conversations(); - - // Reserve capacity to avoid reallocations during population - _peer_hash_pool.reserve(peer_hashes.size()); - _conversations.reserve(peer_hashes.size()); - _conversation_containers.reserve(peer_hashes.size()); - _badge_pool.reserve(peer_hashes.size()); + gathered.reserve(peer_hashes.size()); { char log_buf[48]; snprintf(log_buf, sizeof(log_buf), " Found %zu conversations", peer_hashes.size()); - INFO(log_buf); + // DEBUG, not INFO: refresh() now runs on every navigation and + // periodic sweep, and serial output happens under the LVGL lock + // (the render task waits on the same lock the serial flush holds). + DEBUG(log_buf); } for (const auto& peer_hash : peer_hashes) { @@ -363,6 +363,53 @@ void ConversationListScreen::refresh() { item.unread_count = (uint16_t)_message_store->get_conversation_unread_count(peer_hash); + gathered.push_back(item); + } + + // Revalidate: skip the LVGL rebuild entirely when nothing changed. + // Rows keep their focus-group membership and the screen keeps its + // scroll position, and the render task costs only the O(1) gather + // above. This is the common case: navigation back to Messages with + // no new traffic, 750ms coalesced refreshes on an idle link, and + // the update_status() name-resolution sweep. + // + // (Badge edge case: a clicked row has its badge widget deleted and a + // mark-read queued; the mark-read flush runs in UIManager::update() + // BEFORE any later refresh takes the lock, so a refresh in that + // window sees the data unchanged (badge stays off) and the first + // refresh after the flush sees unread==0 and rebuilds correctly.) + bool changed = (gathered.size() != _conversations.size()); + if (!changed) { + for (size_t i = 0; i < gathered.size(); ++i) { + const ConversationItem& g = gathered[i]; + const ConversationItem& c = _conversations[i]; + if (g.peer_hash != c.peer_hash || + g.peer_name != c.peer_name || + g.last_message != c.last_message || + g.timestamp != c.timestamp || + g.unread_count != c.unread_count) { + changed = true; + break; + } + } + } + if (!changed) { + return; // rows on screen are current + } + + // Data changed: rebuild the list (full rebuild — a partial per-row + // diff is not worth the extra bookkeeping for a list this size). + lv_obj_clean(_list); + _conversations.clear(); + _conversation_containers.clear(); + _badge_pool.clear(); + _peer_hash_pool.clear(); + _peer_hash_pool.reserve(gathered.size()); + _conversations.reserve(gathered.size()); + _conversation_containers.reserve(gathered.size()); + _badge_pool.reserve(gathered.size()); + + for (const ConversationItem& item : gathered) { _conversations.push_back(item); create_conversation_item(item); } From af4019f933a9c1026a90290093f2ebce3d3ae9f9 Mon Sep 17 00:00:00 2001 From: Pike Date: Thu, 3 Sep 2026 16:02:35 +0000 Subject: [PATCH 07/13] diag(messages): [PERF] stage timings for tap-to-visible path Temporary, removable instrumentation (marked [PERF] throughout): - refresh(): gather / diff / rebuild stage ms + metadata-fallback count, one [PERF] convlist line per call (skip vs build) - show(): unhide + focus-group ms - render_route(MESSAGES): whole route window incl. hide_all_screens All INFO-level so they survive DEBUG-off and land in the serial capture. --- .../UI/LXMF/ConversationListScreen.cpp | 38 +++++++++++++++++++ lib/tdeck_ui/UI/LXMF/UIManager.cpp | 9 ++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 171d8aa9..801220fd 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -198,6 +198,13 @@ void ConversationListScreen::refresh() { return; } + // [PERF] temporary instrumentation — removable. Stage timings for the + // Messages tap path. Kept as INFO so it survives a DEBUG-off build and + // lands in the serial capture. + const uint32_t p_t0 = millis(); + uint32_t p_t_gather = 0, p_t_diff = 0, p_t_rebuild = 0; + size_t p_fallbacks = 0; + // Revalidate-then-render: gather the row data first (now O(1) per // conversation — index preview + hash + unread, no message-file I/O) // and compare it against what is already on screen. The old path @@ -249,6 +256,7 @@ void ConversationListScreen::refresh() { _message_store->get_last_message_preview(peer_hash, preview, preview_ts); bool need_metadata = !have_preview; bool queued_drops = false; + if (need_metadata) p_fallbacks++; ::LXMF::MessageStore::MessageMetadata last_meta; last_meta.valid = false; @@ -378,6 +386,7 @@ void ConversationListScreen::refresh() { // BEFORE any later refresh takes the lock, so a refresh in that // window sees the data unchanged (badge stays off) and the first // refresh after the flush sees unread==0 and rebuilds correctly.) + p_t_gather = millis() - p_t0; bool changed = (gathered.size() != _conversations.size()); if (!changed) { for (size_t i = 0; i < gathered.size(); ++i) { @@ -393,7 +402,15 @@ void ConversationListScreen::refresh() { } } } + p_t_diff = millis() - p_t0; if (!changed) { + // [PERF] temporary instrumentation — removable. + char perf_buf[128]; + snprintf(perf_buf, sizeof(perf_buf), + "[PERF] convlist skip rows=%u fallbacks=%u gather=%ums diff=%ums total=%ums", + (unsigned)gathered.size(), (unsigned)p_fallbacks, + (unsigned)p_t_gather, (unsigned)p_t_diff, (unsigned)(p_t_diff - p_t0)); + INFO(perf_buf); return; // rows on screen are current } @@ -413,6 +430,7 @@ void ConversationListScreen::refresh() { _conversations.push_back(item); create_conversation_item(item); } + p_t_rebuild = millis() - p_t0; // refresh() may run while Messages remains active. Newly created rows are // not automatically restored to the encoder group after lv_obj_clean(). @@ -424,6 +442,16 @@ void ConversationListScreen::refresh() { } } } + + // [PERF] temporary instrumentation — removable. + { + char perf_buf[128]; + snprintf(perf_buf, sizeof(perf_buf), + "[PERF] convlist build rows=%u fallbacks=%u gather=%ums rebuild=%ums total=%ums", + (unsigned)gathered.size(), (unsigned)p_fallbacks, + (unsigned)p_t_gather, (unsigned)p_t_rebuild, (unsigned)(millis() - p_t0)); + INFO(perf_buf); + } } void ConversationListScreen::flush_pending_name_writes() { @@ -604,6 +632,8 @@ void ConversationListScreen::set_home_callback(HomeCallback callback) { void ConversationListScreen::show() { LVGL_LOCK(); + // [PERF] temporary instrumentation — removable. + const uint32_t p_show_start = millis(); _visible = true; lv_obj_clear_flag(_screen, LV_OBJ_FLAG_HIDDEN); lv_obj_move_foreground(_screen); // Bring to front for touch events @@ -631,6 +661,14 @@ void ConversationListScreen::show() { lv_group_focus_obj(_btn_new); } } + + // [PERF] temporary instrumentation — removable. + { + char perf_buf[64]; + snprintf(perf_buf, sizeof(perf_buf), "[PERF] convlist show=%ums", + (unsigned)(millis() - p_show_start)); + INFO(perf_buf); + } } void ConversationListScreen::hide() { diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index f37645c5..b49dfd48 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -1114,12 +1114,19 @@ void UIManager::render_route(Route route) { case Route::HOME: _home_screen->show(); break; - case Route::MESSAGES: + case Route::MESSAGES: { + // [PERF] temporary instrumentation — removable. + const uint32_t p_route_start = millis(); _conversation_list_screen->refresh(); _pending_conversation_refresh = false; _last_conversation_refresh_ms = millis(); _conversation_list_screen->show(); + char perf_buf[64]; + snprintf(perf_buf, sizeof(perf_buf), "[PERF] convlist route=%ums", + (unsigned)(millis() - p_route_start)); + INFO(perf_buf); break; + } case Route::MAP: _map_screen->show(); break; From 8204506b3bfb314571878f00be4fa1baf1187c9b Mon Sep 17 00:00:00 2001 From: Pike Date: Thu, 3 Sep 2026 17:12:39 +0000 Subject: [PATCH 08/13] perf(messages): persist preview warm-up; cache empty-content tails On-device [PERF] capture decomposed the remaining ~1s tap latency: each uncached conversation costs one SPI-LittleFS metadata read (~230ms). Repeat taps still paid 2 perpetual fallbacks (empty- content tails never got cached), and the first tap after every boot paid all 9 (the in-memory repop was never committed to the index). - bump microLXMF pin c8d3156 -> 6bea23c (feat/conversation- preview-cache): preview_valid index flag so 'cached empty' differs from 'unpopulated'; bounded preview copy (the old strncpy read past the non-terminated content Bytes); public commit_index(). - refresh() re-pops empty-content tails as a valid cached preview and arms a one-shot deferred index commit; UIManager::update() drains it out-of-lock (flush_pending_ index_commit, between drops and mark-read) so the warmed previews persist and the next cold boot reads them from the index. - fix the [PERF] skip-log total= wrap (printed p_t_diff - p_t0, a uint32 underflow; total was already p_t_diff). [PERF] instrumentation stays in this commit (temporary, marked); it is removed before merge once the fix is validated on-device. --- .gitignore | 3 ++ .../UI/LXMF/ConversationListScreen.cpp | 37 +++++++++++++++---- lib/tdeck_ui/UI/LXMF/ConversationListScreen.h | 19 ++++++++++ lib/tdeck_ui/UI/LXMF/UIManager.cpp | 4 ++ platformio.ini | 8 +++- .../test_microlxmf_telemetry_field.py | 2 +- tools/audit_release_build.py | 2 +- 7 files changed, 65 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index e4450103..ef17fbf9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ docs/flasher/firmware/ # is newer (see should_regenerate), so contributors with the deps # installed get fresh rebuilds while contributors without them still # get a working binary from the committed copy. + +# IDE-generated build database +compile_commands.json diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 801220fd..e8e7aaab 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -307,14 +307,19 @@ void ConversationListScreen::refresh() { // Re-pop the preview cache from the metadata we just read (the // next refresh becomes O(1) with no I/O). Only meaningful on the // fallback path — when the cache was already serving us there is - // nothing new to write. Skipped when drops were queued: the - // drained deletes update last_message_hash, so a preview written - // for the old tail would be stale for one refresh — let the next - // fallback re-read the converged tail instead. - if (need_metadata && last_meta.valid && !last_meta.content.empty() - && !queued_drops) { + // nothing new to write. An EMPTY content re-pops as a valid + // cached empty preview (the store marks it populated), so + // empty-content tails — location shares, blank pings — also stop + // falling back. The one-shot index commit below persists the + // repop so a reboot does not re-pay every fallback. Skipped + // when drops were queued: the drained deletes update + // last_message_hash, so a preview written for the old tail would + // be stale for one refresh — let the next fallback re-read the + // converged tail instead. + if (need_metadata && last_meta.valid && !queued_drops) { _message_store->set_last_message_preview(peer_hash, last_meta.content.substr(0, 47)); + _index_commit_pending = true; } // Create conversation item @@ -409,7 +414,7 @@ void ConversationListScreen::refresh() { snprintf(perf_buf, sizeof(perf_buf), "[PERF] convlist skip rows=%u fallbacks=%u gather=%ums diff=%ums total=%ums", (unsigned)gathered.size(), (unsigned)p_fallbacks, - (unsigned)p_t_gather, (unsigned)p_t_diff, (unsigned)(p_t_diff - p_t0)); + (unsigned)p_t_gather, (unsigned)p_t_diff, (unsigned)p_t_diff); INFO(perf_buf); return; // rows on screen are current } @@ -597,6 +602,24 @@ void ConversationListScreen::flush_pending_drops() { _pending_drops.swap(remaining); } +void ConversationListScreen::flush_pending_index_commit() { + // Called from UIManager::update() BEFORE it takes the LVGL lock, + // right after draining drops. Commits the persisted conversation + // index once per boot when refresh() had to fall back to + // load_message_metadata() (unpopulated preview cache): the in-memory + // repop would otherwise be lost on reboot and the next cold boot + // would re-read every newest message file again. save_index() + // rewrites the whole index, so it also persists any preview cache + // writes made by save_message() since the last store-originated + // commit — one-shot by design; the flag clears even if the commit + // fails and a later fallback re-arms it. + if (!_message_store || !_index_commit_pending) { + return; + } + _message_store->commit_index(); + _index_commit_pending = false; +} + void ConversationListScreen::clear_unread_badge(lv_obj_t* container) { // Remove the unread badge from a rendered conversation row. The badge // is tracked in _badge_pool in lockstep with _conversation_containers, diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h index 541f4ed3..995b26ef 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h @@ -134,6 +134,19 @@ public: */ void flush_pending_drops(); + /** + * Commit the persisted conversation index once when this boot has + * fallen back to message-file reads (the one-shot preview-cache + * warm-up: cold boot after a firmware/index upgrade, or a tail + * cleared by a delete). The in-memory repop alone is lost on + * reboot, so without this commit the first list refresh after + * EVERY boot re-reads every newest message file. MUST be called + * OUTSIDE the LVGL lock (UIManager::update() calls it after + * draining drops, before mark-read, so all three land in one + * sensible order — each save_index() rewrites the whole index). + */ + void flush_pending_index_commit(); + /** * Set callback for conversation selection * @param callback Function to call when conversation is selected @@ -257,6 +270,12 @@ private: // before taking the lock so the index converges to the newest // readable message. std::vector _pending_drops; + // Set by refresh() when at least one conversation fell back to + // load_message_metadata() this boot (unpopulated preview cache: + // cold boot on a pre-c8d3156-generation index, or a tail cleared by + // delete). flush_pending_index_commit() persists the repop once so + // the next boot serves previews straight from the index. + bool _index_commit_pending = false; ConversationSelectedCallback _conversation_selected_callback; ComposeCallback _compose_callback; diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index b49dfd48..67f1fe6c 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -724,6 +724,10 @@ void UIManager::update() { // sensible order — each save_index() rewrites the whole index, // and mark-read's final write carries the cleared unread count. _conversation_list_screen->flush_pending_drops(); + // One-shot preview-cache warm-up commit (index carries + // last_preview now; see the method). Lands before mark-read so + // that final index write also carries the warmed previews. + _conversation_list_screen->flush_pending_index_commit(); // Commit queued mark-read index writes the same way — the LVGL // click handler only recorded the peer hash. _conversation_list_screen->flush_pending_mark_reads(); diff --git a/platformio.ini b/platformio.ini index ad5702b1..67ca7161 100644 --- a/platformio.ini +++ b/platformio.ini @@ -126,7 +126,13 @@ lib_deps = ; preview + timestamp cached per conversation in the index — list refresh ; becomes O(1) in-memory with zero message-file reads (one fallback ; write-through per conversation when the index predates the field). - https://github.com/torlando-tech/microLXMF.git#c8d3156907287052ded8574dcce6e8a87bbb3411 + ; e54a102 (same branch): preview copy bounded (no strncpy over-read past + ; the content Bytes); empty-content tails (location shares, blank pings) + ; cached as a valid empty preview via a preview_valid index flag — + ; callers fall back only when the cache is unpopulated. + ; 6bea23c (same branch): public commit_index() so the list's one-shot + ; preview warm-up persists to the index without a save/delete. + https://github.com/torlando-tech/microLXMF.git#6bea23c82b56088e75e1452adc545b3f2120e364 lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 diff --git a/tests/reference/test_microlxmf_telemetry_field.py b/tests/reference/test_microlxmf_telemetry_field.py index 1288d785..3902ad3c 100644 --- a/tests/reference/test_microlxmf_telemetry_field.py +++ b/tests/reference/test_microlxmf_telemetry_field.py @@ -22,7 +22,7 @@ def test_pinned_microlxmf_telemetry_field_roundtrip(tmp_path): if not (source / "LXMF" / "LXMessage.cpp").is_file(): pytest.fail(f"pinned microLXMF source is unavailable at {microlxmf}") - expected = "c8d3156907287052ded8574dcce6e8a87bbb3411" + expected = "6bea23c82b56088e75e1452adc545b3f2120e364" actual = subprocess.run( ["git", "-C", str(microlxmf), "rev-parse", "HEAD"], capture_output=True, diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index 3035048f..b4ebc2e1 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "c8d3156907287052ded8574dcce6e8a87bbb3411", + "microLXMF": "6bea23c82b56088e75e1452adc545b3f2120e364", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From 09bf277aa4d070df54a16aed67e4bd2f11baa566 Mon Sep 17 00:00:00 2001 From: Pike Date: Thu, 3 Sep 2026 17:46:38 +0000 Subject: [PATCH 09/13] chore(messages): remove temporary [PERF] instrumentation On-device capture confirmed the fix (cold-boot tap: fallbacks=0, gather=2ms, total=48ms vs 2086ms pre-fix; repeat taps no longer perpetually fall back). All [PERF] timing markers and stage variables are removed from refresh()/show()/render_route; the branch is now clean of diagnostic code. --- .../UI/LXMF/ConversationListScreen.cpp | 38 ------------------- lib/tdeck_ui/UI/LXMF/UIManager.cpp | 9 +---- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index e8e7aaab..a71f3dcb 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -198,13 +198,6 @@ void ConversationListScreen::refresh() { return; } - // [PERF] temporary instrumentation — removable. Stage timings for the - // Messages tap path. Kept as INFO so it survives a DEBUG-off build and - // lands in the serial capture. - const uint32_t p_t0 = millis(); - uint32_t p_t_gather = 0, p_t_diff = 0, p_t_rebuild = 0; - size_t p_fallbacks = 0; - // Revalidate-then-render: gather the row data first (now O(1) per // conversation — index preview + hash + unread, no message-file I/O) // and compare it against what is already on screen. The old path @@ -256,7 +249,6 @@ void ConversationListScreen::refresh() { _message_store->get_last_message_preview(peer_hash, preview, preview_ts); bool need_metadata = !have_preview; bool queued_drops = false; - if (need_metadata) p_fallbacks++; ::LXMF::MessageStore::MessageMetadata last_meta; last_meta.valid = false; @@ -391,7 +383,6 @@ void ConversationListScreen::refresh() { // BEFORE any later refresh takes the lock, so a refresh in that // window sees the data unchanged (badge stays off) and the first // refresh after the flush sees unread==0 and rebuilds correctly.) - p_t_gather = millis() - p_t0; bool changed = (gathered.size() != _conversations.size()); if (!changed) { for (size_t i = 0; i < gathered.size(); ++i) { @@ -407,15 +398,7 @@ void ConversationListScreen::refresh() { } } } - p_t_diff = millis() - p_t0; if (!changed) { - // [PERF] temporary instrumentation — removable. - char perf_buf[128]; - snprintf(perf_buf, sizeof(perf_buf), - "[PERF] convlist skip rows=%u fallbacks=%u gather=%ums diff=%ums total=%ums", - (unsigned)gathered.size(), (unsigned)p_fallbacks, - (unsigned)p_t_gather, (unsigned)p_t_diff, (unsigned)p_t_diff); - INFO(perf_buf); return; // rows on screen are current } @@ -435,7 +418,6 @@ void ConversationListScreen::refresh() { _conversations.push_back(item); create_conversation_item(item); } - p_t_rebuild = millis() - p_t0; // refresh() may run while Messages remains active. Newly created rows are // not automatically restored to the encoder group after lv_obj_clean(). @@ -447,16 +429,6 @@ void ConversationListScreen::refresh() { } } } - - // [PERF] temporary instrumentation — removable. - { - char perf_buf[128]; - snprintf(perf_buf, sizeof(perf_buf), - "[PERF] convlist build rows=%u fallbacks=%u gather=%ums rebuild=%ums total=%ums", - (unsigned)gathered.size(), (unsigned)p_fallbacks, - (unsigned)p_t_gather, (unsigned)p_t_rebuild, (unsigned)(millis() - p_t0)); - INFO(perf_buf); - } } void ConversationListScreen::flush_pending_name_writes() { @@ -655,8 +627,6 @@ void ConversationListScreen::set_home_callback(HomeCallback callback) { void ConversationListScreen::show() { LVGL_LOCK(); - // [PERF] temporary instrumentation — removable. - const uint32_t p_show_start = millis(); _visible = true; lv_obj_clear_flag(_screen, LV_OBJ_FLAG_HIDDEN); lv_obj_move_foreground(_screen); // Bring to front for touch events @@ -684,14 +654,6 @@ void ConversationListScreen::show() { lv_group_focus_obj(_btn_new); } } - - // [PERF] temporary instrumentation — removable. - { - char perf_buf[64]; - snprintf(perf_buf, sizeof(perf_buf), "[PERF] convlist show=%ums", - (unsigned)(millis() - p_show_start)); - INFO(perf_buf); - } } void ConversationListScreen::hide() { diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index 67f1fe6c..472e521c 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -1118,19 +1118,12 @@ void UIManager::render_route(Route route) { case Route::HOME: _home_screen->show(); break; - case Route::MESSAGES: { - // [PERF] temporary instrumentation — removable. - const uint32_t p_route_start = millis(); + case Route::MESSAGES: _conversation_list_screen->refresh(); _pending_conversation_refresh = false; _last_conversation_refresh_ms = millis(); _conversation_list_screen->show(); - char perf_buf[64]; - snprintf(perf_buf, sizeof(perf_buf), "[PERF] convlist route=%ums", - (unsigned)(millis() - p_route_start)); - INFO(perf_buf); break; - } case Route::MAP: _map_screen->show(); break; From 403035f1155053ce1c76d9a9371b70fe28a26e0a Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:30:16 +0000 Subject: [PATCH 10/13] perf(messages): instant chat open/reopen + deferred full-message view Uses the microLXMF in-process message-metadata cache (bumped pin 6bea23c -> 58a6eb0, branch feat/conversation-preview-cache): - Reopening a conversation (and background page fill / paging) is now O(1) in-memory once warmed instead of re-reading each message file from SPI LittleFS (~230ms/read, ~2.3s per open measured on the T-Deck). - The cache table is PSRAM-allocated on ESP32; a static .bss placement starved internal DRAM and the LVGL task's 8 KiB stack allocation failed at boot ("Failed to create LVGL task", hang at startup logo). - Long-press full-message view now defers to the main loop: the LVGL event handler only records the hash; tick_pending_full_message() does load_message_content() (uncapped, no msgpack unpack) off the LVGL task, then builds the modal. Fixes the capped (600-char) text that the in-memory rows carry. - A peer change cancels an in-flight background fill from the previous conversation (it would otherwise prepend the old conversation's rows into the new one). Validated on the T-Deck (T-Deck Plus, 8MB PSRAM): warm opens reads=3 sync=0-2ms total=15-35ms (was ~2.3s); cold first open per conversation still pays ~0.6-1.2s disk while warming the cache; no panics/watchdogs over a ~6-minute interaction session. Host gates: microLXMF conformance 6/6 (incl. 32-assertion test_message_metadata_cache), native reference test against pinned 58a6eb0, tdeck build SUCCESS. --- lib/tdeck_ui/UI/LXMF/ChatScreen.cpp | 85 ++++++++++++++----- lib/tdeck_ui/UI/LXMF/ChatScreen.h | 23 +++++ lib/tdeck_ui/UI/LXMF/UIManager.cpp | 3 + platformio.ini | 14 ++- .../test_microlxmf_telemetry_field.py | 2 +- tools/audit_release_build.py | 2 +- 6 files changed, 105 insertions(+), 24 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ChatScreen.cpp b/lib/tdeck_ui/UI/LXMF/ChatScreen.cpp index 36c7509a..c30bd9ae 100644 --- a/lib/tdeck_ui/UI/LXMF/ChatScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ChatScreen.cpp @@ -171,6 +171,15 @@ void ChatScreen::load_conversation(const Bytes& peer_hash, ::LXMF::MessageStore& _peer_hash = peer_hash; _message_store = &store; + // A peer change cancels any in-flight background fill from the + // previous conversation (it would otherwise prepend the previous + // conversation's older rows into the new one). refresh() re-arms the + // fill for the new peer. + if (_bg_fill_active.exchange(false)) { + _keep_bottom_during_background_fill.store(false); + _bg_fill_target = _display_start_idx; // fill is a no-op now + } + { char log_buf[64]; snprintf(log_buf, sizeof(log_buf), "Loading conversation with peer %.8s...", @@ -251,9 +260,9 @@ void ChatScreen::refresh() { for (size_t i = _display_start_idx; i < _all_message_hashes.size(); i++) { const auto& msg_hash = _all_message_hashes[i]; - // Use fast metadata loader (no msgpack unpacking) + // Use fast metadata loader (cache hit: O(1) in-memory; miss: one + // LittleFS read that warms the cache for every later touch). ::LXMF::MessageStore::MessageMetadata meta = _message_store->load_message_metadata(msg_hash); - if (!meta.valid) { continue; } @@ -335,6 +344,8 @@ void ChatScreen::load_more_messages(size_t batch) { --i; // Decrement first since we're iterating backwards const auto& msg_hash = _all_message_hashes[i]; + // Use fast metadata loader (cache hit: O(1) in-memory; miss: one + // LittleFS read that warms the cache for every later touch). ::LXMF::MessageStore::MessageMetadata meta = _message_store->load_message_metadata(msg_hash); if (!meta.valid) { continue; @@ -735,34 +746,66 @@ void ChatScreen::on_message_long_pressed(lv_event_t* event) { lv_obj_t* bubble = lv_event_get_target(event); lv_obj_t* row = lv_obj_get_parent(bubble); - // Bubbles render a truncated copy of long messages, so recover the FULL - // content for this row (reverse-lookup row -> hash -> stored item) for both - // the detail view and Copy. - String full; + // Bubbles render a display-capped copy (metadata cache / MAX_DISPLAY_ + // CHARS), so the full text must be recovered from the store. This + // handler runs on the LVGL task and the store is main-loop-only, so + // record the request and let tick_pending_full_message() (main loop) + // do the disk read + modal build. + RNS::Bytes hash; for (const auto& kv : screen->_message_rows) { if (kv.second == row) { - for (const auto& m : screen->_messages) { - if (m.message_hash == kv.first) { - full = m.content; - break; - } - } + hash = kv.first; break; } } - if (full.length() == 0) { // fallback to the (possibly truncated) label text - lv_obj_t* label = lv_obj_get_child(bubble, 0); - if (label) { - const char* t = lv_label_get_text(label); - if (t) full = t; - } + if (hash.size() == 0) { + return; } + if (screen->_pending_full_message.load()) { + // A request is already queued; the next modal supersedes this one. + return; + } + // Runs while the LVGL task holds the mutex (lvgl_task wraps + // lv_task_handler in it), so these writes serialize with the main + // loop's copy of _pending_full_message_hash under LVGL_LOCK. + screen->_pending_full_message_hash = hash; + screen->_pending_copy_text = ""; + screen->_pending_full_message.store(true); +} + +// Complete a pending long-press full-message request (see +// on_message_long_pressed). Runs on the main loop: the disk-bound +// load_message_content() happens OUTSIDE the LVGL lock (same task that +// does all other store I/O), then the modal is built under LVGL_LOCK(). +// This path is user-triggered (once per long press), not per frame. +void ChatScreen::tick_pending_full_message() { + if (!_pending_full_message.load()) { + return; + } + // Copy the hash under the LVGL lock: the LVGL task writes it while + // holding that same lock (see on_message_long_pressed), so this is a + // safe handoff. Then the disk read happens OUTSIDE the lock. + RNS::Bytes hash; + { + LVGL_LOCK(); + if (!_pending_full_message.load()) { + return; + } + hash = _pending_full_message_hash; + // Clear before the read: a superseding long-press can re-arm while + // we are here and must not be lost. + _pending_full_message.store(false); + } + if (!_message_store) { + return; + } + String full = String(_message_store->load_message_content(hash).c_str()); if (full.length() == 0) { return; } - - screen->_pending_copy_text = full; - screen->show_full_message(full); + LVGL_LOCK(); + _pending_copy_text = full; + show_full_message(full); } // Full-screen scrollable view of a single message's complete text, with Copy. diff --git a/lib/tdeck_ui/UI/LXMF/ChatScreen.h b/lib/tdeck_ui/UI/LXMF/ChatScreen.h index add3f4bd..6a9b5e66 100644 --- a/lib/tdeck_ui/UI/LXMF/ChatScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ChatScreen.h @@ -108,6 +108,21 @@ public: */ void tick_background_fill(); + /** + * Complete a pending long-press full-message request. Call from + * UIManager::update() on the main loop after the chat screen is shown. + * + * The long-press handler (LVGL task) only records which message was + * pressed; the actual full content read + * (MessageStore::load_message_content — one bounded JSON read, no + * msgpack unpack) and the modal build happen here, off the LVGL task. + * The store is main-loop-only (it shares one JsonDocument between + * save + load), and the rendered rows only hold the display-capped + * content (the metadata cache caps at 600 chars), so the full view + * must come from disk. + */ + void tick_pending_full_message(); + /** * Set callback for back button * @param callback Function to call when back button is pressed @@ -217,6 +232,14 @@ private: // user-triggered pagination at the top must preserve the user's position. std::atomic _keep_bottom_during_background_fill{false}; + // Long-press full-message view, deferred to the main loop. The LVGL + // event handler only records the hash; tick_pending_full_message() + // does the (disk-bound) load_message_content() off the LVGL task and + // builds the modal. Same set-before-arm ordering as the background + // fill above. + std::atomic _pending_full_message{false}; + RNS::Bytes _pending_full_message_hash; + // Load more messages (infinite scroll + background fill) void load_more_messages(size_t batch = MESSAGES_PER_PAGE); void scroll_to_bottom(); diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index 472e521c..4221c552 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -802,6 +802,9 @@ void UIManager::update() { // whole page blocking it past LVGLLock's 5s timeout. if (_navigation.current() == Route::CHAT && _chat_screen) { _chat_screen->tick_background_fill(); + // Long-press full-message view: the LVGL event handler only records + // the hash; the disk read + modal build run here on the main loop. + _chat_screen->tick_pending_full_message(); } // Gather + render the announce list off the LVGL lock on the main loop (its // refresh() just arms a flag). Iterating the path table on the LVGL task raced diff --git a/platformio.ini b/platformio.ini index 67ca7161..f1b9b6f1 100644 --- a/platformio.ini +++ b/platformio.ini @@ -132,7 +132,19 @@ lib_deps = ; callers fall back only when the cache is unpopulated. ; 6bea23c (same branch): public commit_index() so the list's one-shot ; preview warm-up persists to the index without a save/delete. - https://github.com/torlando-tech/microLXMF.git#6bea23c82b56088e75e1452adc545b3f2120e364 + ; bb9d387 (same branch): bounded in-process message-metadata cache in + ; MessageStore (64-slot FIFO, file scope) — chat reopen, + ; background page fill, and pagination no longer re-read message + ; files from SPI; state kept in sync by update_message_state(), + ; eviction on delete_message(). + ; 8e8e689 (same branch): load_message_content() full-view accessor — + ; the long-press full-message view reads the complete stored text + ; (uncapped) off the metadata-cache display cap. + ; 3438222/58a6eb0 (same branch): the cache table is PSRAM-allocated on + ; ESP32 (heap_caps_malloc/MALLOC_CAP_SPIRAM; static .bss placement + ; starved internal DRAM and the LVGL task failed to create at boot); + ; host builds keep the static table. + https://github.com/torlando-tech/microLXMF.git#58a6eb088a428453cf0bce63c893e53f0a2ec551 lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 diff --git a/tests/reference/test_microlxmf_telemetry_field.py b/tests/reference/test_microlxmf_telemetry_field.py index 3902ad3c..86a2af09 100644 --- a/tests/reference/test_microlxmf_telemetry_field.py +++ b/tests/reference/test_microlxmf_telemetry_field.py @@ -22,7 +22,7 @@ def test_pinned_microlxmf_telemetry_field_roundtrip(tmp_path): if not (source / "LXMF" / "LXMessage.cpp").is_file(): pytest.fail(f"pinned microLXMF source is unavailable at {microlxmf}") - expected = "6bea23c82b56088e75e1452adc545b3f2120e364" + expected = "58a6eb088a428453cf0bce63c893e53f0a2ec551" actual = subprocess.run( ["git", "-C", str(microlxmf), "rev-parse", "HEAD"], capture_output=True, diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index b4ebc2e1..9f70fea4 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "6bea23c82b56088e75e1452adc545b3f2120e364", + "microLXMF": "58a6eb088a428453cf0bce63c893e53f0a2ec551", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From 67c1083f2277d9b42829ba7874d0f4a455c9b42d Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:32:57 +0000 Subject: [PATCH 11/13] fix(ui): mutex the conversation-list deferred queues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1 on the list-perf commit: the deferred-queue pair (requester task -> main-loop flush) was unsynchronized. Producers push from the LVGL task (click handlers, refresh() during navigation) while flush_*() swaps the queues on the main loop without the LVGL lock, so concurrent push/swap is a data race. A single FreeRTOS mutex (QueueGuard, fail-closed) now guards all four shared queues: _pending_mark_reads, _pending_drops, _pending_name_writes, and the _index_commit_pending flag. Critical sections are bounded vector operations only — no store I/O, no LVGL lock, so acquisition cannot deadlock and the LVGL task is never stalled on LittleFS. Lock order is LVGL-lock -> queue-mutex only; queue sections never take the LVGL lock. The mutex is created before the screen's LVGL_LOCK section and deleted after it in the destructor. --- .../UI/LXMF/ConversationListScreen.cpp | 147 +++++++++++++++--- lib/tdeck_ui/UI/LXMF/ConversationListScreen.h | 9 ++ 2 files changed, 137 insertions(+), 19 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index a71f3dcb..287ed27c 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -35,6 +35,10 @@ ConversationListScreen::ConversationListScreen(lv_obj_t* parent) _label_battery_icon(nullptr), _label_battery_pct(nullptr), _lora_interface(nullptr), _ble_interface(nullptr), _gps(nullptr), _message_store(nullptr) { + // Queue mutex guards the deferred queues shared between the LVGL + // task (producers) and the main loop (flush_* consumers) — create + // before any task can touch them. + _queue_mutex = xSemaphoreCreateMutex(); LVGL_LOCK(); // Create screen object @@ -66,8 +70,37 @@ ConversationListScreen::~ConversationListScreen() { if (_screen) { lv_obj_del(_screen); } + // No task outlives the screen: safe to delete outside the lock. + if (_queue_mutex) { + vSemaphoreDelete(_queue_mutex); + _queue_mutex = nullptr; + } } +namespace { +// RAII guard for _queue_mutex. The guarded sections are bounded vector +// operations (no store I/O, no LVGL lock), so a missed acquisition is a +// logic error — treat it like an assert by failing closed. +class QueueGuard { +public: + explicit QueueGuard(SemaphoreHandle_t m) : _m(m) { + if (_m) { + xSemaphoreTake(_m, portMAX_DELAY); + } + } + ~QueueGuard() { + if (_m) { + xSemaphoreGive(_m); + } + } + QueueGuard(const QueueGuard&) = delete; + QueueGuard& operator=(const QueueGuard&) = delete; + +private: + SemaphoreHandle_t _m; +}; +} // namespace + void ConversationListScreen::create_header() { _header = lv_obj_create(_screen); lv_obj_set_size(_header, LV_PCT(100), 36); @@ -311,6 +344,9 @@ void ConversationListScreen::refresh() { if (need_metadata && last_meta.valid && !queued_drops) { _message_store->set_last_message_preview(peer_hash, last_meta.content.substr(0, 47)); + // refresh() runs on the LVGL task; the main-loop drain reads + // the flag — set under _queue_mutex (brief, no I/O inside). + QueueGuard guard(_queue_mutex); _index_commit_pending = true; } @@ -336,6 +372,9 @@ void ConversationListScreen::refresh() { // (held by UIManager::update()). UIManager::update() flushes // these before it takes the lock so the I/O never stalls the // render task. (Same rationale as on_message_received.) + // Appended from the LVGL task; the main-loop drain swaps + // the queue — take _queue_mutex (brief, no I/O inside). + QueueGuard guard(_queue_mutex); _pending_name_writes.emplace_back( peer_hash, std::string(display_name.c_str())); } @@ -435,14 +474,23 @@ void ConversationListScreen::flush_pending_name_writes() { // Called from UIManager::update() BEFORE it takes the LVGL lock. // set_display_name() hits microStore/LittleFS; running it inside refresh() // (under the lock) serially stalls the LVGL render task on a cold-boot - // announce burst. Same fix as UIManager::on_message_received. - if (!_message_store || _pending_name_writes.empty()) { + // announce burst. Same fix as UIManager::on_message_received. Swap the + // queue under _queue_mutex (refresh() appends from the LVGL task), then + // do the I/O outside it. + std::vector> batch; + { + QueueGuard guard(_queue_mutex); + if (_pending_name_writes.empty()) { + return; + } + batch.swap(_pending_name_writes); + } + if (!_message_store) { return; } - for (const auto& w : _pending_name_writes) { + for (const auto& w : batch) { _message_store->set_display_name(w.first, w.second); } - _pending_name_writes.clear(); } void ConversationListScreen::create_conversation_item(const ConversationItem& item) { @@ -523,24 +571,37 @@ void ConversationListScreen::request_mark_read(const Bytes& peer_hash) { // Defers the actual store mutation. mark_conversation_read() commits // the index to LittleFS, so it runs from UIManager::update() BEFORE // the LVGL lock (same pattern as flush_pending_name_writes) — not - // from this LVGL event callback. - for (const auto& h : _pending_mark_reads) { - if (h == peer_hash) return; + // from this LVGL event callback. The queue itself is shared with the + // main-loop drain, so take _queue_mutex (brief, no I/O inside). + { + QueueGuard guard(_queue_mutex); + for (const auto& h : _pending_mark_reads) { + if (h == peer_hash) return; + } + _pending_mark_reads.push_back(peer_hash); } - _pending_mark_reads.push_back(peer_hash); } void ConversationListScreen::flush_pending_mark_reads() { // Called from UIManager::update() BEFORE it takes the LVGL lock. // mark_conversation_read() hits microStore/LittleFS; the LVGL event - // that requested the mark-read only set the flag. - if (!_message_store || _pending_mark_reads.empty()) { + // that requested the mark-read only set the flag. Swap the queue + // under _queue_mutex, then do the I/O outside it (a new request + // arriving during the I/O waits its one drain tick). + std::vector batch; + { + QueueGuard guard(_queue_mutex); + if (_pending_mark_reads.empty()) { + return; + } + batch.swap(_pending_mark_reads); + } + if (!_message_store) { return; } - for (const auto& h : _pending_mark_reads) { + for (const auto& h : batch) { _message_store->mark_conversation_read(h); } - _pending_mark_reads.clear(); } void ConversationListScreen::request_drop_message(const Bytes& message_hash) { @@ -549,7 +610,9 @@ void ConversationListScreen::request_drop_message(const Bytes& message_hash) { // last_message_hash update) runs from flush_pending_drops() OUTSIDE // the LVGL lock. A bounded number of unreadable messages can be // queued per refresh (one per conversation, walking the tail), so the - // queue cannot grow without bound. + // queue cannot grow without bound. Shared with the main-loop drain, + // so take _queue_mutex (brief, no I/O inside). + QueueGuard guard(_queue_mutex); for (const auto& h : _pending_drops) { if (h == message_hash) return; } @@ -561,17 +624,55 @@ void ConversationListScreen::flush_pending_drops() { // Drop the queued unreadable messages so the conversation index // converges (deletion updates last_message_hash to the new tail); the // next list refresh then previews the newest readable message. A - // delete that fails (store not ready) is left for a later drain. - if (!_message_store || _pending_drops.empty()) { + // delete that fails (store not ready) is requeued for a later drain, + // deduped against any request that arrived during the I/O. + std::vector batch; + { + QueueGuard guard(_queue_mutex); + if (_pending_drops.empty()) { + return; + } + batch.swap(_pending_drops); + } + if (!_message_store) { + // Store not ready: hand the batch back (merged, deduped) so + // nothing is lost while the store comes up. + QueueGuard guard(_queue_mutex); + for (const auto& h : batch) { + bool present = false; + for (const auto& q : _pending_drops) { + if (q == h) { + present = true; + break; + } + } + if (!present) { + _pending_drops.push_back(h); + } + } return; } std::vector remaining; - for (const auto& h : _pending_drops) { + for (const auto& h : batch) { if (!_message_store->delete_message(h)) { remaining.push_back(h); } } - _pending_drops.swap(remaining); + if (!remaining.empty()) { + QueueGuard guard(_queue_mutex); + for (const auto& h : remaining) { + bool present = false; + for (const auto& q : _pending_drops) { + if (q == h) { + present = true; + break; + } + } + if (!present) { + _pending_drops.push_back(h); + } + } + } } void ConversationListScreen::flush_pending_index_commit() { @@ -585,11 +686,19 @@ void ConversationListScreen::flush_pending_index_commit() { // writes made by save_message() since the last store-originated // commit — one-shot by design; the flag clears even if the commit // fails and a later fallback re-arms it. - if (!_message_store || !_index_commit_pending) { + // refresh() (LVGL task) sets it; the main-loop drain consumes it — + // test-and-clear under _queue_mutex (a lost set would skip the + // one-shot index commit and cost every boot the full fallback pass). + bool pending; + { + QueueGuard guard(_queue_mutex); + pending = _index_commit_pending; + _index_commit_pending = false; + } + if (!_message_store || !pending) { return; } _message_store->commit_index(); - _index_commit_pending = false; } void ConversationListScreen::clear_unread_badge(lv_obj_t* container) { diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h index 995b26ef..66c9bed7 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h @@ -276,6 +276,15 @@ private: // delete). flush_pending_index_commit() persists the repop once so // the next boot serves previews straight from the index. bool _index_commit_pending = false; + // Guards _pending_name_writes / _pending_mark_reads / _pending_drops + // / _index_commit_pending: producers run under the LVGL lock (LVGL + // task: click handlers, refresh() during navigation) while the + // flush_*() consumers run on the main loop WITHOUT the LVGL lock + // (their store I/O must not run under it). Critical sections hold + // no LVGL lock, no store I/O, and never block — a brief vector + // swap/push — so portMAX_DELAY acquisition cannot deadlock or stall + // a task. + SemaphoreHandle_t _queue_mutex = nullptr; ConversationSelectedCallback _conversation_selected_callback; ComposeCallback _compose_callback; From bff993b2f824f65049ad2fe1b9f6ba497bb0c701 Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:36:44 +0000 Subject: [PATCH 12/13] chore(deps): bump microLXMF pin to d7e05fd (metadata cache fixes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracks microLXMF PR #11 head d7e05fd: archived-branch cache sync, bulk-delete cache invalidation, and the uniform capped-content contract for load_message_metadata(). No Pyxis code change — the chat/list paths already rely on the capped preview and the full-view accessor. --- platformio.ini | 4 ++-- tests/reference/test_microlxmf_telemetry_field.py | 2 +- tools/audit_release_build.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio.ini b/platformio.ini index f1b9b6f1..e9e9d6fe 100644 --- a/platformio.ini +++ b/platformio.ini @@ -140,11 +140,11 @@ lib_deps = ; 8e8e689 (same branch): load_message_content() full-view accessor — ; the long-press full-message view reads the complete stored text ; (uncapped) off the metadata-cache display cap. - ; 3438222/58a6eb0 (same branch): the cache table is PSRAM-allocated on + ; 3438222/58a6eb0/d7e05fd (same branch): the cache table is PSRAM-allocated on ; ESP32 (heap_caps_malloc/MALLOC_CAP_SPIRAM; static .bss placement ; starved internal DRAM and the LVGL task failed to create at boot); ; host builds keep the static table. - https://github.com/torlando-tech/microLXMF.git#58a6eb088a428453cf0bce63c893e53f0a2ec551 + https://github.com/torlando-tech/microLXMF.git#d7e05fdc10fe11bca0c4e3d8c0214b0be7d9a911 lvgl/lvgl@8.4.0 bblanchon/ArduinoJson@^7.4.2 hideakitai/MsgPack@^0.4.2 diff --git a/tests/reference/test_microlxmf_telemetry_field.py b/tests/reference/test_microlxmf_telemetry_field.py index 86a2af09..8ad69305 100644 --- a/tests/reference/test_microlxmf_telemetry_field.py +++ b/tests/reference/test_microlxmf_telemetry_field.py @@ -22,7 +22,7 @@ def test_pinned_microlxmf_telemetry_field_roundtrip(tmp_path): if not (source / "LXMF" / "LXMessage.cpp").is_file(): pytest.fail(f"pinned microLXMF source is unavailable at {microlxmf}") - expected = "58a6eb088a428453cf0bce63c893e53f0a2ec551" + expected = "d7e05fdc10fe11bca0c4e3d8c0214b0be7d9a911" actual = subprocess.run( ["git", "-C", str(microlxmf), "rev-parse", "HEAD"], capture_output=True, diff --git a/tools/audit_release_build.py b/tools/audit_release_build.py index 9f70fea4..826c1a01 100644 --- a/tools/audit_release_build.py +++ b/tools/audit_release_build.py @@ -46,7 +46,7 @@ EXCLUDED_SYMBOLS = ( ) PINNED_DEPENDENCIES = { "microReticulum": "cd0338e7fc07d3a7785a450656ba766491cbf6e8", - "microLXMF": "58a6eb088a428453cf0bce63c893e53f0a2ec551", + "microLXMF": "d7e05fdc10fe11bca0c4e3d8c0214b0be7d9a911", "microStore": "2762f7606800ffb23f4a593947d4f58e259cda7a", } From 362db2c8054c3edc2b9989c87369a0592f64d3f8 Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:49:26 +0000 Subject: [PATCH 13/13] fix(ui): fail closed when the queue mutex is unavailable Greptile follow-up on 67c1083: if xSemaphoreCreateMutex() failed, QueueGuard silently locked nothing and the deferred queues still ran unsynchronized. The constructor now logs the allocation failure and every guarded site refuses mutation when _queue_mutex is null: requesters stop enqueuing, flushers stop draining (nothing queued can exist), and the one-shot preview commit is skipped. The screen keeps rendering and chatting; only the deferred list mutations degrade. --- .../UI/LXMF/ConversationListScreen.cpp | 60 +++++++++++++++---- lib/tdeck_ui/UI/LXMF/ConversationListScreen.h | 4 +- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 287ed27c..771f4206 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -37,8 +37,15 @@ ConversationListScreen::ConversationListScreen(lv_obj_t* parent) _message_store(nullptr) { // Queue mutex guards the deferred queues shared between the LVGL // task (producers) and the main loop (flush_* consumers) — create - // before any task can touch them. + // before any task can touch them. Fail closed: if allocation + // fails, every guarded site below refuses queue mutation and the + // screen degrades (no mark-read/drop/deferred name writes) rather + // than running the queues unsynchronized. _queue_mutex = xSemaphoreCreateMutex(); + if (!_queue_mutex) { + ERROR("ConversationListScreen: queue mutex allocation failed; " + "deferred mark-read/drop/name-write queues disabled (fail closed)"); + } LVGL_LOCK(); // Create screen object @@ -346,8 +353,12 @@ void ConversationListScreen::refresh() { last_meta.content.substr(0, 47)); // refresh() runs on the LVGL task; the main-loop drain reads // the flag — set under _queue_mutex (brief, no I/O inside). - QueueGuard guard(_queue_mutex); - _index_commit_pending = true; + // Fail closed: without the mutex the deferred commit cannot + // run safely and would leave the preview uncommitted anyway. + if (_queue_mutex) { + QueueGuard guard(_queue_mutex); + _index_commit_pending = true; + } } // Create conversation item @@ -374,9 +385,12 @@ void ConversationListScreen::refresh() { // render task. (Same rationale as on_message_received.) // Appended from the LVGL task; the main-loop drain swaps // the queue — take _queue_mutex (brief, no I/O inside). - QueueGuard guard(_queue_mutex); - _pending_name_writes.emplace_back( - peer_hash, std::string(display_name.c_str())); + // Fail closed (see constructor) if the mutex is unavailable. + if (_queue_mutex) { + QueueGuard guard(_queue_mutex); + _pending_name_writes.emplace_back( + peer_hash, std::string(display_name.c_str())); + } } } if (!resolved && _message_store) { @@ -476,9 +490,13 @@ void ConversationListScreen::flush_pending_name_writes() { // (under the lock) serially stalls the LVGL render task on a cold-boot // announce burst. Same fix as UIManager::on_message_received. Swap the // queue under _queue_mutex (refresh() appends from the LVGL task), then - // do the I/O outside it. + // do the I/O outside it. Fail closed (no draining) if the mutex is + // unavailable. std::vector> batch; { + if (!_queue_mutex) { + return; + } QueueGuard guard(_queue_mutex); if (_pending_name_writes.empty()) { return; @@ -572,7 +590,11 @@ void ConversationListScreen::request_mark_read(const Bytes& peer_hash) { // the index to LittleFS, so it runs from UIManager::update() BEFORE // the LVGL lock (same pattern as flush_pending_name_writes) — not // from this LVGL event callback. The queue itself is shared with the - // main-loop drain, so take _queue_mutex (brief, no I/O inside). + // main-loop drain, so take _queue_mutex (brief, no I/O inside). Fail + // closed (no enqueue) if the mutex is unavailable. + if (!_queue_mutex) { + return; + } { QueueGuard guard(_queue_mutex); for (const auto& h : _pending_mark_reads) { @@ -587,9 +609,13 @@ void ConversationListScreen::flush_pending_mark_reads() { // mark_conversation_read() hits microStore/LittleFS; the LVGL event // that requested the mark-read only set the flag. Swap the queue // under _queue_mutex, then do the I/O outside it (a new request - // arriving during the I/O waits its one drain tick). + // arriving during the I/O waits its one drain tick). Fail closed + // (no draining) if the mutex is unavailable. std::vector batch; { + if (!_queue_mutex) { + return; + } QueueGuard guard(_queue_mutex); if (_pending_mark_reads.empty()) { return; @@ -611,7 +637,11 @@ void ConversationListScreen::request_drop_message(const Bytes& message_hash) { // the LVGL lock. A bounded number of unreadable messages can be // queued per refresh (one per conversation, walking the tail), so the // queue cannot grow without bound. Shared with the main-loop drain, - // so take _queue_mutex (brief, no I/O inside). + // so take _queue_mutex (brief, no I/O inside). Fail closed (no + // enqueue) if the mutex is unavailable. + if (!_queue_mutex) { + return; + } QueueGuard guard(_queue_mutex); for (const auto& h : _pending_drops) { if (h == message_hash) return; @@ -625,7 +655,11 @@ void ConversationListScreen::flush_pending_drops() { // converges (deletion updates last_message_hash to the new tail); the // next list refresh then previews the newest readable message. A // delete that fails (store not ready) is requeued for a later drain, - // deduped against any request that arrived during the I/O. + // deduped against any request that arrived during the I/O. Fail + // closed (no draining) if the mutex is unavailable. + if (!_queue_mutex) { + return; + } std::vector batch; { QueueGuard guard(_queue_mutex); @@ -689,6 +723,10 @@ void ConversationListScreen::flush_pending_index_commit() { // refresh() (LVGL task) sets it; the main-loop drain consumes it — // test-and-clear under _queue_mutex (a lost set would skip the // one-shot index commit and cost every boot the full fallback pass). + // Fail closed (no commit drain) if the mutex is unavailable. + if (!_queue_mutex) { + return; + } bool pending; { QueueGuard guard(_queue_mutex); diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h index 66c9bed7..0ac71787 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.h @@ -283,7 +283,9 @@ private: // (their store I/O must not run under it). Critical sections hold // no LVGL lock, no store I/O, and never block — a brief vector // swap/push — so portMAX_DELAY acquisition cannot deadlock or stall - // a task. + // a task. NULL (allocation failure) means every guarded site fails + // closed: queue mutation is refused and the affected features + // degrade rather than run unsynchronized. SemaphoreHandle_t _queue_mutex = nullptr; ConversationSelectedCallback _conversation_selected_callback;