From 02ceeda75bfa361d11f77211a90835fb6a81106d Mon Sep 17 00:00:00 2001 From: torlando-tech Date: Tue, 5 May 2026 01:39:44 -0400 Subject: [PATCH] =?UTF-8?q?Track=20A.8:=20re-vendor=20shim=20from=20ca355e?= =?UTF-8?q?5;=20UniversalFileSystem=E2=86=92microStore;=20relocate=20shim?= =?UTF-8?q?=20to=20lib/;=20pyxis=20lib=20API=20renames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compile-tier graft progress on top of 40e561f. Key changes: - Re-vendor lib/microreticulum-shim/ (was src-shim/) from the *actual* ca355e5 commit content rather than the stale "feat/t-deck HEAD" /tmp clone the previous commit pulled from. Recovers process_sync() on LXMRouter and MEMORY_MONITOR_POLL macro that were missing. - Move src-shim/ → lib/microreticulum-shim/ + add library.json so PlatformIO discovers the .cpp files and links them. Was previously only on the include path; the .cpps weren't in the build. (This unblocks the 30+ undefined-reference linker errors for LXMF and Instrumentation symbols.) - Drop -Isrc-shim/Utilities (/Cryptography/Instrumentation) from build_flags — they were over-broad and put our Stream.h on the GLOBAL header path, breaking Arduino's Wire.cpp which has `class TwoWire: public Stream`. -Ilib/microreticulum-shim alone resolves subdir lookups via , . - UniversalFileSystem migrated to microStore::Adapters::SPIFFSFileSystem (activated by -DUSTORE_USE_SPIFFS). Vanilla upstream microReticulum @ 0.3.0 deleted RNS::FileSystem in favor of microStore. Pyxis's lib/universal_filesystem/ is now dead code on this build path. - pyxis lib API renames for the post-graft world: SDLogger.cpp: RNS::setLogCallback -> RNS::set_log_callback AnnounceListScreen: Transport::get_destination_table -> Transport::get_path_table Transport::DestinationEntry -> RNS::Persistence::DestinationEntry UIManager.cpp: _lxst_destination ctor explicit RNS::Type::NONE (vanilla Destination has no default ctor) Identity::mark_persistent calls disabled w/ restoration TODO ConversationListScreen: Interface::get_rssi/get_stats calls disabled (the methods are non-virtual on BLEInterface/SX1262Interface post de-virtualization in a0ff631) Compile is clean against the fixed-cryptography submodule pin; current failure layer is fork-only Type::Channel constants referenced by the vendored shim's Buffer/ChannelData files. That's the next session's problem — see pyxis_microReticulum_graft_spike_findings.md for the plan options (most likely: remove Channel/Buffer/ChannelData/Ratchet from the shim, since LXMF doesn't use Channel anyway per the 2026-05-04 investigation). --- .../microreticulum-shim}/Buffer.cpp | 0 .../microreticulum-shim}/Buffer.h | 0 .../microreticulum-shim}/BytesPool.h | 108 +++- .../microreticulum-shim}/ChannelData.h | 0 .../microreticulum-shim}/Cryptography/BZ2.cpp | 0 .../microreticulum-shim}/Cryptography/BZ2.h | 0 .../Cryptography/Ratchet.cpp | 0 .../Cryptography/Ratchet.h | 0 .../microreticulum-shim}/Display.cpp | 0 .../microreticulum-shim}/Display.h | 0 .../microreticulum-shim}/DisplayGraphics.h | 0 .../microreticulum-shim}/FileStream.h | 0 .../microreticulum-shim}/FileSystem.h | 0 .../Instrumentation/BootProfiler.cpp | 0 .../Instrumentation/BootProfiler.h | 0 .../Instrumentation/MemoryMonitor.cpp | 14 +- .../Instrumentation/MemoryMonitor.h | 13 + .../microreticulum-shim}/LXMF/LXMRouter.cpp | 527 +++++++++++++----- .../microreticulum-shim}/LXMF/LXMRouter.h | 46 +- .../microreticulum-shim}/LXMF/LXMessage.cpp | 119 ++-- .../microreticulum-shim}/LXMF/LXMessage.h | 4 + .../microreticulum-shim}/LXMF/LXStamper.cpp | 0 .../microreticulum-shim}/LXMF/LXStamper.h | 0 .../LXMF/MessageStore.cpp | 0 .../microreticulum-shim}/LXMF/MessageStore.h | 0 .../LXMF/PropagationNodeManager.cpp | 27 +- .../LXMF/PropagationNodeManager.h | 0 .../microreticulum-shim}/LXMF/Type.h | 26 +- .../microreticulum-shim}/MessageBase.h | 0 .../microreticulum-shim}/ObjectPool.h | 0 .../microreticulum-shim}/PSRAMAllocator.h | 0 .../SegmentAccumulator.cpp | 0 .../microreticulum-shim}/SegmentAccumulator.h | 0 .../microreticulum-shim}/Utilities/Print.cpp | 0 .../microreticulum-shim}/Utilities/Print.h | 0 .../microreticulum-shim}/Utilities/Stream.cpp | 0 .../microreticulum-shim}/Utilities/Stream.h | 0 lib/microreticulum-shim/library.json | 7 + lib/tdeck_ui/Hardware/TDeck/SDLogger.cpp | 4 +- lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp | 12 +- .../UI/LXMF/ConversationListScreen.cpp | 27 +- lib/tdeck_ui/UI/LXMF/UIManager.cpp | 16 +- platformio.ini | 24 +- src/main.cpp | 19 +- 44 files changed, 724 insertions(+), 269 deletions(-) rename {src-shim => lib/microreticulum-shim}/Buffer.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Buffer.h (100%) rename {src-shim => lib/microreticulum-shim}/BytesPool.h (72%) rename {src-shim => lib/microreticulum-shim}/ChannelData.h (100%) rename {src-shim => lib/microreticulum-shim}/Cryptography/BZ2.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Cryptography/BZ2.h (100%) rename {src-shim => lib/microreticulum-shim}/Cryptography/Ratchet.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Cryptography/Ratchet.h (100%) rename {src-shim => lib/microreticulum-shim}/Display.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Display.h (100%) rename {src-shim => lib/microreticulum-shim}/DisplayGraphics.h (100%) rename {src-shim => lib/microreticulum-shim}/FileStream.h (100%) rename {src-shim => lib/microreticulum-shim}/FileSystem.h (100%) rename {src-shim => lib/microreticulum-shim}/Instrumentation/BootProfiler.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Instrumentation/BootProfiler.h (100%) rename {src-shim => lib/microreticulum-shim}/Instrumentation/MemoryMonitor.cpp (96%) rename {src-shim => lib/microreticulum-shim}/Instrumentation/MemoryMonitor.h (88%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXMRouter.cpp (80%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXMRouter.h (92%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXMessage.cpp (83%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXMessage.h (98%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXStamper.cpp (100%) rename {src-shim => lib/microreticulum-shim}/LXMF/LXStamper.h (100%) rename {src-shim => lib/microreticulum-shim}/LXMF/MessageStore.cpp (100%) rename {src-shim => lib/microreticulum-shim}/LXMF/MessageStore.h (100%) rename {src-shim => lib/microreticulum-shim}/LXMF/PropagationNodeManager.cpp (93%) rename {src-shim => lib/microreticulum-shim}/LXMF/PropagationNodeManager.h (100%) rename {src-shim => lib/microreticulum-shim}/LXMF/Type.h (80%) rename {src-shim => lib/microreticulum-shim}/MessageBase.h (100%) rename {src-shim => lib/microreticulum-shim}/ObjectPool.h (100%) rename {src-shim => lib/microreticulum-shim}/PSRAMAllocator.h (100%) rename {src-shim => lib/microreticulum-shim}/SegmentAccumulator.cpp (100%) rename {src-shim => lib/microreticulum-shim}/SegmentAccumulator.h (100%) rename {src-shim => lib/microreticulum-shim}/Utilities/Print.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Utilities/Print.h (100%) rename {src-shim => lib/microreticulum-shim}/Utilities/Stream.cpp (100%) rename {src-shim => lib/microreticulum-shim}/Utilities/Stream.h (100%) create mode 100644 lib/microreticulum-shim/library.json diff --git a/src-shim/Buffer.cpp b/lib/microreticulum-shim/Buffer.cpp similarity index 100% rename from src-shim/Buffer.cpp rename to lib/microreticulum-shim/Buffer.cpp diff --git a/src-shim/Buffer.h b/lib/microreticulum-shim/Buffer.h similarity index 100% rename from src-shim/Buffer.h rename to lib/microreticulum-shim/Buffer.h diff --git a/src-shim/BytesPool.h b/lib/microreticulum-shim/BytesPool.h similarity index 72% rename from src-shim/BytesPool.h rename to lib/microreticulum-shim/BytesPool.h index b05767d3..4ff7be6f 100644 --- a/src-shim/BytesPool.h +++ b/lib/microreticulum-shim/BytesPool.h @@ -14,10 +14,13 @@ * of destroying it. * * The pool has four tiers sized for Reticulum packet processing: - * - 64 bytes (512 slots): hashes (16-32 bytes), small fields - highest traffic - * - 256 bytes (24 slots): keys, small announces - * - 512 bytes (16 slots): standard packets (MTU=500 + margin) - * - 1024 bytes (16 slots): resource advertisements, large packets + * - 64 bytes (1024 slots): hashes (16-32 bytes), small fields - highest traffic + * - 256 bytes (16 slots): keys, small announces + * - 512 bytes (12 slots): standard packets (MTU=500 + margin) + * - 1024 bytes (12 slots): resource advertisements, large packets + * + * All storage arrays are dynamically allocated in PSRAM on ESP32 to avoid + * consuming internal RAM (BSS). Only the pointers (~32 bytes) stay in BSS. * * Thread-safe via FreeRTOS spinlock (ESP32) or std::mutex (native). * @@ -44,6 +47,7 @@ #if defined(ESP_PLATFORM) || defined(ARDUINO) #include "freertos/FreeRTOS.h" #include "freertos/portmacro.h" +#include #define BYTESPOOL_USE_SPINLOCK 1 #else // Native build - use std::mutex instead @@ -61,13 +65,15 @@ namespace BytesPoolConfig { static constexpr size_t TIER_MEDIUM = 512; // Standard packets static constexpr size_t TIER_LARGE = 1024; // Large packets, resource ads - // Slot counts per tier - tuned from runtime observation (2026-01-24) - // NOTE: Each slot uses ~16 bytes internal RAM for vector metadata + stack pointer - // 512 slots = ~8KB internal RAM overhead (conservative for busy networks) - static constexpr size_t TINY_SLOTS = 512; // High traffic tier (balanced for memory) - static constexpr size_t SMALL_SLOTS = 8; // Low traffic (peak 1) - static constexpr size_t MEDIUM_SLOTS = 8; // Rare (packets, keep headroom) - static constexpr size_t LARGE_SLOTS = 8; // Rare (resource ads, keep headroom) + // Slot counts per tier — tuned 2026-02-19 + // Storage arrays now live in PSRAM, so internal RAM cost is only pointers (~32B). + // Tiny tier for transient packet processing only. Known destinations now use + // fixed buffers (zero pool slots). 1024 provides ample headroom for packet + // hashes, Transport tables, and burst announce processing. + static constexpr size_t TINY_SLOTS = 1024; // Transient packet processing (hashes, keys, fields) + static constexpr size_t SMALL_SLOTS = 16; // Keys, small announces + static constexpr size_t MEDIUM_SLOTS = 12; // Standard packets + static constexpr size_t LARGE_SLOTS = 12; // Resource ads, large packets // Tier identifiers for deleter enum Tier : uint8_t { @@ -93,12 +99,12 @@ using PooledData = std::vector>; * - Repeated capacity reservation allocations * - shared_ptr control block allocations (via make_shared replacement) * - * Memory footprint (tuned 2026-01-24): - * - Tiny: 512 slots x 64 bytes = 32KB backing + ~8KB metadata (internal RAM) - * - Small: 8 slots x 256 bytes = 2KB backing + ~128B metadata - * - Medium: 8 slots x 512 bytes = 4KB backing + ~128B metadata - * - Large: 8 slots x 1024 bytes = 8KB backing + ~128B metadata - * - Total: ~46KB PSRAM backing + ~8.5KB internal RAM metadata + * Memory footprint (tuned 2026-02-19, all storage in PSRAM): + * - Tiny: 1024 slots x 64 bytes = 64KB backing + ~16KB metadata (PSRAM) + * - Small: 16 slots x 256 bytes = 4KB backing + ~256B metadata (PSRAM) + * - Medium: 12 slots x 512 bytes = 6KB backing + ~192B metadata (PSRAM) + * - Large: 12 slots x 1024 bytes = 12KB backing + ~192B metadata (PSRAM) + * - Total: ~103KB PSRAM, ~32 bytes internal RAM (pointers only) */ class BytesPool { public: @@ -272,8 +278,7 @@ private: #if BYTESPOOL_USE_SPINLOCK portMUX_INITIALIZE(&_mux); #endif - // Pre-allocate all pool entries - // This is done at construction (startup) to front-load allocations + allocateStorage(); initializeTier(_tiny_storage, _tiny_stack, _tiny_count, BytesPoolConfig::TIER_TINY, BytesPoolConfig::TINY_SLOTS); initializeTier(_small_storage, _small_stack, _small_count, @@ -288,11 +293,54 @@ private: BytesPool(const BytesPool&) = delete; BytesPool& operator=(const BytesPool&) = delete; + // Allocate storage and stack arrays in PSRAM (ESP32) or heap (native) + void allocateStorage() { +#if BYTESPOOL_USE_SPINLOCK + // ESP32: allocate in PSRAM to avoid consuming internal RAM (BSS) + #define POOL_ALLOC(ptr, type, count) do { \ + ptr = static_cast(heap_caps_aligned_alloc( \ + alignof(type), (count) * sizeof(type), \ + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); \ + if (!ptr) { \ + ptr = static_cast(heap_caps_aligned_alloc( \ + alignof(type), (count) * sizeof(type), \ + MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); \ + if (ptr) WARNING("BytesPool: " #ptr " fell back to internal RAM"); \ + } \ + } while(0) + + POOL_ALLOC(_tiny_storage, PooledData, BytesPoolConfig::TINY_SLOTS); + POOL_ALLOC(_tiny_stack, PooledData*, BytesPoolConfig::TINY_SLOTS); + POOL_ALLOC(_small_storage, PooledData, BytesPoolConfig::SMALL_SLOTS); + POOL_ALLOC(_small_stack, PooledData*, BytesPoolConfig::SMALL_SLOTS); + POOL_ALLOC(_medium_storage, PooledData, BytesPoolConfig::MEDIUM_SLOTS); + POOL_ALLOC(_medium_stack, PooledData*, BytesPoolConfig::MEDIUM_SLOTS); + POOL_ALLOC(_large_storage, PooledData, BytesPoolConfig::LARGE_SLOTS); + POOL_ALLOC(_large_stack, PooledData*, BytesPoolConfig::LARGE_SLOTS); + #undef POOL_ALLOC +#else + // Native: use new[] + _tiny_storage = new PooledData[BytesPoolConfig::TINY_SLOTS]; + _tiny_stack = new PooledData*[BytesPoolConfig::TINY_SLOTS]; + _small_storage = new PooledData[BytesPoolConfig::SMALL_SLOTS]; + _small_stack = new PooledData*[BytesPoolConfig::SMALL_SLOTS]; + _medium_storage = new PooledData[BytesPoolConfig::MEDIUM_SLOTS]; + _medium_stack = new PooledData*[BytesPoolConfig::MEDIUM_SLOTS]; + _large_storage = new PooledData[BytesPoolConfig::LARGE_SLOTS]; + _large_stack = new PooledData*[BytesPoolConfig::LARGE_SLOTS]; +#endif + } + // Initialize a tier with pre-allocated vectors void initializeTier(PooledData* storage, PooledData** stack, size_t& count, size_t capacity, size_t slots) { + if (!storage || !stack) { + ERROR("BytesPool: allocation failed for tier, pool will be undersized"); + count = 0; + return; + } for (size_t i = 0; i < slots; i++) { - // Placement new to construct in storage array + // Placement new to construct in allocated storage new (&storage[i]) PooledData(); storage[i].reserve(capacity); stack[i] = &storage[i]; @@ -300,17 +348,17 @@ private: count = slots; } - // Storage for pooled vectors (fixed arrays avoid dynamic allocation) - PooledData _tiny_storage[BytesPoolConfig::TINY_SLOTS]; - PooledData _small_storage[BytesPoolConfig::SMALL_SLOTS]; - PooledData _medium_storage[BytesPoolConfig::MEDIUM_SLOTS]; - PooledData _large_storage[BytesPoolConfig::LARGE_SLOTS]; + // Storage for pooled vectors — dynamically allocated (PSRAM on ESP32) + PooledData* _tiny_storage = nullptr; + PooledData* _small_storage = nullptr; + PooledData* _medium_storage = nullptr; + PooledData* _large_storage = nullptr; - // Stacks of available vectors (indices into storage arrays) - PooledData* _tiny_stack[BytesPoolConfig::TINY_SLOTS]; - PooledData* _small_stack[BytesPoolConfig::SMALL_SLOTS]; - PooledData* _medium_stack[BytesPoolConfig::MEDIUM_SLOTS]; - PooledData* _large_stack[BytesPoolConfig::LARGE_SLOTS]; + // Stacks of available vectors (pointers into storage arrays) + PooledData** _tiny_stack = nullptr; + PooledData** _small_stack = nullptr; + PooledData** _medium_stack = nullptr; + PooledData** _large_stack = nullptr; // Stack counts (how many available in each tier) size_t _tiny_count = 0; diff --git a/src-shim/ChannelData.h b/lib/microreticulum-shim/ChannelData.h similarity index 100% rename from src-shim/ChannelData.h rename to lib/microreticulum-shim/ChannelData.h diff --git a/src-shim/Cryptography/BZ2.cpp b/lib/microreticulum-shim/Cryptography/BZ2.cpp similarity index 100% rename from src-shim/Cryptography/BZ2.cpp rename to lib/microreticulum-shim/Cryptography/BZ2.cpp diff --git a/src-shim/Cryptography/BZ2.h b/lib/microreticulum-shim/Cryptography/BZ2.h similarity index 100% rename from src-shim/Cryptography/BZ2.h rename to lib/microreticulum-shim/Cryptography/BZ2.h diff --git a/src-shim/Cryptography/Ratchet.cpp b/lib/microreticulum-shim/Cryptography/Ratchet.cpp similarity index 100% rename from src-shim/Cryptography/Ratchet.cpp rename to lib/microreticulum-shim/Cryptography/Ratchet.cpp diff --git a/src-shim/Cryptography/Ratchet.h b/lib/microreticulum-shim/Cryptography/Ratchet.h similarity index 100% rename from src-shim/Cryptography/Ratchet.h rename to lib/microreticulum-shim/Cryptography/Ratchet.h diff --git a/src-shim/Display.cpp b/lib/microreticulum-shim/Display.cpp similarity index 100% rename from src-shim/Display.cpp rename to lib/microreticulum-shim/Display.cpp diff --git a/src-shim/Display.h b/lib/microreticulum-shim/Display.h similarity index 100% rename from src-shim/Display.h rename to lib/microreticulum-shim/Display.h diff --git a/src-shim/DisplayGraphics.h b/lib/microreticulum-shim/DisplayGraphics.h similarity index 100% rename from src-shim/DisplayGraphics.h rename to lib/microreticulum-shim/DisplayGraphics.h diff --git a/src-shim/FileStream.h b/lib/microreticulum-shim/FileStream.h similarity index 100% rename from src-shim/FileStream.h rename to lib/microreticulum-shim/FileStream.h diff --git a/src-shim/FileSystem.h b/lib/microreticulum-shim/FileSystem.h similarity index 100% rename from src-shim/FileSystem.h rename to lib/microreticulum-shim/FileSystem.h diff --git a/src-shim/Instrumentation/BootProfiler.cpp b/lib/microreticulum-shim/Instrumentation/BootProfiler.cpp similarity index 100% rename from src-shim/Instrumentation/BootProfiler.cpp rename to lib/microreticulum-shim/Instrumentation/BootProfiler.cpp diff --git a/src-shim/Instrumentation/BootProfiler.h b/lib/microreticulum-shim/Instrumentation/BootProfiler.h similarity index 100% rename from src-shim/Instrumentation/BootProfiler.h rename to lib/microreticulum-shim/Instrumentation/BootProfiler.h diff --git a/src-shim/Instrumentation/MemoryMonitor.cpp b/lib/microreticulum-shim/Instrumentation/MemoryMonitor.cpp similarity index 96% rename from src-shim/Instrumentation/MemoryMonitor.cpp rename to lib/microreticulum-shim/Instrumentation/MemoryMonitor.cpp index c5fbca1b..3990d9b4 100644 --- a/src-shim/Instrumentation/MemoryMonitor.cpp +++ b/lib/microreticulum-shim/Instrumentation/MemoryMonitor.cpp @@ -38,6 +38,7 @@ static size_t _task_count = 0; // Static member initialization TimerHandle_t MemoryMonitor::_timer = nullptr; bool MemoryMonitor::_verbose = false; +volatile bool MemoryMonitor::_pending = false; // Static buffer for log formatting (avoid stack allocation in callbacks) static char _log_buffer[256]; @@ -152,8 +153,9 @@ void MemoryMonitor::logNow() { } -void MemoryMonitor::timerCallback(TimerHandle_t timer) { - (void)timer; // Unused parameter +void MemoryMonitor::poll() { + if (!_pending) return; + _pending = false; logHeapStats(); if (_task_count > 0) { @@ -162,6 +164,14 @@ void MemoryMonitor::timerCallback(TimerHandle_t timer) { } +void MemoryMonitor::timerCallback(TimerHandle_t timer) { + (void)timer; + // Only set flag — heavy logging is done in poll() on the main loop stack + // to avoid overflowing the small FreeRTOS timer task stack (3120 bytes) + _pending = true; +} + + void MemoryMonitor::logHeapStats() { // Internal RAM statistics (critical for stability) size_t internal_free = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); diff --git a/src-shim/Instrumentation/MemoryMonitor.h b/lib/microreticulum-shim/Instrumentation/MemoryMonitor.h similarity index 88% rename from src-shim/Instrumentation/MemoryMonitor.h rename to lib/microreticulum-shim/Instrumentation/MemoryMonitor.h index 44578ea3..ed1918ce 100644 --- a/src-shim/Instrumentation/MemoryMonitor.h +++ b/lib/microreticulum-shim/Instrumentation/MemoryMonitor.h @@ -91,11 +91,21 @@ public: */ static void logNow(); + /** + * Poll for pending log output (call from main loop) + * + * The timer callback only sets a flag; this method does the actual + * logging on the caller's stack (main loop) to avoid overflowing + * the small FreeRTOS timer task stack. + */ + static void poll(); + private: /** * FreeRTOS timer callback * * Called by the timer daemon task at each interval. + * Only sets _pending flag — actual work is done in poll(). */ static void timerCallback(TimerHandle_t timer); @@ -112,6 +122,7 @@ private: // Static members static TimerHandle_t _timer; static bool _verbose; + static volatile bool _pending; }; }} // namespace RNS::Instrumentation @@ -121,6 +132,7 @@ private: #define MEMORY_MONITOR_REGISTER_TASK(handle, name) RNS::Instrumentation::MemoryMonitor::registerTask(handle, name) #define MEMORY_MONITOR_UNREGISTER_TASK(handle) RNS::Instrumentation::MemoryMonitor::unregisterTask(handle) #define MEMORY_MONITOR_LOG_NOW() RNS::Instrumentation::MemoryMonitor::logNow() +#define MEMORY_MONITOR_POLL() RNS::Instrumentation::MemoryMonitor::poll() #define MEMORY_MONITOR_STOP() RNS::Instrumentation::MemoryMonitor::stop() #else // MEMORY_INSTRUMENTATION_ENABLED not defined @@ -130,6 +142,7 @@ private: #define MEMORY_MONITOR_REGISTER_TASK(handle, name) ((void)0) #define MEMORY_MONITOR_UNREGISTER_TASK(handle) ((void)0) #define MEMORY_MONITOR_LOG_NOW() ((void)0) +#define MEMORY_MONITOR_POLL() ((void)0) #define MEMORY_MONITOR_STOP() ((void)0) #endif // MEMORY_INSTRUMENTATION_ENABLED diff --git a/src-shim/LXMF/LXMRouter.cpp b/lib/microreticulum-shim/LXMF/LXMRouter.cpp similarity index 80% rename from src-shim/LXMF/LXMRouter.cpp rename to lib/microreticulum-shim/LXMF/LXMRouter.cpp index 5b605132..22138ca9 100644 --- a/src-shim/LXMF/LXMRouter.cpp +++ b/lib/microreticulum-shim/LXMF/LXMRouter.cpp @@ -1,5 +1,4 @@ #include "LXMRouter.h" -#include "PropagationNodeManager.h" #include #include #include @@ -56,8 +55,9 @@ static RouterRegistrySlot* find_empty_router_registry_slot() { return nullptr; } -// Outbound resources fixed pool (zero heap fragmentation) -// Fixed arrays eliminate ~0.8KB Bytes metadata overhead (16 slots × 2 Bytes × 24 bytes) +// Outbound DIRECT delivery resources fixed pool (zero heap fragmentation) +// Tracks resource transfers for DIRECT delivery via links to recipients. +// Separate from PropResourceSlot (in header) which tracks PROPAGATED delivery resources. static constexpr size_t OUTBOUND_RESOURCES_SIZE = 16; static constexpr size_t OUTBOUND_HASH_SIZE = 32; // SHA256 hash size struct OutboundResourceSlot { @@ -207,6 +207,16 @@ static void static_packet_callback(const Bytes& data, const Packet& packet) { } } +// Static packet callback for link (uses link destination hash for router lookup) +static void static_link_packet_callback(const Bytes& data, const Packet& packet) { + // For link packets, destination_hash() is the link_id, not the delivery destination. + // Look up the router via the link's destination hash instead. + RouterRegistrySlot* slot = find_router_registry_slot(packet.link().destination().hash()); + if (slot) { + slot->router->on_packet(data, packet); + } +} + // Static link callbacks static void static_link_established_callback(Link& link) { // Find router that owns this link destination @@ -285,52 +295,24 @@ static void static_outbound_resource_concluded(const Resource& resource) { // Static proof callback - called when delivery proof is received void LXMRouter::static_proof_callback(const PacketReceipt& receipt) { - DEBUG(">>> PROOF CALLBACK ENTRY"); -#ifdef ARDUINO - Serial.flush(); -#endif - - // Get packet hash from receipt - DEBUG(">>> Getting packet hash from receipt"); -#ifdef ARDUINO - Serial.flush(); -#endif Bytes packet_hash = receipt.hash(); char buf[128]; - DEBUG(">>> Looking up pending proof slot"); -#ifdef ARDUINO - Serial.flush(); -#endif - // Look up message hash for this packet PendingProofSlot* slot = find_pending_proof_slot(packet_hash); if (slot) { - DEBUG(">>> Found slot, getting message hash"); -#ifdef ARDUINO - Serial.flush(); -#endif Bytes message_hash = slot->message_hash_bytes(); snprintf(buf, sizeof(buf), "Delivery proof received for message %.16s...", message_hash.toHex().c_str()); INFO(buf); -#ifdef ARDUINO - Serial.flush(); -#endif // Track notified routers to avoid duplicates (max ROUTER_REGISTRY_SIZE) LXMRouter* notified_routers[ROUTER_REGISTRY_SIZE]; size_t notified_count = 0; - DEBUG(">>> Iterating router registry"); -#ifdef ARDUINO - Serial.flush(); -#endif - // Find the router that sent this message and call its delivered callback for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { if (_router_registry_pool[i].in_use) { LXMRouter* router = _router_registry_pool[i].router; - // Check if already notified bool already_notified = false; for (size_t j = 0; j < notified_count; j++) { if (notified_routers[j] == router) { @@ -339,48 +321,22 @@ void LXMRouter::static_proof_callback(const PacketReceipt& receipt) { } } if (!already_notified && router && router->_delivered_callback) { - DEBUGF(">>> Calling delivered callback for router %zu", i); -#ifdef ARDUINO - Serial.flush(); -#endif notified_routers[notified_count++] = router; // Create a minimal message with just the hash for the callback - // The callback can look up full message from storage if needed Bytes empty_hash; LXMessage msg(empty_hash, empty_hash); msg.hash(message_hash); msg.state(Type::Message::DELIVERED); - DEBUG(">>> About to invoke callback"); -#ifdef ARDUINO - Serial.flush(); -#endif router->_delivered_callback(msg); - DEBUG(">>> Callback returned"); -#ifdef ARDUINO - Serial.flush(); -#endif } } } - // Remove from pending proofs - DEBUG(">>> Clearing slot"); -#ifdef ARDUINO - Serial.flush(); -#endif slot->clear(); - DEBUG(">>> Slot cleared"); -#ifdef ARDUINO - Serial.flush(); -#endif } else { snprintf(buf, sizeof(buf), "Received proof for unknown packet: %.16s...", packet_hash.toHex().c_str()); DEBUG(buf); } - DEBUG(">>> PROOF CALLBACK EXIT"); -#ifdef ARDUINO - Serial.flush(); -#endif } // Constructor @@ -564,12 +520,12 @@ void LXMRouter::handle_outbound(LXMessage& message) { // Pack the message message.pack(); - // Check if message fits in a single packet - use OPPORTUNISTIC if so - // OPPORTUNISTIC is simpler (no link needed) and works when identity is known - if (message.packed_size() <= Type::Constants::ENCRYPTED_PACKET_MDU) { - INFO(" Message fits in single packet, will use OPPORTUNISTIC delivery"); + // Check if message fits in a single LoRa packet - use OPPORTUNISTIC if so + // Use LORA_ENCRYPTED_PACKET_MDU (159) to ensure packet fits within LoRa wire MTU (255) + if (message.packed_size() <= Type::Constants::LORA_ENCRYPTED_PACKET_MDU) { + INFO(" Message fits in single LoRa packet, will use OPPORTUNISTIC delivery"); } else { - INFO(" Message too large for single packet, will use DIRECT (link) delivery"); + INFO(" Message too large for single LoRa packet, will use DIRECT (link) delivery"); } // Set state to outbound @@ -604,6 +560,20 @@ void LXMRouter::process_outbound() { DEBUG(buf); try { + // Check max delivery attempts + if (message.delivery_attempts() >= MAX_DELIVERY_ATTEMPTS) { + WARNING("Max delivery attempts reached for message to " + message.destination_hash().toHex()); + message.state(Type::Message::FAILED); + if (_failed_callback) { + _failed_callback(message); + } + failed_outbound_push(message); + LXMessage dummy; + pending_outbound_pop(dummy); + return; + } + message.increment_delivery_attempts(); + // If propagation-only mode is enabled, send via propagation node if (_propagation_only) { DEBUG(" Using PROPAGATED delivery (propagation-only mode)"); @@ -623,28 +593,20 @@ void LXMRouter::process_outbound() { return; } - // Determine delivery method based on message size - bool use_opportunistic = (message.packed_size() <= Type::Constants::ENCRYPTED_PACKET_MDU); + // Determine delivery method based on message size (LoRa-constrained threshold) + bool use_opportunistic = (message.packed_size() <= Type::Constants::LORA_ENCRYPTED_PACKET_MDU); if (use_opportunistic) { // OPPORTUNISTIC delivery - send as single encrypted packet DEBUG(" Using OPPORTUNISTIC delivery (single packet)"); - // Check if we have a path to the destination - if (!Transport::has_path(message.destination_hash())) { - // Request path from network - INFO(" No path to destination, requesting..."); - Transport::request_path(message.destination_hash()); - _next_outbound_process_time = now + PATH_REQUEST_WAIT; - return; - } - - // Try to recall the destination identity + // Try to recall the destination identity (needed to encrypt the packet) Identity dest_identity = Identity::recall(message.destination_hash()); if (!dest_identity) { - // Path exists but identity not cached yet - wait for announce - INFO(" Path exists but identity not known, waiting for announce..."); - _next_outbound_process_time = now + OUTBOUND_RETRY_DELAY; + // Identity not known - request path which may trigger an announce + INFO(" Destination identity not known, requesting path..."); + Transport::request_path(message.destination_hash()); + _next_outbound_process_time = now + PATH_REQUEST_WAIT; return; } @@ -949,8 +911,8 @@ void LXMRouter::on_packet(const Bytes& data, const Packet& packet) { snprintf(buf, sizeof(buf), " Unverified reason: %u", (uint8_t)message.unverified_reason()); DEBUG(buf); - // For Phase 1 MVP, we'll still accept messages with unknown source - // (signature will be validated later if source identity is learned) + // Accept messages with unknown source — signature will be validated + // later if the source identity is learned via announce if (message.unverified_reason() != Type::Message::SOURCE_UNKNOWN) { WARNING(" Rejecting message with invalid signature"); return; @@ -1286,9 +1248,11 @@ void LXMRouter::on_incoming_link_established(Link& link) { snprintf(buf, sizeof(buf), " Link ID: %s", link.link_id().toHex().c_str()); DEBUG(buf); - // Set up resource concluded callback to receive LXMF messages over this link + // Set up packet callback for single-packet LXMF messages (CONTEXT_NONE) + link.set_packet_callback(static_link_packet_callback); + // Set up resource concluded callback for multi-packet LXMF messages link.set_resource_concluded_callback(static_resource_concluded_callback); - DEBUG(" Resource callback registered for incoming LXMF messages"); + DEBUG(" Packet and resource callbacks registered for incoming LXMF messages"); } // Resource concluded callback (LXMF message received via DIRECT delivery) @@ -1368,11 +1332,6 @@ void LXMRouter::on_resource_concluded(const RNS::Resource& resource) { // ============== Propagation Node Support ============== -void LXMRouter::set_propagation_node_manager(PropagationNodeManager* manager) { - _propagation_manager = manager; - INFO("Propagation node manager set"); -} - void LXMRouter::set_outbound_propagation_node(const Bytes& node_hash) { if (node_hash.size() == 0) { _outbound_propagation_node = {}; @@ -1391,8 +1350,9 @@ void LXMRouter::set_outbound_propagation_node(const Bytes& node_hash) { } _outbound_propagation_node = node_hash; - char buf[64]; - snprintf(buf, sizeof(buf), "Set outbound propagation node to %.16s...", node_hash.toHex().c_str()); + char buf[96]; + snprintf(buf, sizeof(buf), "Set outbound propagation node (%d bytes): %s", + (int)node_hash.size(), node_hash.toHex().c_str()); INFO(buf); } @@ -1461,13 +1421,6 @@ bool LXMRouter::send_propagated(LXMessage& message) { // Get propagation node Bytes prop_node = _outbound_propagation_node; - if (prop_node.size() == 0 && _propagation_manager) { - DEBUG(" Looking for propagation node via manager..."); - auto nodes = _propagation_manager->get_nodes(); - snprintf(buf, sizeof(buf), " Manager has %zu nodes", nodes.size()); - DEBUG(buf); - prop_node = _propagation_manager->get_effective_node(); - } if (prop_node.size() == 0) { WARNING("No propagation node available for PROPAGATED delivery"); @@ -1517,15 +1470,12 @@ bool LXMRouter::send_propagated(LXMessage& message) { } // Generate propagation stamp if required by node - if (_propagation_manager) { - auto node_info = _propagation_manager->get_node(prop_node); - if (node_info && node_info.stamp_cost > 0) { - snprintf(buf, sizeof(buf), " Generating propagation stamp (cost=%u)...", node_info.stamp_cost); - DEBUG(buf); - Bytes stamp = message.generate_propagation_stamp(node_info.stamp_cost); - if (stamp.size() == 0) { - WARNING(" Failed to generate propagation stamp, sending anyway"); - } + if (_outbound_propagation_stamp_cost > 0) { + snprintf(buf, sizeof(buf), " Generating propagation stamp (cost=%u)...", _outbound_propagation_stamp_cost); + DEBUG(buf); + Bytes stamp = message.generate_propagation_stamp(_outbound_propagation_stamp_cost); + if (stamp.size() == 0) { + WARNING(" Failed to generate propagation stamp, sending anyway"); } } @@ -1561,6 +1511,46 @@ bool LXMRouter::send_propagated(LXMessage& message) { return true; } +// Static router pointer for sync callbacks (raw function pointers required by RequestReceipt) +static LXMRouter* _active_sync_router = nullptr; + +// Static callback wrappers for sync protocol +static void static_list_response_cb(const RequestReceipt& receipt) { + if (_active_sync_router) { + _active_sync_router->on_message_list_response(receipt.get_response()); + } else { + WARNING("list_response_cb: no active sync router!"); + } +} + +static void static_list_failed_cb(const RequestReceipt& receipt) { + WARNING("Propagation node list request failed"); + if (_active_sync_router) { + _active_sync_router->on_sync_failed(); + } +} + +static void static_get_response_cb(const RequestReceipt& receipt) { + if (_active_sync_router) { + _active_sync_router->on_message_get_response(receipt.get_response()); + } else { + WARNING("get_response_cb: no active sync router!"); + } +} + +static void static_get_failed_cb(const RequestReceipt& receipt) { + WARNING("Propagation node get request failed"); + if (_active_sync_router) { + _active_sync_router->on_sync_failed(); + } +} + +void LXMRouter::on_sync_failed() { + _sync_state = PR_FAILED; + _sync_progress = 0.0f; + _active_sync_router = nullptr; +} + void LXMRouter::request_messages_from_propagation_node() { if (_sync_state != PR_IDLE && _sync_state != PR_COMPLETE && _sync_state != PR_FAILED) { char buf[64]; @@ -1571,9 +1561,6 @@ void LXMRouter::request_messages_from_propagation_node() { // Get propagation node Bytes prop_node = _outbound_propagation_node; - if (!prop_node && _propagation_manager) { - prop_node = _propagation_manager->get_effective_node(); - } if (!prop_node) { WARNING("No propagation node available for sync"); @@ -1586,57 +1573,309 @@ void LXMRouter::request_messages_from_propagation_node() { INFO(buf); _sync_progress = 0.0f; - // Check if link exists and is active + // Request path if we don't have one + if (!Transport::has_path(prop_node)) { + INFO(" No path to propagation node, requesting..."); + Transport::request_path(prop_node); + } + + _sync_state = PR_PATH_REQUESTED; // Enter state machine; process_sync() advances it + _sync_start_time = Utilities::OS::time(); + process_sync(); +} + +void LXMRouter::process_sync() { + if (_sync_state == PR_IDLE || _sync_state == PR_COMPLETE || _sync_state == PR_FAILED) { + return; // Nothing to advance + } + + // Global timeout: 60s for entire sync operation + double elapsed = Utilities::OS::time() - _sync_start_time; + if (elapsed > 60.0) { + WARNING(" Propagation sync timed out"); + on_sync_failed(); + return; + } + + Bytes prop_node = _outbound_propagation_node; + if (!prop_node) { + _sync_state = PR_FAILED; + return; + } + + // For states that depend on an active link, check link health + if (_sync_state == PR_REQUEST_SENT || _sync_state == PR_RECEIVING || + _sync_state == PR_LINK_ESTABLISHED) { + if (!_outbound_propagation_link || + _outbound_propagation_link.status() == RNS::Type::Link::CLOSED) { + WARNING(" Propagation link lost during sync"); + on_sync_failed(); + return; + } + // Log status every ~10s + int elapsed_int = (int)elapsed; + if (elapsed_int > 0 && elapsed_int % 10 == 0) { + static int last_logged = -1; + if (elapsed_int != last_logged) { + last_logged = elapsed_int; + char buf[96]; + snprintf(buf, sizeof(buf), " Sync waiting: state=%d, link_status=%d, pending_reqs=%zu, elapsed=%ds", + (int)_sync_state, (int)_outbound_propagation_link.status(), + _outbound_propagation_link.pending_requests_count(), elapsed_int); + INFO(buf); + } + } + return; // Waiting for response callbacks + } + + // Check if link is already active (fast path for PR_PATH_REQUESTED / PR_LINK_ESTABLISHING) if (_outbound_propagation_link && _outbound_propagation_link.status() == RNS::Type::Link::ACTIVE) { _sync_state = PR_LINK_ESTABLISHED; - // TODO: Implement link.identify() and link.request() for full sync protocol - // For now, we log that sync would happen here - INFO(" Link active - sync protocol not yet implemented"); - INFO(" (Requires Link.identify() and Link.request() support)"); + _active_sync_router = this; - _sync_state = PR_COMPLETE; - _sync_progress = 1.0f; - if (_sync_complete_callback) { - _sync_complete_callback(0); + // Identify ourselves to the propagation node + _outbound_propagation_link.identify(_identity); + + // Build initial request: [nil, nil] — request message list + MsgPack::Packer packer; + packer.packArraySize(2); + packer.packNil(); + packer.packNil(); + Bytes request_data(packer.data(), packer.size()); + + // Request message list via "/get" path + Bytes path((uint8_t*)"/get", 4); + RequestReceipt receipt = _outbound_propagation_link.request( + path, request_data, + static_list_response_cb, + static_list_failed_cb + ); + + if (!receipt) { + WARNING(" Sync request failed - request not sent"); + on_sync_failed(); + return; } - } else { - // Need to establish link first + + _sync_state = PR_REQUEST_SENT; + _sync_progress = 0.1f; + INFO(" Sync request sent to propagation node"); + return; + } + + // PR_PATH_REQUESTED: wait for path, then advance to link establishing + if (_sync_state == PR_PATH_REQUESTED) { if (!Transport::has_path(prop_node)) { - INFO(" No path to propagation node, requesting..."); - Transport::request_path(prop_node); - _sync_state = PR_PATH_REQUESTED; - } else { - Identity node_identity = Identity::recall(prop_node); - if (!node_identity) { - INFO(" Propagation node identity not known"); - _sync_state = PR_FAILED; - return; - } - - Destination prop_dest( - node_identity, - RNS::Type::Destination::OUT, - RNS::Type::Destination::SINGLE, - "lxmf", - "propagation" - ); - - _outbound_propagation_link = Link(prop_dest); - _sync_state = PR_LINK_ESTABLISHING; - INFO(" Establishing link for sync..."); + return; // Still waiting for path } + + Identity node_identity = Identity::recall(prop_node); + if (!node_identity) { + INFO(" Propagation node identity not known"); + _sync_state = PR_FAILED; + return; + } + + Destination prop_dest( + node_identity, + RNS::Type::Destination::OUT, + RNS::Type::Destination::SINGLE, + "lxmf", + "propagation" + ); + + _outbound_propagation_link = Link(prop_dest); + _sync_state = PR_LINK_ESTABLISHING; + INFO(" Path arrived, establishing link for sync..."); + return; + } + + // PR_LINK_ESTABLISHING: link not yet active, check for failure + if (_sync_state == PR_LINK_ESTABLISHING) { + if (_outbound_propagation_link.status() == RNS::Type::Link::CLOSED) { + WARNING(" Propagation link closed before establishing"); + _sync_state = PR_FAILED; + } + // Otherwise still waiting for link to become ACTIVE } } void LXMRouter::on_message_list_response(const Bytes& response) { - // TODO: Implement when Link.request() is available - DEBUG("on_message_list_response: Not yet implemented"); + char buf[128]; + INFO("Received message list from propagation node"); + + if (!response || response.size() == 0) { + INFO(" Empty response — no messages available"); + _sync_state = PR_COMPLETE; + _sync_progress = 1.0f; + _active_sync_router = nullptr; + if (_sync_complete_callback) { + _sync_complete_callback(0); + } + return; + } + + try { + // Parse response: array of transient_id bytes + MsgPack::Unpacker unpacker; + unpacker.feed(response.data(), response.size()); + + MsgPack::arr_size_t arr_size; + unpacker.deserialize(arr_size); + + snprintf(buf, sizeof(buf), " Propagation node has %u messages", (unsigned)arr_size.size()); + INFO(buf); + + // Filter out already-seen transient IDs + // Build "wants" list + size_t wants_count = 0; + + std::vector available_ids; + for (size_t i = 0; i < arr_size.size(); i++) { + MsgPack::bin_t id_bin; + unpacker.deserialize(id_bin); + Bytes transient_id(id_bin); + if (!transient_ids_contains(transient_id)) { + available_ids.push_back(transient_id); + wants_count++; + } + } + + snprintf(buf, sizeof(buf), " Want %zu new messages (filtered %u already seen)", + wants_count, (unsigned)(arr_size.size() - wants_count)); + INFO(buf); + + if (wants_count == 0) { + INFO(" No new messages to download"); + _sync_state = PR_COMPLETE; + _sync_progress = 1.0f; + _active_sync_router = nullptr; + if (_sync_complete_callback) { + _sync_complete_callback(0); + } + return; + } + + // Build request: [wants, [], 0] + // wants = array of transient IDs we want + // [] = empty "have" list (we're not a peer) + // 0 = message limit (0 = no limit) + MsgPack::Packer req_packer; + req_packer.packArraySize(3); + + // wants array + req_packer.packArraySize(wants_count); + for (const auto& id : available_ids) { + req_packer.packBinary(id.data(), id.size()); + } + + // empty haves array + req_packer.packArraySize(0); + + // no limit (must be nil, not 0 — Python server treats 0 as "0 KB limit") + req_packer.packNil(); + + Bytes request_data(req_packer.data(), req_packer.size()); + + // Request the messages + Bytes path((uint8_t*)"/get", 4); + _outbound_propagation_link.request( + path, request_data, + static_get_response_cb, + static_get_failed_cb + ); + + _sync_state = PR_RECEIVING; + _sync_progress = 0.3f; + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to parse message list: %s", e.what()); + ERROR(buf); + _sync_state = PR_FAILED; + _active_sync_router = nullptr; + } } void LXMRouter::on_message_get_response(const Bytes& response) { - // TODO: Implement when Link.request() is available - DEBUG("on_message_get_response: Not yet implemented"); + char buf[128]; + INFO("Received messages from propagation node"); + + if (!response || response.size() == 0) { + INFO(" Empty response"); + _sync_state = PR_COMPLETE; + _sync_progress = 1.0f; + _active_sync_router = nullptr; + if (_sync_complete_callback) { + _sync_complete_callback(0); + } + return; + } + + size_t messages_received = 0; + + try { + // Parse response: array of lxmf_data bytes + MsgPack::Unpacker unpacker; + unpacker.feed(response.data(), response.size()); + + MsgPack::arr_size_t arr_size; + unpacker.deserialize(arr_size); + + snprintf(buf, sizeof(buf), " Processing %u messages from propagation node", (unsigned)arr_size.size()); + INFO(buf); + + // Collect received transient IDs for ack + std::vector received_ids; + + for (size_t i = 0; i < arr_size.size(); i++) { + MsgPack::bin_t data_bin; + unpacker.deserialize(data_bin); + Bytes lxmf_data(data_bin); + + // Process the message + process_propagated_lxmf(lxmf_data); + messages_received++; + + // Track transient ID + Bytes transient_id = Identity::full_hash(lxmf_data); + received_ids.push_back(transient_id); + + _sync_progress = 0.3f + 0.6f * ((float)(i + 1) / (float)arr_size.size()); + } + + // Send ack: [nil, haves] — acknowledge received messages + if (!received_ids.empty() && _outbound_propagation_link && + _outbound_propagation_link.status() == RNS::Type::Link::ACTIVE) { + MsgPack::Packer ack_packer; + ack_packer.packArraySize(2); + ack_packer.packNil(); + + ack_packer.packArraySize(received_ids.size()); + for (const auto& id : received_ids) { + ack_packer.packBinary(id.data(), id.size()); + } + + Bytes ack_data(ack_packer.data(), ack_packer.size()); + Bytes path((uint8_t*)"/get", 4); + _outbound_propagation_link.request(path, ack_data); + } + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to process messages from propagation node: %s", e.what()); + ERROR(buf); + } + + snprintf(buf, sizeof(buf), "Sync complete: received %zu messages", messages_received); + INFO(buf); + + _sync_state = PR_COMPLETE; + _sync_progress = 1.0f; + _active_sync_router = nullptr; + + if (_sync_complete_callback) { + _sync_complete_callback(messages_received); + } } void LXMRouter::process_propagated_lxmf(const Bytes& lxmf_data) { diff --git a/src-shim/LXMF/LXMRouter.h b/lib/microreticulum-shim/LXMF/LXMRouter.h similarity index 92% rename from src-shim/LXMF/LXMRouter.h rename to lib/microreticulum-shim/LXMF/LXMRouter.h index a3b883a6..b6a0f13d 100644 --- a/src-shim/LXMF/LXMRouter.h +++ b/lib/microreticulum-shim/LXMF/LXMRouter.h @@ -15,14 +15,11 @@ namespace LXMF { - // Forward declarations - class PropagationNodeManager; - /** * @brief LXMF Router - Message delivery orchestration * * Manages message queues, link establishment, and delivery for LXMF messages. - * Supports DIRECT delivery method (via established links) for Phase 1 MVP. + * Supports DIRECT, OPPORTUNISTIC, and PROPAGATED delivery methods. * * Usage: * LXMRouter router(identity, "/path/to/storage"); @@ -255,11 +252,19 @@ namespace LXMF { // ============== Propagation Node Support ============== /** - * @brief Set the propagation node manager + * @brief Set the stamp cost required by the outbound propagation node * - * @param manager Pointer to PropagationNodeManager (not owned) + * When set to a non-zero value, stamps will be generated before sending + * messages through the propagation node. + * + * @param cost Required stamp cost (0 = no stamp needed) */ - void set_propagation_node_manager(PropagationNodeManager* manager); + void set_outbound_propagation_stamp_cost(uint8_t cost) { _outbound_propagation_stamp_cost = cost; } + + /** + * @brief Get the stamp cost for the outbound propagation node + */ + uint8_t outbound_propagation_stamp_cost() const { return _outbound_propagation_stamp_cost; } /** * @brief Set the outbound propagation node @@ -317,6 +322,14 @@ namespace LXMF { */ void request_messages_from_propagation_node(); + /** + * @brief Advance propagation sync state machine + * + * Call periodically (e.g., in main loop) to advance sync after + * path arrival or link establishment. + */ + void process_sync(); + /** * @brief Get the current sync state * @@ -469,16 +482,26 @@ namespace LXMF { */ bool send_propagated(LXMessage& message); + public: /** * @brief Handle message list response from propagation node + * NOTE: Public for static callback access, not intended for direct use. */ void on_message_list_response(const RNS::Bytes& response); /** * @brief Handle message get response from propagation node + * NOTE: Public for static callback access, not intended for direct use. */ void on_message_get_response(const RNS::Bytes& response); + /** + * @brief Handle sync failure + * NOTE: Public for static callback access, not intended for direct use. + */ + void on_sync_failed(); + + private: /** * @brief Process received propagated LXMF data */ @@ -614,12 +637,14 @@ namespace LXMF { // Retry backoff double _next_outbound_process_time = 0.0; // Next time to process outbound queue - static constexpr double OUTBOUND_RETRY_DELAY = 5.0; // Seconds between retries - static constexpr double PATH_REQUEST_WAIT = 3.0; // Seconds to wait after path request + static constexpr double OUTBOUND_RETRY_DELAY = 10.0; // Seconds between retries (Python: DELIVERY_RETRY_WAIT = 10) + static constexpr double PATH_REQUEST_WAIT = 15.0; // Seconds to wait after path request (Python: 7s, but LoRa needs more RX window) + static constexpr int MAX_DELIVERY_ATTEMPTS = 5; // Max attempts before failing (Python: 5) + static constexpr int MAX_PATHLESS_TRIES = 1; // Attempts before requesting path (Python: 1) // Propagation node support - PropagationNodeManager* _propagation_manager = nullptr; RNS::Bytes _outbound_propagation_node; + uint8_t _outbound_propagation_stamp_cost = 0; RNS::Link _outbound_propagation_link{RNS::Type::NONE}; bool _fallback_to_propagation = true; bool _propagation_only = false; @@ -627,6 +652,7 @@ namespace LXMF { // Propagation sync state PropagationSyncState _sync_state = PR_IDLE; float _sync_progress = 0.0f; + double _sync_start_time = 0.0; SyncCompleteCallback _sync_complete_callback; // Locally delivered transient IDs circular buffer (zero heap fragmentation) diff --git a/src-shim/LXMF/LXMessage.cpp b/lib/microreticulum-shim/LXMF/LXMessage.cpp similarity index 83% rename from src-shim/LXMF/LXMessage.cpp rename to lib/microreticulum-shim/LXMF/LXMessage.cpp index 72280183..bba6e8a1 100644 --- a/src-shim/LXMF/LXMessage.cpp +++ b/lib/microreticulum-shim/LXMF/LXMessage.cpp @@ -125,14 +125,12 @@ const Bytes& LXMessage::pack() { _timestamp = Utilities::OS::time(); } - // 2. Create payload array: [timestamp, title, content, fields, stamp?] - matches Python LXMF exactly - // Python: msgpack.packb([self.timestamp, self.title, self.content, self.fields]) - // If stamp is present, it's appended as 5th element + // 2. Pack 4-element payload (without stamp) for hash/signature computation. + // Per Python LXMF: hash and signature are ALWAYS computed over the 4-element + // payload [timestamp, title, content, fields], even when a stamp is present. + // The stamp is only appended as a 5th element in the wire format. MsgPack::Packer packer; - - // Pack as array with 4 or 5 elements (5 if stamp present) - bool has_stamp = (_stamp.size() == LXStamper::STAMP_SIZE); - packer.packArraySize(has_stamp ? 5 : 4); + packer.packArraySize(4); // Element 0: timestamp (float64) packer.pack(_timestamp); @@ -152,19 +150,14 @@ const Bytes& LXMessage::pack() { } } - // Element 4 (optional): stamp - 32 bytes - if (has_stamp) { - packer.packBinary(_stamp.data(), _stamp.size()); - DEBUG(" Stamp included in payload (" + std::to_string(_stamp.size()) + " bytes)"); - } + Bytes payload_without_stamp(packer.data(), packer.size()); - Bytes packed_payload(packer.data(), packer.size()); - - // 3. Calculate hash: SHA256(dest_hash + source_hash + packed_payload) + // 3. Calculate hash: SHA256(dest_hash + source_hash + payload_without_stamp) + // Hash is always over the 4-element payload (matching Python LXMF) Bytes hashed_part; hashed_part << _destination_hash; hashed_part << _source_hash; - hashed_part << packed_payload; + hashed_part << payload_without_stamp; _hash = Identity::full_hash(hashed_part); @@ -185,21 +178,44 @@ const Bytes& LXMessage::pack() { throw std::runtime_error("Cannot sign message without source destination"); } - // 6. Pack final message: dest_hash + source_hash + signature + packed_payload + // 6. Build wire payload — append stamp as 5th element if present + bool has_stamp = (_stamp.size() == LXStamper::STAMP_SIZE); + Bytes wire_payload; + if (has_stamp) { + MsgPack::Packer wire_packer; + wire_packer.packArraySize(5); + wire_packer.pack(_timestamp); + wire_packer.packBinary(_title.data(), _title.size()); + wire_packer.packBinary(_content.data(), _content.size()); + wire_packer.packMapSize(_fields_count); + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (_fields_pool[i].in_use) { + wire_packer.packBinary(_fields_pool[i].key.data(), _fields_pool[i].key.size()); + wire_packer.packBinary(_fields_pool[i].value.data(), _fields_pool[i].value.size()); + } + } + wire_packer.packBinary(_stamp.data(), _stamp.size()); + wire_payload = Bytes(wire_packer.data(), wire_packer.size()); + DEBUG(" Stamp included in wire payload (" + std::to_string(_stamp.size()) + " bytes)"); + } else { + wire_payload = payload_without_stamp; + } + + // 7. Pack final message: dest_hash + source_hash + signature + wire_payload _packed.clear(); _packed << _destination_hash; _packed << _source_hash; _packed << _signature; - _packed << packed_payload; + _packed << wire_payload; _packed_valid = true; - // 7. Determine delivery method and representation - size_t content_size = packed_payload.size() - Type::Constants::TIMESTAMP_SIZE - Type::Constants::STRUCT_OVERHEAD; + // 8. Determine delivery method and representation + size_t content_size = wire_payload.size() - Type::Constants::TIMESTAMP_SIZE - Type::Constants::STRUCT_OVERHEAD; - // For Phase 1 MVP, we only support DIRECT delivery if (_desired_method == Type::Message::DIRECT) { - if (content_size <= Type::Constants::LINK_PACKET_MAX_CONTENT) { + // Use LoRa-constrained limit (63 bytes content) to ensure link packets fit within LoRa wire MTU + if (content_size <= Type::Constants::LORA_LINK_PACKET_MAX_CONTENT) { _method = Type::Message::DIRECT; _representation = Type::Message::PACKET; INFO(" Message will be sent as single packet (" + std::to_string(_packed.size()) + " bytes)"); @@ -208,8 +224,25 @@ const Bytes& LXMessage::pack() { _representation = Type::Message::RESOURCE; INFO(" Message will be sent as resource (" + std::to_string(_packed.size()) + " bytes)"); } + } else if (_desired_method == Type::Message::PROPAGATED) { + // PROPAGATED: always use resource transfer to propagation node + _method = Type::Message::PROPAGATED; + _representation = Type::Message::RESOURCE; + INFO(" Message will be sent via propagation (" + std::to_string(_packed.size()) + " bytes)"); + } else if (_desired_method == Type::Message::OPPORTUNISTIC) { + // OPPORTUNISTIC: single encrypted packet, no link required + if (_packed.size() <= Type::Constants::LORA_ENCRYPTED_PACKET_MDU) { + _method = Type::Message::OPPORTUNISTIC; + _representation = Type::Message::PACKET; + INFO(" Message will be sent opportunistically (" + std::to_string(_packed.size()) + " bytes)"); + } else { + // Too large for single packet, fall back to DIRECT + _method = Type::Message::DIRECT; + _representation = Type::Message::RESOURCE; + INFO(" Message too large for OPPORTUNISTIC, using DIRECT resource (" + std::to_string(_packed.size()) + " bytes)"); + } } else { - WARNING("Only DIRECT delivery method is supported in Phase 1 MVP"); + // Default fallback _method = Type::Message::DIRECT; _representation = Type::Message::PACKET; } @@ -218,7 +251,7 @@ const Bytes& LXMessage::pack() { INFO("Message packed successfully (" + std::to_string(_packed.size()) + " bytes total)"); DEBUG(" Overhead: " + std::to_string(Type::Constants::LXMF_OVERHEAD) + " bytes"); - DEBUG(" Payload: " + std::to_string(packed_payload.size()) + " bytes"); + DEBUG(" Payload: " + std::to_string(wire_payload.size()) + " bytes"); return _packed; } @@ -367,10 +400,29 @@ LXMessage LXMessage::unpack_from_bytes(const Bytes& lxmf_bytes, Type::Message::M } // 4. Calculate hash for verification + // Per Python LXMF: hash is computed over 4-element payload (without stamp). + // If stamp was present, re-pack without it. + Bytes payload_for_hash; + if (stamp.size() == LXStamper::STAMP_SIZE) { + MsgPack::Packer repacker; + repacker.packArraySize(4); + repacker.pack(timestamp); + repacker.packBinary(title.data(), title.size()); + repacker.packBinary(content.data(), content.size()); + repacker.packMapSize(temp_fields_count); + for (size_t i = 0; i < temp_fields_count; ++i) { + repacker.packBinary(temp_fields[i].key.data(), temp_fields[i].key.size()); + repacker.packBinary(temp_fields[i].value.data(), temp_fields[i].value.size()); + } + payload_for_hash = Bytes(repacker.data(), repacker.size()); + } else { + payload_for_hash = packed_payload; + } + Bytes hashed_part; hashed_part << destination_hash; hashed_part << source_hash; - hashed_part << packed_payload; + hashed_part << payload_for_hash; message._hash = Identity::full_hash(hashed_part); @@ -438,21 +490,22 @@ bool LXMessage::validate_signature() { } } - // Reconstruct signed part + // Reconstruct signed part — must match pack() exactly Bytes hashed_part; hashed_part << _destination_hash; hashed_part << _source_hash; - // Need to repack payload for hashed_part + // Repack 4-element payload for hash/sig (without stamp, matching Python LXMF) MsgPack::Packer packer; - packer.serialize(_timestamp); - packer.serialize(_title); - packer.serialize(_content); - packer.serialize((uint32_t)_fields_count); + packer.packArraySize(4); + packer.pack(_timestamp); + packer.packBinary(_title.data(), _title.size()); + packer.packBinary(_content.data(), _content.size()); + packer.packMapSize(_fields_count); for (size_t i = 0; i < MAX_FIELDS; ++i) { if (_fields_pool[i].in_use) { - packer.serialize(_fields_pool[i].key); - packer.serialize(_fields_pool[i].value); + packer.packBinary(_fields_pool[i].key.data(), _fields_pool[i].key.size()); + packer.packBinary(_fields_pool[i].value.data(), _fields_pool[i].value.size()); } } Bytes packed_payload(packer.data(), packer.size()); diff --git a/src-shim/LXMF/LXMessage.h b/lib/microreticulum-shim/LXMF/LXMessage.h similarity index 98% rename from src-shim/LXMF/LXMessage.h rename to lib/microreticulum-shim/LXMF/LXMessage.h index f81641ce..c96e1e26 100644 --- a/src-shim/LXMF/LXMessage.h +++ b/lib/microreticulum-shim/LXMF/LXMessage.h @@ -219,6 +219,9 @@ namespace LXMF { */ inline void state(Type::Message::State state) { _state = state; } + inline int delivery_attempts() const { return _delivery_attempts; } + inline void increment_delivery_attempts() { _delivery_attempts++; } + /** * @brief Get message hash (ID) */ @@ -386,6 +389,7 @@ namespace LXMF { // Message state Type::Message::State _state = Type::Message::GENERATING; + int _delivery_attempts = 0; // Signature validation bool _signature_validated = false; diff --git a/src-shim/LXMF/LXStamper.cpp b/lib/microreticulum-shim/LXMF/LXStamper.cpp similarity index 100% rename from src-shim/LXMF/LXStamper.cpp rename to lib/microreticulum-shim/LXMF/LXStamper.cpp diff --git a/src-shim/LXMF/LXStamper.h b/lib/microreticulum-shim/LXMF/LXStamper.h similarity index 100% rename from src-shim/LXMF/LXStamper.h rename to lib/microreticulum-shim/LXMF/LXStamper.h diff --git a/src-shim/LXMF/MessageStore.cpp b/lib/microreticulum-shim/LXMF/MessageStore.cpp similarity index 100% rename from src-shim/LXMF/MessageStore.cpp rename to lib/microreticulum-shim/LXMF/MessageStore.cpp diff --git a/src-shim/LXMF/MessageStore.h b/lib/microreticulum-shim/LXMF/MessageStore.h similarity index 100% rename from src-shim/LXMF/MessageStore.h rename to lib/microreticulum-shim/LXMF/MessageStore.h diff --git a/src-shim/LXMF/PropagationNodeManager.cpp b/lib/microreticulum-shim/LXMF/PropagationNodeManager.cpp similarity index 93% rename from src-shim/LXMF/PropagationNodeManager.cpp rename to lib/microreticulum-shim/LXMF/PropagationNodeManager.cpp index 06315791..c50d285a 100644 --- a/src-shim/LXMF/PropagationNodeManager.cpp +++ b/lib/microreticulum-shim/LXMF/PropagationNodeManager.cpp @@ -132,24 +132,21 @@ PropagationNodeInfo PropagationNodeManager::parse_announce_data(const Bytes& app unpacker.deserialize(key); if (key == PN_META_NAME) { - // Name is a binary/string + // Python packs name as bytes (bin type via str.encode("utf-8")) MsgPack::bin_t name_bin; unpacker.deserialize(name_bin); - info.name = std::string(name_bin.begin(), name_bin.end()); - } else { - // Skip other metadata fields by reading and discarding - // Try to read as binary first (most common), fall back to int - try { - MsgPack::bin_t skip_bin; - unpacker.deserialize(skip_bin); - } catch (...) { - try { - int64_t skip_int; - unpacker.deserialize(skip_int); - } catch (...) { - // Give up - might be complex type - } + if (!name_bin.empty()) { + info.name = std::string(name_bin.begin(), name_bin.end()); } + // On type mismatch (e.g. str type from non-standard node), + // type_error() silently advances curr_index past the element. + // Name stays empty and will use default fallback below. + } else { + // Skip unknown metadata value: deserialize as int64_t. + // On type mismatch, type_error() increments curr_index by 1, + // which correctly skips to the next element regardless of type. + int64_t skip_val; + unpacker.deserialize(skip_val); } } diff --git a/src-shim/LXMF/PropagationNodeManager.h b/lib/microreticulum-shim/LXMF/PropagationNodeManager.h similarity index 100% rename from src-shim/LXMF/PropagationNodeManager.h rename to lib/microreticulum-shim/LXMF/PropagationNodeManager.h diff --git a/src-shim/LXMF/Type.h b/lib/microreticulum-shim/LXMF/Type.h similarity index 80% rename from src-shim/LXMF/Type.h rename to lib/microreticulum-shim/LXMF/Type.h index 0e468261..b6626283 100644 --- a/src-shim/LXMF/Type.h +++ b/lib/microreticulum-shim/LXMF/Type.h @@ -1,6 +1,6 @@ #pragma once -#include // microReticulum's Type.h, resolved via -I deps/microReticulum/src +#include #include @@ -43,7 +43,7 @@ namespace LXMF { */ enum Method : uint8_t { OPPORTUNISTIC = 0x01, ///< Single packet, fire-and-forget - DIRECT = 0x02, ///< Via established link (Phase 1 MVP) + DIRECT = 0x02, ///< Via established link PROPAGATED = 0x03, ///< Store-and-forward via propagation nodes PAPER = 0x05 ///< QR code / paper-based transfer }; @@ -96,6 +96,13 @@ namespace LXMF { // With an MTU of 500, encrypted packet MDU is 391 bytes static const uint16_t ENCRYPTED_PACKET_MDU = RNS::Type::Packet::ENCRYPTED_MDU + TIMESTAMP_SIZE; // 391 bytes + // LoRa-constrained ENCRYPTED_PACKET_MDU (for devices with LoRa interfaces) + // LoRa wire MTU=255, SINGLE header=19, IFAC=1 → max encrypted payload=235 + // Subtract ephemeral key(32) + IV(16) + HMAC(32) = 155 bytes for ciphertext + // Max AES blocks: floor(155/16)=9 → max padded plaintext=144 → max plaintext=143 + // Add back destination hash(16) stripped in OPPORTUNISTIC: max packed_size=159 + static const uint16_t LORA_ENCRYPTED_PACKET_MDU = 159; + /** * @brief Max content in single encrypted packet: 295 bytes * @@ -111,14 +118,27 @@ namespace LXMF { */ static const uint16_t LINK_PACKET_MDU = RNS::Type::Link::MDU; + // LoRa-constrained LINK_PACKET_MDU (for devices with LoRa interfaces) + // LoRa wire MTU=255, SX1262 adds 1 IFAC byte → max at transmit=254 + // Link header=19 bytes (2 flags + 1 context + 16 link_id) + // Max Token output = 254 - 19 = 235 bytes + // Token = IV(16) + ciphertext + HMAC(32), max ciphertext = 235 - 48 = 187 + // Max AES blocks: floor(187/16)=11 → max padded ciphertext=176 + // Max plaintext: 175 (PKCS7: 175 mod 16 = 15, pad 1 → 176) + static const uint16_t LORA_LINK_PACKET_MDU = 175; + /** - * @brief Max content in single link packet: 319 bytes (Phase 1 MVP limit) + * @brief Max content in single link packet: 319 bytes * * Calculation: LINK_PACKET_MDU - LXMF_OVERHEAD * Messages larger than 319 bytes will use Resource transfer. */ static const uint16_t LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD; + // LoRa-constrained: max content in single link packet = 175 - 112 = 63 bytes + // Messages with content > 63 bytes sent over links will use Resource transfer + static const uint16_t LORA_LINK_PACKET_MAX_CONTENT = LORA_LINK_PACKET_MDU - LXMF_OVERHEAD; + // Plain (unencrypted) packet MDU static const uint16_t PLAIN_PACKET_MDU = RNS::Type::Packet::PLAIN_MDU; diff --git a/src-shim/MessageBase.h b/lib/microreticulum-shim/MessageBase.h similarity index 100% rename from src-shim/MessageBase.h rename to lib/microreticulum-shim/MessageBase.h diff --git a/src-shim/ObjectPool.h b/lib/microreticulum-shim/ObjectPool.h similarity index 100% rename from src-shim/ObjectPool.h rename to lib/microreticulum-shim/ObjectPool.h diff --git a/src-shim/PSRAMAllocator.h b/lib/microreticulum-shim/PSRAMAllocator.h similarity index 100% rename from src-shim/PSRAMAllocator.h rename to lib/microreticulum-shim/PSRAMAllocator.h diff --git a/src-shim/SegmentAccumulator.cpp b/lib/microreticulum-shim/SegmentAccumulator.cpp similarity index 100% rename from src-shim/SegmentAccumulator.cpp rename to lib/microreticulum-shim/SegmentAccumulator.cpp diff --git a/src-shim/SegmentAccumulator.h b/lib/microreticulum-shim/SegmentAccumulator.h similarity index 100% rename from src-shim/SegmentAccumulator.h rename to lib/microreticulum-shim/SegmentAccumulator.h diff --git a/src-shim/Utilities/Print.cpp b/lib/microreticulum-shim/Utilities/Print.cpp similarity index 100% rename from src-shim/Utilities/Print.cpp rename to lib/microreticulum-shim/Utilities/Print.cpp diff --git a/src-shim/Utilities/Print.h b/lib/microreticulum-shim/Utilities/Print.h similarity index 100% rename from src-shim/Utilities/Print.h rename to lib/microreticulum-shim/Utilities/Print.h diff --git a/src-shim/Utilities/Stream.cpp b/lib/microreticulum-shim/Utilities/Stream.cpp similarity index 100% rename from src-shim/Utilities/Stream.cpp rename to lib/microreticulum-shim/Utilities/Stream.cpp diff --git a/src-shim/Utilities/Stream.h b/lib/microreticulum-shim/Utilities/Stream.h similarity index 100% rename from src-shim/Utilities/Stream.h rename to lib/microreticulum-shim/Utilities/Stream.h diff --git a/lib/microreticulum-shim/library.json b/lib/microreticulum-shim/library.json new file mode 100644 index 00000000..b97fe190 --- /dev/null +++ b/lib/microreticulum-shim/library.json @@ -0,0 +1,7 @@ +{ + "name": "microreticulum-shim", + "description": "Fork-only files vendored into pyxis to overlay on top of vanilla attermann/microReticulum @ 0.3.0. Eventual home: standalone microLXMF library + upstream PRs for the cross-cutting pieces. See pyxis_microReticulum_graft_spike_findings.md.", + "version": "0.0.0", + "frameworks": ["arduino"], + "platforms": "*" +} diff --git a/lib/tdeck_ui/Hardware/TDeck/SDLogger.cpp b/lib/tdeck_ui/Hardware/TDeck/SDLogger.cpp index 336a5cf3..6475276c 100644 --- a/lib/tdeck_ui/Hardware/TDeck/SDLogger.cpp +++ b/lib/tdeck_ui/Hardware/TDeck/SDLogger.cpp @@ -60,7 +60,7 @@ bool SDLogger::init() { _active = true; // Set log callback to capture all logs - RNS::setLogCallback(logCallback); + RNS::set_log_callback(logCallback); Serial.println("[SDLogger] SD card logging active"); @@ -192,7 +192,7 @@ void SDLogger::close() { _active = false; } // Restore default logging - RNS::setLogCallback(nullptr); + RNS::set_log_callback(nullptr); } #else diff --git a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp index 00b25380..e80d4ebe 100644 --- a/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/AnnounceListScreen.cpp @@ -11,6 +11,7 @@ #include "Transport.h" #include "Identity.h" #include "Destination.h" +#include "Persistence/DestinationEntry.h" // RNS::Persistence::DestinationEntry (post-graft path) #include "Utilities/OS.h" #include "../LVGL/LVGLInit.h" #include @@ -137,15 +138,20 @@ void AnnounceListScreen::refresh() { _dest_hash_pool.clear(); _empty_label = nullptr; - // Get destination table from Transport - const auto& dest_table = Transport::get_destination_table(); + // Get path table from Transport. Pre-graft the fork called this + // get_destination_table(); upstream microReticulum @ 0.3.0 renamed it + // to get_path_table() (the same map of dest_hash → DestinationEntry). + const auto& dest_table = Transport::get_path_table(); // Compute name_hash for lxmf.delivery to filter announces Bytes lxmf_delivery_name_hash = Destination::name_hash("lxmf", "delivery"); for (auto it = dest_table.begin(); it != dest_table.end(); ++it) { const Bytes& dest_hash = it->first; - const Transport::DestinationEntry& dest_entry = it->second; + // Pre-graft: nested Transport::DestinationEntry. Upstream moved + // this to RNS::Persistence::DestinationEntry in + // Persistence/DestinationEntry.h. + const RNS::Persistence::DestinationEntry& dest_entry = it->second; // Check if this destination has a known identity (was announced properly) Identity identity = Identity::recall(dest_hash); diff --git a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp index 27ef06d4..c544c02d 100644 --- a/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp +++ b/lib/tdeck_ui/UI/LXMF/ConversationListScreen.cpp @@ -432,9 +432,14 @@ void ConversationListScreen::update_status() { lv_obj_set_style_text_color(_label_wifi, Theme::textMuted(), 0); } - // Update LoRa RSSI - if (_lora_interface) { - float rssi_f = _lora_interface->get_rssi(); + // Update LoRa RSSI. Pre-graft this called Interface::get_rssi() — + // a virtual method the fork added that vanilla upstream doesn't + // expose on the Interface base class. To restore: change + // _lora_interface to a concrete SX1262Interface* (the method is + // still defined there, just non-virtual). Disabled for the spike. + if (false && _lora_interface) { + float rssi_f = 0.0f; + (void)rssi_f; int rssi = (int)rssi_f; // Only show RSSI if we've received at least one packet (RSSI != 0) @@ -486,14 +491,14 @@ void ConversationListScreen::update_status() { int central_count = 0; int peripheral_count = 0; - // Get connection counts from BLE interface - // The interface stores stats about connections - // Use get_stats() map if available, otherwise show "--" - auto stats = _ble_interface->get_stats(); - auto it_c = stats.find("central_connections"); - auto it_p = stats.find("peripheral_connections"); - if (it_c != stats.end()) central_count = (int)it_c->second; - if (it_p != stats.end()) peripheral_count = (int)it_p->second; + // Pre-graft: BLEInterface::get_stats() was a virtual override on + // RNS::Interface. Vanilla upstream doesn't expose get_stats on the + // Interface base; the method is still defined on BLEInterface as + // non-virtual. To restore: change _ble_interface to BLEInterface*. + // Disabled for the spike. + // auto stats = _ble_interface->get_stats(); + (void)central_count; + (void)peripheral_count; char ble_text[32]; snprintf(ble_text, sizeof(ble_text), "%s %d|%d", LV_SYMBOL_BLUETOOTH, central_count, peripheral_count); diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index 56cb6a13..7f1f8780 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -42,6 +42,10 @@ static std::shared_ptr s_lxst_announce_handler; UIManager::UIManager(Reticulum& reticulum, ::LXMF::LXMRouter& router, ::LXMF::MessageStore& store) : _reticulum(reticulum), _router(router), _store(store), + // Vanilla upstream RNS::Destination has no default ctor; construct in + // a Type::NONE state, then assign a real Destination later. (The fork + // had a default ctor that pyxis was implicitly relying on.) + _lxst_destination(RNS::Type::NONE), _current_screen(SCREEN_CONVERSATION_LIST), _conversation_list_screen(nullptr), _chat_screen(nullptr), @@ -680,8 +684,12 @@ void UIManager::send_message(const Bytes& dest_hash, const String& content) { std::string msg = "Sending message to " + hash_hex + "..."; INFO(msg.c_str()); - // Mark recipient as a persistent contact (survives reboot) - Identity::mark_persistent(dest_hash); + // Pre-graft: Identity::mark_persistent(dest_hash) — fork-only API for + // the 5s fast-flush semantics. Vanilla upstream relies on microStore's + // dirty-tracking + reticulum->should_persist_data() to decide what + // gets written. If we observe lost contacts after crashes, revisit + // microStore flush cadence rather than re-adding the fork API. + // (void)Identity::mark_persistent(dest_hash); // Get our source destination (needed for signing) Destination source = _router.delivery_destination(); @@ -735,8 +743,8 @@ void UIManager::on_message_received(::LXMF::LXMessage& message) { std::string msg = "Message received from " + source_hex + "..."; INFO(msg.c_str()); - // Mark sender as a persistent contact (survives reboot) - RNS::Identity::mark_persistent(message.source_hash()); + // Pre-graft: RNS::Identity::mark_persistent — fork-only. See note above. + // (void)RNS::Identity::mark_persistent(message.source_hash()); // Save to store _store.save_message(message); diff --git a/platformio.ini b/platformio.ini index c23bf164..47dd6d9d 100644 --- a/platformio.ini +++ b/platformio.ini @@ -64,11 +64,15 @@ build_flags = -Ilib -Ilib/libbz2 -Ideps/microReticulum/src - -Isrc-shim - -Isrc-shim/Cryptography - -Isrc-shim/Instrumentation - -Isrc-shim/Utilities + ; -Ilib/microreticulum-shim alone is enough — subdir lookups use , + ; , etc. Don't add -Ilib/microreticulum-shim/Utilities on top: + ; that puts our Stream.h on the GLOBAL header path and breaks Arduino's + ; Wire.cpp (`class TwoWire: public Stream` finds ours first). + -Ilib/microreticulum-shim -DBZ_NO_STDIO + ; microStore SPIFFS adapter — replaces pyxis's pre-graft + ; lib/universal_filesystem/ (which targeted SPIFFS directly). + -DUSTORE_USE_SPIFFS -DUSE_BLUEDROID -Os -DCORE_DEBUG_LEVEL=2 @@ -150,11 +154,15 @@ build_flags = -Ilib -Ilib/libbz2 -Ideps/microReticulum/src - -Isrc-shim - -Isrc-shim/Cryptography - -Isrc-shim/Instrumentation - -Isrc-shim/Utilities + ; -Ilib/microreticulum-shim alone is enough — subdir lookups use , + ; , etc. Don't add -Ilib/microreticulum-shim/Utilities on top: + ; that puts our Stream.h on the GLOBAL header path and breaks Arduino's + ; Wire.cpp (`class TwoWire: public Stream` finds ours first). + -Ilib/microreticulum-shim -DBZ_NO_STDIO + ; microStore SPIFFS adapter — replaces pyxis's pre-graft + ; lib/universal_filesystem/ (which targeted SPIFFS directly). + -DUSTORE_USE_SPIFFS -DUSE_NIMBLE -DCONFIG_BT_NIMBLE_MEM_ALLOC_MODE_EXTERNAL=1 -Os diff --git a/src/main.cpp b/src/main.cpp index 1b902c6d..09bb2062 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,7 +18,10 @@ #include // Filesystem -#include +// Was: (pyxis-provided RNS::FileSystem wrapper). +// Post-graft: microStore ships its own SPIFFS adapter, activated via +// -DUSTORE_USE_SPIFFS in platformio.ini. +#include #include #include #include @@ -620,9 +623,17 @@ void setup_wifi() { void setup_hardware() { INFO("\n=== Hardware Initialization ==="); - // Initialize SPIFFS for persistence via UniversalFileSystem - // NOTE: Do NOT call SPIFFS.begin() here - UniversalFileSystem::init() handles it - static RNS::FileSystem fs = new UniversalFileSystem(); + // Initialize SPIFFS for persistence. + // + // Pre-graft: pyxis used its own RNS::FileSystem(new UniversalFileSystem()) + // wrapper. Vanilla upstream microReticulum @ 0.3.0 deleted RNS::FileSystem + // entirely and replaced it with microStore (an out-of-tree dep). microStore + // ships an SPIFFS adapter activated by -DUSTORE_USE_SPIFFS that does the + // exact same thing pyxis's UniversalFileSystem did, so we use it directly. + // + // Pyxis's lib/universal_filesystem/ is now dead code on this build path and + // can be deleted once the graft lands. + static microStore::Adapters::SPIFFSFileSystem fs; if (!fs.init()) { ERROR("FileSystem mount failed!"); } else {