From eced19abfc40cc818530afa84f71972fb63fe602 Mon Sep 17 00:00:00 2001 From: liu weikai Date: Mon, 20 Jul 2026 22:28:48 +0800 Subject: [PATCH] refactor(storage): partition protocol persistence --- ..._PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md | 265 ++ ...ROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md | 156 + .../CHAT_DELIVERY_RUNTIME_SPEC.md | 35 +- .../PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md | 266 ++ .../RUNTIME_OWNERSHIP_BOUNDARY_FREEZE.md | 86 + .../chat/delivery/chat_message_ledger.h | 82 +- .../include/chat/domain/mesh_peer_directory.h | 19 + .../chat/infra/mesh_peer_directory_core.h | 3 + .../include/chat/ports/i_chat_store.h | 66 +- .../include/chat/ports/i_contact_store.h | 7 + .../include/chat/ports/i_node_store.h | 6 + .../chat/ports/i_protocol_peer_repository.h | 25 + .../include/chat/usecase/chat_service.h | 48 + .../include/chat/usecase/contact_service.h | 4 + .../src/delivery/chat_message_ledger.cpp | 579 ++- .../src/infra/mesh_peer_directory_core.cpp | 53 +- .../core_chat/src/usecase/chat_service.cpp | 387 +- .../core_chat/src/usecase/contact_service.cpp | 20 + .../tests/test_chat_message_ledger.cpp | 108 + .../tests/test_chat_service_resend.cpp | 50 +- .../test_esp_sd_store_read_state_contract.cpp | 158 +- .../test_mesh_peer_directory_contract.cpp | 35 + .../test_protocol_chat_storage_v2_codec.cpp | 104 + .../test_protocol_peer_storage_v2_codec.cpp | 179 + .../app/app_context_platform_bindings.h | 10 +- .../chat/chat_workspace_snapshot.h | 2 +- .../src/chat/chat_workspace_model.cpp | 2 +- .../ui/screens/chat/chat_ui_controller.h | 1 - .../chat_presentation_source.cpp | 2 +- .../runtime_chat_action_sink.cpp | 2 +- .../chat/chat_conversation_components.cpp | 2 +- .../ui/screens/chat/chat_ui_controller.cpp | 19 +- .../tests/test_chat_presentation_source.cpp | 1 + .../arduino_common/include/app/app_context.h | 14 +- .../arduino_common/chat/infra/contact_store.h | 68 - .../chat/infra/lxmf/lxmf_memory.h | 88 +- .../chat/infra/meshtastic/node_store.h | 98 - .../chat/infra/store/fixed_slot_journal.h | 92 + .../chat/infra/store/protocol_chat_codec.h | 118 + .../chat/infra/store/protocol_peer_codec.h | 47 + .../infra/store/sd_protocol_peer_repository.h | 216 + .../chat/infra/store/sd_store.h | 414 +- .../arduino_common/memory/psram_allocator.h | 96 + .../esp/arduino_common/src/app_context.cpp | 27 +- .../src/app_context_platform_bindings.cpp | 296 +- .../src/chat/infra/contact_store.cpp | 246 -- .../chat/infra/meshcore/meshcore_adapter.cpp | 2 + .../src/chat/infra/meshtastic/node_store.cpp | 853 ---- .../chat/infra/store/fixed_slot_journal.cpp | 276 ++ .../chat/infra/store/protocol_chat_codec.cpp | 789 ++++ .../chat/infra/store/protocol_peer_codec.cpp | 724 +++ .../store/sd_protocol_peer_repository.cpp | 2195 ++++++++++ .../src/chat/infra/store/sd_store.cpp | 3862 +++++++++-------- ...latform_ui_reticulum_directory_runtime.cpp | 2 +- scripts/check_esp_stack_hygiene.py | 13 +- 55 files changed, 9368 insertions(+), 3950 deletions(-) create mode 100644 docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md create mode 100644 docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md create mode 100644 docs/specification/PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md create mode 100644 modules/core_chat/include/chat/ports/i_protocol_peer_repository.h create mode 100644 modules/core_chat/tests/test_protocol_chat_storage_v2_codec.cpp create mode 100644 modules/core_chat/tests/test_protocol_peer_storage_v2_codec.cpp delete mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/contact_store.h delete mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/meshtastic/node_store.h create mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h create mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h create mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h create mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h create mode 100644 platform/esp/arduino_common/include/platform/esp/arduino_common/memory/psram_allocator.h delete mode 100644 platform/esp/arduino_common/src/chat/infra/contact_store.cpp delete mode 100644 platform/esp/arduino_common/src/chat/infra/meshtastic/node_store.cpp create mode 100644 platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp create mode 100644 platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp create mode 100644 platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp create mode 100644 platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp diff --git a/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md b/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md new file mode 100644 index 00000000..b0cd61ee --- /dev/null +++ b/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md @@ -0,0 +1,265 @@ +# Protocol-Partitioned Storage V2 Detailed Design + +## Components + +### `FixedSlotJournalEngine` + +Owns journal framing and descriptor validation: + +```text +magic | schema | protocol | kind | slot_size | descriptor_crc +slot 0 +slot 1 +... +``` + +The engine reports `Missing`, `Ready`, `Incompatible`, `PartialTail`, or +`IoError`. It never interprets protocol payloads. + +`replaceFileAtomically` and `recoverAtomicFile` own snapshot rename recovery. +Callers must not duplicate remove/rename sequences. + +### Protocol Chat Codecs + +`protocol_chat_codec` owns MT, MC, and RT message/catalog/read/status/seen slot +layouts. Each slot has its own CRC. + +Text capacity is protocol-fixed. It is not represented by a single large union +record. RT-only identity and LXMF hash fields exist only in RT slots. + +### `SdStore` + +Owns: + +- message segments; +- catalog hot projection; +- read-watermark hot projection; +- outgoing status hot projection; +- RT seen hot projection. + +Hot vectors use strict PSRAM allocators. The 512-byte codec scratch buffer is a +store member, not a large task-stack local. + +### Protocol Peer Codecs + +`protocol_peer_codec` owns protocol peer and contact slots. + +Peer slot rules: + +- common prefix: identity kind, timestamps, display fact, source, radio + observations, device metrics, position; +- MT suffix: MT node facts, next hop, public key, verification state; +- MC suffix: MC node facts, public key, peer hash, next hop, node-id hint; +- RT suffix: destination/identity hashes, public keys, ratchet, delivery and + propagation capabilities. + +Contact slot rules: + +- MT key: node ID; +- MC key: public key plus node-id projection hint; +- RT key: destination hash; +- payload: alias and user flags. + +Peer slots intentionally exclude alias and user flags. + +### `SdProtocolPeerRepository` + +Owns canonical PSRAM vectors: + +- `peers_`; +- `contacts_`; +- `pending_peer_deltas_`. + +It also owns a temporary active-protocol `NodeEntry` projection required by the +current `INodeStore` API. That vector is a read projection and is never a fact +source. + +`NodeStoreView` and `ContactStoreView` forward application operations into the +repository. They have no files and no independent caches. + +## Initialization + +```text +create SdStore + -> acquire startup storage lease + -> recover snapshot backups + -> load protocol projections + -> reconcile catalog from message segments + -> rebuild RT seen ledger if required + -> compact threshold-exceeding projections + +create SdProtocolPeerRepository(SdStore) + -> acquire startup storage lease + -> create /data/v2/{mt,mc,rt} + -> recover peer/contact snapshot backups + -> load peer snapshot + delta + -> load contact snapshot + delta + -> reconcile temporary/stable identities + -> overlay contact facts + -> compact threshold-exceeding journals + +create ContactService(repository.nodeStoreView, + repository.contactStoreView) +``` + +The repository outlives `ContactService`; AppContext stores node/contact as +non-owning pointers and owns the repository in the chat service bundle. + +## Message Transactions + +### Inbound MT/MC + +1. Append the protocol message slot and flush it. +2. Return durable success to `MessageLedger`. +3. Update the in-memory catalog. +4. Attempt catalog delta append. +5. If step 4 fails, mark the catalog dirty but keep durable success. + +### Inbound RT + +1. Detect an existing LXMF hash from hot seen state or the seen journal. +2. Append the RT message slot. +3. Append the LXMF hash to `seen.delta`. +4. Only after both writes succeed, return durable success. +5. Catalog failure remains a rebuildable projection failure. + +If step 2 succeeds and step 3 fails, retry sees the same stored message and +completes the missing seen append without duplicating the message. + +## Peer Merge Transaction + +For an exact stable identity: + +1. Merge non-empty names and fresh observations. +2. Preserve first-seen minimum and last-seen maximum. +3. Preserve contact alias and flags from the contact projection. +4. Preserve a verified key when a conflicting unverified key arrives. +5. Append a complete merged peer delta or queue it in PSRAM. + +For temporary-to-stable identity upgrade: + +1. Find temporary peer by projected node ID. +2. Merge temporary facts into the stable incoming record. +3. Append/queue the stable record. +4. Append/queue the temporary identity tombstone. +5. Replace the in-memory identity. + +Stable-first ordering prefers a recoverable duplicate over loss. Startup +reconciliation merges a duplicate temporary/stable pair if power fails before +the tombstone is durable. + +## Contact Transaction + +1. Resolve the node ID in the active protocol partition. +2. Require MT node identity, MC public key, or RT destination identity. +3. Build a complete contact projection with alias and flags. +4. Acquire a bounded durable storage lease. +5. Append and flush `contacts.delta`. +6. Apply the projection to memory. +7. Refresh only the affected peer/node projection. + +Failure before step 5 leaves memory unchanged and returns failure to the user +intent. A contact edit is never queued invisibly. + +## Retention Algorithm + +Insertion first resolves exact or upgrade identity. Capacity handling occurs +only for a genuinely new peer. + +At the 2048 nearby threshold, the repository loads the protocol conversation +catalog once, then scans peers once. It does not query storage separately for +each peer. + +A candidate is eligible only when: + +- it belongs to the target protocol; +- it has no contact/user-fact projection; +- no conversation references its stable identity or projected node ID. + +The eligible peer with the smallest `last_seen_s` receives a tombstone. If no +eligible peer exists, insertion returns `CapacityExceeded`. + +## Pending Peer Delta Queue + +Peer observations must not block packet or UI work on shared SPI contention. + +The queue uses a PSRAM vector plus a head index; draining does not repeatedly +erase the first vector element. Consecutive pending full-record updates for the +same identity and deletion state are coalesced to the newest projection. + +Each service tick drains at most four entries. Call realtime preemption prevents +the AppContext flush call, so audio resource ownership remains intact. + +## Snapshot Compaction + +Snapshot compaction uses complete current projections and runs at startup when +delta thresholds are exceeded: + +- catalog: 512 deltas; +- read state: 256 deltas; +- status: 2048 deltas; +- peer/contact: 1024 deltas. + +Normal chat `flush()` does not inspect and compact all healthy journals. It +only retries a dirty protocol projection, at most once per five seconds, using +a non-blocking background SPI lease. + +Snapshot replacement sequence: + +1. remove stale temp; +2. create and completely flush temp journal; +3. remove stale backup; +4. rename final to backup; +5. rename temp to final; +6. remove backup; +7. truncate/remove the now-covered delta. + +Startup recovery restores backup when final is missing. If final exists, +stale temp and backup are removed. + +## Corruption Recovery + +Message segments are authoritative and are never discarded because a catalog +slot is corrupt. + +Catalog reconciliation enumerates conversation segment directories and +reconstructs count, latest preview, timestamp, and unread state. + +RT seen recovery streams all RT message slots and writes every valid LXMF hash +into a new seen snapshot. It does not allocate a complete message corpus in +memory. + +Peer/contact incompatible v2 headers fail repository startup for that v2 +partition. This is an explicit storage error, not permission to read legacy +files. Partial tails retain complete slots; contact/peer snapshots are recovered +through backup files. + +## Concurrency And SPI + +The peer repository serializes its aggregate with a FreeRTOS mutex. It holds one +repository lock while applying an aggregate transaction. + +All peer/contact journal I/O uses the shared storage bus arbiter. Peer appends +request a zero-wait background lease; contact edits use a bounded durable lease. + +Chat authoritative append uses a zero-wait durable lease and delegates retry to +`MessageLedger`. No renderer waits for radio TX, MQTT forwarding, LoRa airtime, +or projection compaction. + +## Test Matrix + +| Case | Expected result | +|---|---| +| MQTT inbound, catalog append fails | Message publishes; catalog dirty | +| RT message slot exists, seen append failed | Retry completes seen; no duplicate message | +| Seen journal corrupt | Rebuilt from RT message journals | +| Peer append SPI busy | Peer visible in memory; ordered delta queued | +| Contact append SPI busy | Contact edit fails; memory unchanged | +| MC node observed before key | Temporary peer persists; cannot become contact | +| MC key later arrives | Stable record then temporary tombstone | +| RT protocol event before destination identity | Temporary record upgrades to destination | +| Conflicting unverified key | Verified key retained | +| Nearby capacity full | Oldest unprotected nearby evicted | +| All peers protected | New nearby rejected; contacts retained | +| Power loss during snapshot rename | Backup restored at startup | + diff --git a/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md b/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md new file mode 100644 index 00000000..4c3c4a3b --- /dev/null +++ b/docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md @@ -0,0 +1,156 @@ +# Protocol-Partitioned Storage V2 Overview Design + +## Context + +The previous ESP runtime maintained overlapping facts in chat files, +`/nodes.bin`, `/contacts.dat`, and `/mesh/peers.bin`. A single remote node could +be represented by multiple records with different update semantics. Whole-file +projection writes also allowed an index failure to prevent a durable inbound +message from reaching the UI. + +Storage v2 replaces that shape with two owners: + +```text +ChatService / MessageLedger + -> SdStore + -> MT chat partition + -> MC chat partition + -> RT chat partition + +Protocol adapters + ContactService + -> IProtocolPeerRepository + -> SdProtocolPeerRepository + -> MT peer/contact partition + -> MC peer/contact partition + -> RT peer/contact partition +``` + +## Architectural Pattern + +The design combines four patterns. + +**Repository:** `SdProtocolPeerRepository` owns the aggregate of stable peer +identity, protocol facts, observations, and user contact facts. + +**Ports and views:** Protocol adapters use `IMeshPeerDirectory`; application +contact logic uses `INodeStore` and `IContactStore`. Node/contact views do not +own storage and cannot diverge from the repository. + +**Append-only event journal:** Messages, delivery status, read watermarks, +peer changes, and contact changes append immutable slots. Snapshots are compact +projections, not authoritative mutable blobs. + +**CQRS projection:** UI queries read catalog/contact/node projections. Incoming +message and peer writes do not wait for expensive UI-oriented reconstruction. + +## High-Level Commit Flows + +### Incoming Message + +```text +protocol RX + -> decode ChatMessage + -> MessageLedger + -> SdStore.appendIncomingDurably + -> message slot + -> RT seen identity when required + -> update in-memory catalog projection + -> attempt catalog delta; mark dirty on contention/failure + -> observer publication + -> model + -> event bus + -> notification + -> background catalog projection retry/rebuild when dirty +``` + +### Peer Observation + +```text +protocol announce/node info/key observation + -> IMeshPeerDirectory.record + -> merge by protocol stable identity + -> preserve verified key and contact facts + -> non-blocking peer delta append + -> success + -> or ordered PSRAM pending queue +``` + +### Contact Edit + +```text +user contact intent + -> ContactService + -> IContactStore view + -> resolve active-protocol stable identity + -> durable contact delta + -> update repository aggregate + -> invalidate contact projection +``` + +## Why Protocol Partitions + +Physical partitioning removes fields that are meaningless to a protocol: + +- MT does not carry RT destination/identity/LXMF hashes; +- MC does not carry MT channel-key state; +- RT does not carry Meshtastic role/hardware/MAC fields. + +The business layer remains shared. This preserves one delivery state machine, +one read-state meaning, one retry policy, and one UI presentation model. + +## Capacity And Cost + +Approximate fixed slot sizes are: + +| Data | MT | MC | RT | +|---|---:|---:|---:| +| Message | 279 B | 217 B | 365 B | +| Peer | 217 B | 225 B | 264 B | +| Contact | 29 B | 61 B | 41 B | + +The previous peer union was roughly 488 B and coexisted with a separate node +record of roughly 180 B. A normal MT peer therefore drops from two overlapping +records totaling about 668 B to one protocol peer slot of about 217 B, plus a +29 B contact slot only when the user owns contact facts. + +At fixed-slot payload level, one MiB stores approximately: + +- 3758 MT messages; +- 4832 MC messages; +- 2872 RT messages; +- 4832 MT peers, 4660 MC peers, or 3971 RT peers. + +Filesystem, journal headers, segments, status deltas, and catalog projections +reduce those theoretical counts. There is no 200-node physical store limit and +no fixed message-count cap; practical capacity is governed by SD space and the +explicit runtime retention policy. + +The runtime retains up to 2048 unprotected nearby peers per protocol. Contacts +and conversation-referenced peers are outside that eviction budget. Up to 4096 +contact/user-fact projections are accepted in total; existing contacts are +never evicted to admit strangers. + +## Evolution Space + +The partition boundary allows protocol evolution without widening every +record: + +- MT can add a new slot schema carrying MT-only telemetry or key state; +- MC can change route/peer-hash facts without touching MT or RT files; +- RT can add propagation-node, ratchet, or resource metadata only to RT; +- a new protocol can add a codec and partition while reusing MessageLedger, + ChatService, delivery projection, and UI contracts; +- catalogs can gain secondary indexes without changing message durability; +- old message segments can be archived or pruned by segment, not per-message + file churn; +- background compaction can move to a dedicated storage worker without + changing repository semantics; +- active-protocol paging can replace the current transient node vector without + changing persisted facts. + +## Product Result + +Protocol switching now changes the complete Chat/Contacts/Nearby projection, +not just a UI filter. A peer has one canonical repository record, contact facts +cannot be overwritten by node refreshes, and projection I/O can no longer hide +an already durable MQTT/LoRa/Reticulum message. diff --git a/docs/specification/CHAT_DELIVERY_RUNTIME_SPEC.md b/docs/specification/CHAT_DELIVERY_RUNTIME_SPEC.md index 0f6c8e3e..a58cfebd 100644 --- a/docs/specification/CHAT_DELIVERY_RUNTIME_SPEC.md +++ b/docs/specification/CHAT_DELIVERY_RUNTIME_SPEC.md @@ -89,6 +89,33 @@ platform feedback through `IChatDeliveryFeedbackPort`. `ChatPresentationSource` reads the delivery read model and projects state into `MessageRow`. +`ChatMessageLedger` also owns transient persistence pressure. A persistent +store reports authoritative append success through `appendDurably(...)`. +Transient failure enters a bounded pending-write queue owned by the ledger; +the presentation source reads a merged ledger page, never a page-local fallback. + +Incoming publication is a two-phase boundary: + +```text +decode -> durable append/defer -> delivery commit -> model/event/notification +``` + +`appendIncomingDurably(...)` failure must not drop the message and must not +publish it early. The runtime retries with a bounded per-tick budget. Once the +append succeeds, observers are invoked exactly once. Queue exhaustion is an +explicit rejection, not a silent loss. + +On ESP, the physical commit and recovery rules are frozen by +`PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md`. A protocol message slot is +authoritative; catalog/read/status are projections. Once an inbound message and +its required RT LXMF dedup identity are durable, catalog failure must not block +observer, UI, or notification publication. + +On ESP shared-SPI targets, chat append and status transactions use a +non-blocking outer SPI lease. Failure to obtain the lease produces `Deferred`; +the hot path must not multiply the storage runtime's per-operation wait or run +a synchronous full index rebuild. + ## Boundaries `ChatDeliveryReadModel` may: @@ -200,9 +227,15 @@ Examples: Phase 7.1 does not implement a full retry engine. +The bounded persistence retry owned by `ChatMessageLedger` is not a protocol +retransmission engine. It only completes local authoritative message/status +commit after transient storage contention. + Phase 7.1 does not change radio packet format. -Phase 7.1 does not move all ChatService storage. +Phase 7.1 does not move delivery business ownership into the physical store. +`SdStore` may own protocol-partitioned journals, but delivery meaning remains in +MessageLedger and the delivery projector. Phase 7.1 does not make `ChatWorkspaceModel` own delivery state. diff --git a/docs/specification/PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md b/docs/specification/PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md new file mode 100644 index 00000000..90f76c64 --- /dev/null +++ b/docs/specification/PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md @@ -0,0 +1,266 @@ +# Protocol-Partitioned Storage V2 Specification + +## Purpose + +This specification freezes the authoritative storage model for MT, MC, and RT +chat, peer, contact, delivery, and read-state data on the ESP Arduino product. + +The design has four goals: + +- a received message becomes visible only after its authoritative message + record is durable; +- projection failures never suppress an already durable message; +- protocol-specific facts are not stored in a cross-protocol union; +- contacts and conversation-referenced peers cannot be displaced by nearby + discovery traffic. + +## Product Boundary + +The active ESP Arduino runtime must use only the v2 layout: + +```text +/data/v2/mt/ +/data/v2/mc/ +/data/v2/rt/ +``` + +It must not read or register compatibility readers for: + +```text +/chat/* +/nodes.bin +/contacts.dat +/mesh/peers.bin +``` + +Failure to load a v2 journal may trigger v2 recovery from an authoritative v2 +journal. It must not fall back to a v1 file. + +## Ownership + +`SdStore` owns authoritative chat message journals and their projections. + +`SdProtocolPeerRepository` is the single owner of protocol peer facts and user +contact facts. It exposes three ports: + +- `IMeshPeerDirectory` for protocol runtimes; +- `INodeStore` as an active-protocol contact projection; +- `IContactStore` as an active-protocol user-fact projection. + +The node and contact ports are views of the same repository. They must never be +backed by independent ESP stores. + +`ChatService`, `MessageLedger`, `ChatDeliveryEventProjector`, and UI +presentation remain protocol-independent business owners. Physical protocol +partitioning must not duplicate these services or their state machines. + +## Authoritative Commit Rule + +For an inbound message: + +```text +decode + -> append protocol message slot + -> append RT LXMF dedup identity when applicable + -> commit MessageLedger delivery + -> publish model/event/notification + -> append/rebuild catalog projection +``` + +The message slot and required RT dedup identity are authoritative. Catalog, +preview, unread count, and status indexes are projections. + +If the message append fails, publication is deferred by `MessageLedger`. + +If the message append succeeds but a catalog projection append fails, the +message is still committed and published. The catalog is marked dirty and is +rebuilt from message journals. + +This rule specifically prevents the former MQTT failure mode where a downlink +reached `ChatService`, storage projection I/O failed, and the UI/notification +observer never saw the message. + +## Chat Physical Layout + +Each protocol owns independent chat files: + +```text +/data/v2//chat/ + catalog.snapshot + catalog.delta + read.snapshot + read.delta + status.snapshot + status.delta + conversations//0000.msg + conversations//0001.msg +``` + +RT additionally owns: + +```text +/data/v2/rt/chat/seen.snapshot +/data/v2/rt/chat/seen.delta +``` + +Message segments are 128 KiB fixed-slot journals. Slots are protocol-specific: + +- MT text capacity is 233 bytes and the MT slot is fixed-size; +- MC text capacity follows the MC wire/product limit; +- RT text and identity fields follow the LXMF product limit; +- only RT slots carry RT destination, identity, and LXMF hash fields. + +Status changes are appended as status projection records. Existing message +slots are never rewritten to update delivery state. + +Read state is a `last_read_sequence` projection. Receiving a message does not +rewrite the complete read-state file. + +## Peer And Contact Physical Layout + +Each protocol owns: + +```text +/data/v2// + peers.snapshot + peers.delta + contacts.snapshot + contacts.delta +``` + +Peer slots contain only protocol identity, protocol facts, timestamps, source, +display facts, metrics, and position observations. + +Contact slots contain only stable identity plus user-owned facts: + +- alias; +- favorite; +- ignored; +- trusted; +- MC node-id projection hint where required by the current UI identity. + +Alias and user flags must not be duplicated in peer slots. + +## Identity Rules + +MT stable identity is node ID. Its public key is a protocol fact. + +MC stable identity is the 32-byte public key. A node-ID-only observation may be +persisted temporarily, but it cannot become a contact until a stable public key +is known. When the key arrives, the repository appends the stable record before +tombstoning the temporary identity. + +RT stable identity is the destination hash. A node-ID-only observation may be +persisted temporarily. A later destination identity upgrades it using the same +stable-first ordering. + +A manually verified MT key or verified MC key must not be replaced by a +conflicting unverified runtime observation. Other fresh node facts may still be +merged. + +## Retention Rules + +Nearby/discovered records are preemptible storage entries. + +Per protocol, at most 2048 unprotected nearby records are retained. When that +budget is full, only the oldest unprotected nearby record may be tombstoned. + +A peer is protected when either condition is true: + +- it has a contact/user-fact projection; +- an existing chat conversation references it. + +Protected peers do not consume the nearby eviction budget and must never be +selected as nearby eviction victims. + +The repository accepts up to 4096 protected contact/user-fact projections in +total. Reaching that explicit limit rejects a new contact; it never evicts an +existing contact. This limit is independent of the 2048 nearby budget. + +## Active Protocol Query Rule + +Switching protocol changes the application query partition: + +- Chat returns only conversations for the active protocol; +- Contacts returns only contacts for the active protocol; +- Nearby returns only peers for the active protocol. + +The UI must not load all protocols and filter them locally. + +## Memory Rule + +Canonical chat projections, peer records, contact records, pending peer deltas, +and codec scratch ownership use the single platform `PsramAllocator`. + +On ESP that allocator requests `MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT` and has no +internal-heap fallback. Allocation failure is explicit and fatal rather than a +silent migration of large protocol data to internal RAM. + +Small RTOS objects and the existing `INodeStore` active UI projection may use +internal memory. They must not become an additional fact source. + +## Scheduling Rule + +Message append uses a non-blocking durable SPI lease. Contention defers the +message through the ledger rather than blocking the UI. + +Peer observation deltas use a non-blocking append. Failed appends enter a +PSRAM-backed ordered queue. The runtime drains at most four peer deltas per +service tick and does not drain during call realtime resource preemption. + +Contact edits are explicit user transactions and use a bounded durable lease. +Memory is updated only after the contact delta is durable. + +Large snapshot compaction runs during startup. Normal runtime flush only +retries a dirty chat projection at a throttled interval; it does not compact +healthy read/status journals on every UI loop. + +## Recovery Rule + +Every journal has a versioned descriptor and every slot has a CRC. + +A partial tail exposes only complete slots. Authoritative message journals are +reconciled into catalog/read projections. + +Snapshot replacement uses: + +```text +temp complete + -> final renamed to backup + -> temp renamed to final + -> backup removed +``` + +Startup restores the backup if power was lost between the two renames. + +The RT seen ledger is a projection of authoritative RT messages. If it is +missing, incompatible, partially written, or contains a corrupt slot, it is +rebuilt by streaming RT message journals. This prevents propagation from +re-presenting already stored LXMF messages after index damage. + +## Forbidden Changes + +- Do not add a v1 reader or v2-to-v1 fallback. +- Do not reintroduce an ESP `NodeStore` or `ContactStore` beside the protocol + repository. +- Do not make catalog/index success a prerequisite for publishing a durable + message. +- Do not write contact aliases or flags into peer slots. +- Do not evict contacts to admit nearby observations. +- Do not rewrite complete index/read/contact files on every message. +- Do not compact large snapshots in a renderer, notification callback, packet + callback, or ordinary UI tick. +- Do not use a second PSRAM STL allocator implementation. + +## Verification + +Required checks include: + +- MT/MC/RT max-size message slot round trips and CRC rejection; +- temporary and stable MC/RT peer identity round trips; +- contact identity and MC node-hint round trips; +- peer slots do not round-trip contact flags or aliases; +- verified keys survive conflicting unverified updates; +- ESP source contract contains no registered v1 paths or old store classes; +- MQTT durable-before-projection ordering remains enforced; +- Pager PlatformIO build and ESP stack hygiene pass. diff --git a/docs/specification/RUNTIME_OWNERSHIP_BOUNDARY_FREEZE.md b/docs/specification/RUNTIME_OWNERSHIP_BOUNDARY_FREEZE.md index 013a4749..fd2b6291 100644 --- a/docs/specification/RUNTIME_OWNERSHIP_BOUNDARY_FREEZE.md +++ b/docs/specification/RUNTIME_OWNERSHIP_BOUNDARY_FREEZE.md @@ -272,6 +272,50 @@ Settings 可以配置： 3. 在 retry、delivery action、presentation lookup 中丢掉 protocol 字段。 4. 因为找不到消息就创建另一个同内容 outgoing item。 +### Message Persistence And Publication + +消息内容、发送状态和 UI/通知事件必须走同一条 ledger 主路径： + +```text +decoded incoming + -> MessageLedger durable append or bounded deferred queue + -> incoming delivery commit + -> ChatModel / EventBus / notification + -> conversation projection + +outgoing protocol acceptance + -> MessageLedger durable append or bounded deferred queue + -> protocol-aware delivery event + -> conversation projection +``` + +硬约束： + +1. incoming 只有在 authoritative message record、dedup identity 和 read-state commit + 成功后才能发布 `ChatNewMessageEvent`、通知、震动或声音。 +2. SD/SPI 暂时不可用时，incoming 进入固定深度 deferred queue；重试成功后只提交和发布 + 一次。队列满必须输出明确 rejected/drop reason，并向支持 two-phase commit 的协议返回失败。 +3. outgoing record 或后续 status 写入失败时，由 `MessageLedger` 保留 bounded pending write； + message page、conversation page 和 lookup 必须合并该 pending state，不能谎报 `stored`， + 也不能让 UI 另造临时消息。 +4. pending write 每个 runtime tick 只允许执行有限预算。ESP chat store 必须先以 non-blocking + 方式取得共享 SPI transaction lease；拿不到立即 deferred，不能在同一 tick 连续等待多个 + 250ms SD 操作。 +5. conversation index、header mirror 和 UI cache 仍然只是 projection。projection 写失败可以让 + ledger operation 保持 pending，但不得触发收发热路径中的同步全盘 `rebuildIndex()`。 +6. message/status retry 必须幂等。已经写入 conversation log、但后续 ledger/projection 写失败的 + 消息，重试时只能完成未完成的提交，不能追加第二条相同记录。 +7. chat workspace 最新页固定为 10 条；翻页继续使用同一个 ledger page API。runtime event 对 + 当前会话最多触发一次 snapshot reload，不允许辅助函数和调用者各做一次全量重建。 + +禁止： + +1. `appendIncomingDurably(...) == false` 后直接 `continue` 并丢弃消息。 +2. 调用返回 `void append(...)` 后无条件记录 `stored`。 +3. UI 监听 raw MQTT/LoRa packet，或在持久化失败时自行合成 message bubble。 +4. 为了修复通知而绕过 durable commit 直接调用 notification/audio。 +5. index 写失败后在消息收发 tick 内同步扫描所有 conversation log。 + ## Read And Unread Contract `ReadStateLedger` 是 read/unread 的唯一权威。 @@ -494,6 +538,41 @@ Contacts 和 Network 都只能消费 projection，不允许读取 raw announce 4. old mutation 已删除，而不是留作 fallback。 5. compatibility code 与 product graph 隔离。 +## Protocol-Partitioned Storage V2 Contract + +ESP Arduino 产品运行时的 Chat/Peer/Contact 持久化只允许使用： + +```text +/data/v2/mt +/data/v2/mc +/data/v2/rt +``` + +权威规则： + +1. message journal 是消息事实；catalog/read/status 是可重建 projection。 +2. RT message journal 是 LXMF seen ledger 的重建来源。 +3. `SdProtocolPeerRepository` 是 peer facts 和 contact user facts 的唯一 owner。 +4. `INodeStore`、`IContactStore` 是 repository view，不是独立 store。 +5. contact alias/favorite/ignored/trusted 只写 contact journal，不重复写 peer slot。 +6. active protocol 必须在 application query 边界过滤，UI 不读取三协议全量后再过滤。 +7. nearby 只能淘汰 unprotected peer；contact 和 conversation reference 永远受保护。 +8. snapshot 只能通过 temp/backup/final 原子替换；掉电后只恢复 v2 backup。 +9. 大 projection/peer/contact/pending buffer 优先使用唯一的 strict PSRAM allocator。 +10. 启动阶段可以做有界 compaction；普通 UI tick 不得反复扫描或重写完整 projection。 + +ESP product graph 禁止重新注册： + +```text +/chat/* +/nodes.bin +/contacts.dat +/mesh/peers.bin +``` + +禁止以“兼容”为名在 v2 失败后读取旧格式。完整规则见 +`PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md`。 + ## Prohibited Patch Patterns 以下修改方式禁止进入主线： @@ -546,6 +625,10 @@ Contacts 和 Network 都只能消费 projection，不允许读取 raw announce 14. message alerts/contact alerts/vibration/audio volume 策略不改变 delivery 或 unread 事实。 15. Contacts 不出现 propagation、service、gateway/interface、unknown announce。 16. Network 能呈现服务和网络基础设施，不污染 Contacts。 +17. MQTT/LoRa/RT durable message 不因 catalog projection 失败而不展示或不通知。 +18. peer refresh 不会覆盖 contact alias/flags 或已验证 key。 +19. nearby 达到容量时不会淘汰 contact 或已有 conversation peer。 +20. RT seen journal 损坏后从 authoritative RT message journal 重建，历史消息不重复投影。 ## Relationship To Existing Specs @@ -559,5 +642,8 @@ Contacts 和 Network 都只能消费 projection，不允许读取 raw announce 6. `docs/reticulum_client_architecture.md` 7. `docs/wifi_access_resource_policy.md` 8. `docs/MULTI_PROTOCOL_SUPPORT.md` +9. `docs/specification/PROTOCOL_PARTITIONED_STORAGE_V2_SPEC.md` +10. `docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_OVERVIEW.md` +11. `docs/design/PROTOCOL_PARTITIONED_STORAGE_V2_DETAILED_DESIGN.md` 当实现与本文档冲突时，不能通过局部代码补丁解决；必须回到 owner 边界，修正主路径。 diff --git a/modules/core_chat/include/chat/delivery/chat_message_ledger.h b/modules/core_chat/include/chat/delivery/chat_message_ledger.h index bd7b0b43..f3d7dcbf 100644 --- a/modules/core_chat/include/chat/delivery/chat_message_ledger.h +++ b/modules/core_chat/include/chat/delivery/chat_message_ledger.h @@ -4,9 +4,21 @@ #include "chat/domain/chat_model.h" #include "chat/ports/i_chat_store.h" +#include +#include +#include +#include + namespace chat::delivery { +enum class LedgerPersistence : uint8_t +{ + Durable, + Deferred, + Rejected, +}; + class ChatMessageLedger final { public: @@ -14,9 +26,11 @@ class ChatMessageLedger final void setDeliveryEventPort(IChatDeliveryEventPort* delivery_event_port); - void recordOutbound(const ChatMessage& message, - bool model_enabled, - SendFailureKind failure = SendFailureKind::Unknown); + LedgerPersistence recordOutbound( + const ChatMessage& message, + bool model_enabled, + SendFailureKind failure = SendFailureKind::Unknown); + LedgerPersistence recordIncoming(const ChatMessage& message); bool applyOutboundStatus(MessageId msg_id, MessageStatus status, bool model_enabled, @@ -34,19 +48,70 @@ class ChatMessageLedger final bool markRetryQueuedForProtocol(MessageId msg_id, MeshProtocol protocol, bool model_enabled); + std::size_t flushPendingWrites(std::size_t budget = 1); + std::vector loadPageFromLatest( + const ConversationId& conversation, + std::size_t offset_from_latest, + std::size_t limit, + std::size_t* total) const; + std::vector loadRecent(const ConversationId& conversation, + std::size_t limit) const; + std::vector loadConversationPage(MeshProtocol protocol, + std::size_t offset, + std::size_t limit, + std::size_t* total) const; + bool findMessage(MessageId msg_id, ChatMessage& out) const; + bool findMessageForProtocol(MessageId msg_id, + MeshProtocol protocol, + ChatMessage& out) const; + void clearConversation(const ConversationId& conversation); + void clear(); private: + static constexpr std::size_t kPendingOutboundWriteDepth = 8; + static constexpr std::size_t kPendingStatusWriteDepth = 16; + + struct PendingOutboundWrite + { + bool used = false; + uint32_t sequence = 0; + ChatMessage message{}; + }; + + struct PendingStatusWrite + { + bool used = false; + uint32_t sequence = 0; + MessageId msg_id = 0; + MeshProtocol protocol = MeshProtocol::Meshtastic; + MessageStatus status = MessageStatus::Queued; + }; + bool lookupMessage(MessageId msg_id, ChatMessage& out) const; bool lookupMessageForProtocol(MessageId msg_id, MeshProtocol protocol, ChatMessage& out) const; - bool writeStatus(MessageId msg_id, - MessageStatus status, - bool model_enabled); bool writeStatusForProtocol(MessageId msg_id, MeshProtocol protocol, MessageStatus status, bool model_enabled); + PendingOutboundWrite* findPendingOutbound(MessageId msg_id); + PendingOutboundWrite* findPendingOutboundForProtocol( + MessageId msg_id, + MeshProtocol protocol); + const PendingOutboundWrite* findPendingOutbound(MessageId msg_id) const; + const PendingOutboundWrite* findPendingOutboundForProtocol( + MessageId msg_id, + MeshProtocol protocol) const; + bool enqueuePendingOutbound(const ChatMessage& message); + bool enqueuePendingStatus(MessageId msg_id, + MeshProtocol protocol, + MessageStatus status); + void removePendingStatus(MessageId msg_id, MeshProtocol protocol); + void applyPendingStatus(ChatMessage& message) const; + int oldestPendingOutboundIndex() const; + int oldestPendingStatusIndex() const; + uint32_t nextPendingSequence(); void publishDeliveryEvent(const ChatMessage& message, MessageStatus status, uint32_t timestamp_ms = 0, @@ -56,6 +121,11 @@ class ChatMessageLedger final ChatModel& model_; IChatStore& store_; IChatDeliveryEventPort* delivery_event_port_ = nullptr; + std::array + pending_outbound_writes_{}; + std::array + pending_status_writes_{}; + uint32_t next_pending_sequence_ = 1; }; } // namespace chat::delivery diff --git a/modules/core_chat/include/chat/domain/mesh_peer_directory.h b/modules/core_chat/include/chat/domain/mesh_peer_directory.h index cff18f05..f2e3a88d 100644 --- a/modules/core_chat/include/chat/domain/mesh_peer_directory.h +++ b/modules/core_chat/include/chat/domain/mesh_peer_directory.h @@ -1,6 +1,7 @@ #pragma once #include "chat/domain/chat_types.h" +#include "chat/domain/contact_types.h" #include "chat/domain/reticulum_identity.h" #include @@ -18,6 +19,7 @@ constexpr std::size_t kMeshPeerMeshCorePublicKeyLen = 32; constexpr std::size_t kMeshPeerReticulumPublicKeyLen = 32; constexpr std::size_t kMeshPeerReticulumRatchetLen = 32; constexpr std::size_t kMeshPeerMacAddrLen = 6; +constexpr std::size_t kMeshPeerUserAliasMaxLen = 12; enum class MeshPeerSource : uint8_t { @@ -69,6 +71,8 @@ struct MeshPeerNodeFacts struct MeshtasticPeerFacts { MeshPeerNodeFacts node{}; + bool has_next_hop = false; + uint8_t next_hop = 0; bool has_public_key = false; uint8_t public_key[kMeshPeerMeshtasticPublicKeyLen] = {}; bool key_manually_verified = false; @@ -84,6 +88,7 @@ struct MeshCorePeerFacts uint8_t peer_hash = 0; bool has_next_hop = false; uint8_t next_hop = 0; + NodeId node_id_hint = 0; }; struct ReticulumPeerFacts @@ -99,6 +104,18 @@ struct ReticulumPeerFacts bool propagation = false; }; +struct MeshPeerObservations +{ + bool has_snr = false; + float snr = 0.0f; + bool has_rssi = false; + float rssi = 0.0f; + bool has_device_metrics = false; + contacts::NodeDeviceMetrics device_metrics{}; + bool has_position = false; + contacts::NodePosition position{}; +}; + struct MeshPeerRecord { bool valid = false; @@ -107,7 +124,9 @@ struct MeshPeerRecord uint32_t first_seen_s = 0; uint32_t last_seen_s = 0; char display_name[kMeshPeerDisplayNameMaxLen] = {}; + char user_alias[kMeshPeerUserAliasMaxLen + 1] = {}; MeshPeerUserFlags flags{}; + MeshPeerObservations observations{}; MeshtasticPeerFacts meshtastic{}; MeshCorePeerFacts meshcore{}; ReticulumPeerFacts reticulum{}; diff --git a/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h b/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h index 28293cbc..c522df49 100644 --- a/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h +++ b/modules/core_chat/include/chat/infra/mesh_peer_directory_core.h @@ -10,6 +10,9 @@ namespace chat { +MeshPeerRecord mergeMeshPeerRecordFacts(const MeshPeerRecord& existing, + const MeshPeerRecord& incoming); + class MeshPeerDirectoryCore final : public IMeshPeerDirectory { public: diff --git a/modules/core_chat/include/chat/ports/i_chat_store.h b/modules/core_chat/include/chat/ports/i_chat_store.h index 98c93aab..b2c0e07e 100644 --- a/modules/core_chat/include/chat/ports/i_chat_store.h +++ b/modules/core_chat/include/chat/ports/i_chat_store.h @@ -6,6 +6,9 @@ #pragma once #include "../domain/chat_types.h" + +#include +#include #include namespace chat @@ -26,6 +29,19 @@ class IChatStore */ virtual void append(const ChatMessage& msg) = 0; + /** + * Append a message and report whether its authoritative record is durable. + * + * In-memory stores cannot fail independently and may inherit this default. + * Persistent stores override it so the message ledger can defer transient + * storage failures without inventing a second UI or protocol path. + */ + virtual bool appendDurably(const ChatMessage& msg) + { + append(msg); + return true; + } + /** * Persist an incoming message and any durable delivery identity it owns. * @@ -35,8 +51,7 @@ class IChatStore */ virtual bool appendIncomingDurably(const ChatMessage& msg) { - append(msg); - return true; + return appendDurably(msg); } /** @@ -97,6 +112,53 @@ class IChatStore size_t limit, size_t* total) = 0; + /** + * Load conversations owned by one product protocol partition. + * + * Stores with physical protocol partitions override this method. The + * default keeps non-persistent stores source-compatible while ensuring the + * application query, rather than the UI, owns protocol filtering. + */ + virtual std::vector loadConversationPageForProtocol( + MeshProtocol protocol, + size_t offset, + size_t limit, + size_t* total) + { + const MeshProtocol normalized = + protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum : protocol; + size_t all_total = 0; + std::vector all = + loadConversationPage(0, 0, &all_total); + std::vector filtered; + filtered.reserve(all.size()); + for (const ConversationMeta& meta : all) + { + const MeshProtocol item_protocol = + meta.id.protocol == MeshProtocol::RNode + ? MeshProtocol::Reticulum + : meta.id.protocol; + if (item_protocol == normalized) + { + filtered.push_back(meta); + } + } + if (total) + { + *total = filtered.size(); + } + if (offset >= filtered.size()) + { + return {}; + } + const size_t end = + limit == 0 ? filtered.size() + : std::min(filtered.size(), offset + limit); + return std::vector( + filtered.begin() + static_cast(offset), + filtered.begin() + static_cast(end)); + } + /** * @brief Set unread count for conversation * @param conv Conversation ID diff --git a/modules/core_chat/include/chat/ports/i_contact_store.h b/modules/core_chat/include/chat/ports/i_contact_store.h index 8c0dedc6..e2eb301a 100644 --- a/modules/core_chat/include/chat/ports/i_contact_store.h +++ b/modules/core_chat/include/chat/ports/i_contact_store.h @@ -5,6 +5,8 @@ #pragma once +#include "chat/domain/chat_types.h" + #include #include #include @@ -23,6 +25,11 @@ class IContactStore public: virtual ~IContactStore() = default; + virtual void setActiveProtocol(MeshProtocol protocol) + { + (void)protocol; + } + /** * @brief Initialize store (load from persistent storage) */ diff --git a/modules/core_chat/include/chat/ports/i_node_store.h b/modules/core_chat/include/chat/ports/i_node_store.h index 0dfdfd01..3424fa77 100644 --- a/modules/core_chat/include/chat/ports/i_node_store.h +++ b/modules/core_chat/include/chat/ports/i_node_store.h @@ -5,6 +5,7 @@ #pragma once +#include "chat/domain/chat_types.h" #include "chat/domain/contact_types.h" #include #include @@ -64,6 +65,11 @@ class INodeStore public: virtual ~INodeStore() = default; + virtual void setActiveProtocol(MeshProtocol protocol) + { + (void)protocol; + } + /** * @brief Initialize store (load from persistent storage) */ diff --git a/modules/core_chat/include/chat/ports/i_protocol_peer_repository.h b/modules/core_chat/include/chat/ports/i_protocol_peer_repository.h new file mode 100644 index 00000000..95bbd19e --- /dev/null +++ b/modules/core_chat/include/chat/ports/i_protocol_peer_repository.h @@ -0,0 +1,25 @@ +#pragma once + +#include "chat/ports/i_contact_store.h" +#include "chat/ports/i_mesh_peer_directory.h" +#include "chat/ports/i_node_store.h" + +namespace chat +{ + +/** + * Canonical owner for protocol peer facts and user contact facts. + * + * The three inherited ports are separate read/write views of one repository; + * they must never be backed by independent stores in the product runtime. + */ +class IProtocolPeerRepository : public IMeshPeerDirectory +{ + public: + ~IProtocolPeerRepository() override = default; + + virtual contacts::INodeStore& nodeStoreView() = 0; + virtual contacts::IContactStore& contactStoreView() = 0; +}; + +} // namespace chat diff --git a/modules/core_chat/include/chat/usecase/chat_service.h b/modules/core_chat/include/chat/usecase/chat_service.h index e41c1f40..73912176 100644 --- a/modules/core_chat/include/chat/usecase/chat_service.h +++ b/modules/core_chat/include/chat/usecase/chat_service.h @@ -270,6 +270,45 @@ class ChatService std::size_t count = 0; }; + static constexpr std::size_t kDeferredIncomingDepth = 8; + static constexpr std::size_t kDeferredIncomingTextMaxLen = 255; + + struct DeferredIncomingQueue + { + struct Slot + { + MeshProtocol protocol = MeshProtocol::Meshtastic; + ChannelId channel = ChannelId::PRIMARY; + NodeId from = 0; + NodeId to = 0; + MessageId msg_id = 0; + uint32_t timestamp = 0; + uint8_t hop_limit = 0xFF; + bool encrypted = false; + ReticulumPeerIdentity reticulum_identity{}; + bool has_reticulum_lxmf_hash = false; + uint8_t reticulum_lxmf_hash[kReticulumLxmfHashSize] = {}; + bool source_unverified = false; + RxMeta rx_meta{}; + IncomingIdentity identity{}; + std::array text{}; + uint16_t text_len = 0; + }; + + void clear(); + [[nodiscard]] bool empty() const { return count == 0; } + [[nodiscard]] bool contains(const IncomingIdentity& identity) const; + bool push(const MeshIncomingText& incoming, + MeshProtocol protocol, + const IncomingIdentity& identity); + bool peek(MeshIncomingText& incoming, MeshProtocol& protocol) const; + void pop(); + + std::array slots{}; + std::size_t head = 0; + std::size_t count = 0; + }; + ChatModel& model_; IMeshAdapter& adapter_; IChatStore& store_; @@ -280,6 +319,7 @@ class ChatService MeshProtocol active_protocol_ = MeshProtocol::Meshtastic; mutable ChatMessage store_lookup_cache_{}; RecentIncomingWindow recent_incoming_{}; + DeferredIncomingQueue deferred_incoming_{}; std::vector incoming_text_observers_; std::vector incoming_message_observers_; @@ -287,7 +327,15 @@ class ChatService std::vector incoming_data_observers_; [[nodiscard]] bool isDuplicateIncoming(const ChatMessage& msg) const; + [[nodiscard]] bool isDeferredIncoming(const ChatMessage& msg) const; + [[nodiscard]] static IncomingIdentity incomingIdentityForMessage( + const ChatMessage& msg); void rememberIncoming(const ChatMessage& msg); + static ChatMessage incomingChatMessage(const MeshIncomingText& incoming, + MeshProtocol protocol); + void commitIncoming(const MeshIncomingText& incoming, + const ChatMessage& message); + void retryDeferredIncoming(); MeshSendResult sendTextResolvedDetailed( ChannelId channel, const std::string& text, diff --git a/modules/core_chat/include/chat/usecase/contact_service.h b/modules/core_chat/include/chat/usecase/contact_service.h index de497a44..821b528f 100644 --- a/modules/core_chat/include/chat/usecase/contact_service.h +++ b/modules/core_chat/include/chat/usecase/contact_service.h @@ -40,6 +40,9 @@ class ContactService */ void begin(); + /** Selects the only protocol partition visible to contact queries. */ + void setActiveProtocol(MeshProtocol protocol); + void applyNodeUpdate(uint32_t node_id, const NodeUpdate& update); /** @@ -159,6 +162,7 @@ class ContactService private: INodeStore& node_store_; IContactStore& contact_store_; + MeshProtocol active_protocol_ = MeshProtocol::Meshtastic; mutable std::vector cached_nodes_; // Cache for getContacts/getNearby mutable uint32_t cache_timestamp_; static constexpr uint32_t kCacheTimeoutMs = 1000; // 1 second cache diff --git a/modules/core_chat/src/delivery/chat_message_ledger.cpp b/modules/core_chat/src/delivery/chat_message_ledger.cpp index 6b4b909a..5839a55d 100644 --- a/modules/core_chat/src/delivery/chat_message_ledger.cpp +++ b/modules/core_chat/src/delivery/chat_message_ledger.cpp @@ -4,6 +4,9 @@ #include "chat/delivery/chat_delivery_send_result_projection.h" #include "chat/delivery/chat_outbox_service.h" +#include +#include + namespace chat::delivery { @@ -18,23 +21,41 @@ void ChatMessageLedger::setDeliveryEventPort( delivery_event_port_ = delivery_event_port; } -void ChatMessageLedger::recordOutbound(const ChatMessage& message, - bool model_enabled, - SendFailureKind failure) +LedgerPersistence ChatMessageLedger::recordOutbound( + const ChatMessage& message, + bool model_enabled, + SendFailureKind failure) { if (model_enabled) { model_.onSendQueued(message); if (message.status == MessageStatus::Failed && message.msg_id != 0) { - model_.onSendResult(message.msg_id, false); + model_.updateMessageStatusForProtocol(message.msg_id, + message.protocol, + MessageStatus::Failed); } } - store_.append(message); + LedgerPersistence persistence = LedgerPersistence::Durable; + if (!store_.appendDurably(message)) + { + persistence = enqueuePendingOutbound(message) + ? LedgerPersistence::Deferred + : LedgerPersistence::Rejected; + } if (ChatOutboxService::isOutboundStatusUpdate(message.status)) { publishDeliveryEvent(message, message.status, 0, failure); } + return persistence; +} + +LedgerPersistence ChatMessageLedger::recordIncoming( + const ChatMessage& message) +{ + return store_.appendIncomingDurably(message) + ? LedgerPersistence::Durable + : LedgerPersistence::Deferred; } bool ChatMessageLedger::applyOutboundStatus(MessageId msg_id, @@ -52,7 +73,10 @@ bool ChatMessageLedger::applyOutboundStatus(MessageId msg_id, { return false; } - if (!writeStatus(msg_id, status, model_enabled)) + if (!writeStatusForProtocol(msg_id, + current.protocol, + status, + model_enabled)) { return false; } @@ -95,7 +119,10 @@ bool ChatMessageLedger::markRetryQueued(MessageId msg_id, bool model_enabled) { return false; } - if (!writeStatus(msg_id, MessageStatus::Queued, model_enabled)) + if (!writeStatusForProtocol(msg_id, + current.protocol, + MessageStatus::Queued, + model_enabled)) { return false; } @@ -133,12 +160,24 @@ bool ChatMessageLedger::lookupMessage(MessageId msg_id, ChatMessage& out) const { return false; } + if (const PendingOutboundWrite* pending = findPendingOutbound(msg_id)) + { + out = pending->message; + applyPendingStatus(out); + return true; + } if (const ChatMessage* message = model_.getMessage(msg_id)) { out = *message; + applyPendingStatus(out); return true; } - return store_.getMessage(msg_id, &out); + if (!store_.getMessage(msg_id, &out)) + { + return false; + } + applyPendingStatus(out); + return true; } bool ChatMessageLedger::lookupMessageForProtocol(MessageId msg_id, @@ -149,29 +188,26 @@ bool ChatMessageLedger::lookupMessageForProtocol(MessageId msg_id, { return false; } + if (const PendingOutboundWrite* pending = + findPendingOutboundForProtocol(msg_id, protocol)) + { + out = pending->message; + applyPendingStatus(out); + return true; + } if (const ChatMessage* message = model_.getMessageForProtocol(msg_id, protocol)) { out = *message; + applyPendingStatus(out); return true; } - return store_.getMessageForProtocol(msg_id, protocol, &out); -} - -bool ChatMessageLedger::writeStatus(MessageId msg_id, - MessageStatus status, - bool model_enabled) -{ - if (msg_id == 0) + if (!store_.getMessageForProtocol(msg_id, protocol, &out)) { return false; } - bool updated = false; - if (model_enabled) - { - updated = model_.updateMessageStatus(msg_id, status); - } - return store_.updateMessageStatus(msg_id, status) || updated; + applyPendingStatus(out); + return true; } bool ChatMessageLedger::writeStatusForProtocol(MessageId msg_id, @@ -189,8 +225,503 @@ bool ChatMessageLedger::writeStatusForProtocol(MessageId msg_id, updated = model_.updateMessageStatusForProtocol(msg_id, protocol, status); } - return store_.updateMessageStatusForProtocol(msg_id, protocol, status) || - updated; + if (PendingOutboundWrite* pending = + findPendingOutboundForProtocol(msg_id, protocol)) + { + pending->message.status = status; + return true; + } + if (store_.updateMessageStatusForProtocol(msg_id, protocol, status)) + { + removePendingStatus(msg_id, protocol); + return true; + } + return enqueuePendingStatus(msg_id, protocol, status) || updated; +} + +std::size_t ChatMessageLedger::flushPendingWrites(std::size_t budget) +{ + std::size_t flushed = 0; + while (flushed < budget) + { + const int outbound_index = oldestPendingOutboundIndex(); + if (outbound_index >= 0) + { + PendingOutboundWrite& pending = + pending_outbound_writes_[static_cast(outbound_index)]; + if (!store_.appendDurably(pending.message)) + { + break; + } + removePendingStatus(pending.message.msg_id, + pending.message.protocol); + pending = PendingOutboundWrite{}; + ++flushed; + continue; + } + + const int status_index = oldestPendingStatusIndex(); + if (status_index < 0) + { + break; + } + PendingStatusWrite& pending = + pending_status_writes_[static_cast(status_index)]; + if (!store_.updateMessageStatusForProtocol(pending.msg_id, + pending.protocol, + pending.status)) + { + break; + } + pending = PendingStatusWrite{}; + ++flushed; + } + return flushed; +} + +std::vector ChatMessageLedger::loadPageFromLatest( + const ConversationId& conversation, + std::size_t offset_from_latest, + std::size_t limit, + std::size_t* total) const +{ + std::array + pending{}; + std::size_t pending_count = 0; + for (const PendingOutboundWrite& entry : pending_outbound_writes_) + { + if (entry.used && + conversationIdForMessage(entry.message) == conversation) + { + pending[pending_count++] = &entry; + } + } + std::sort(pending.begin(), + pending.begin() + static_cast(pending_count), + [](const PendingOutboundWrite* lhs, + const PendingOutboundWrite* rhs) + { + return lhs->sequence < rhs->sequence; + }); + + std::size_t durable_total = 0; + std::vector result; + if (limit == 0) + { + (void)store_.loadPageFromLatest(conversation, 0, 1, &durable_total); + if (total) + { + *total = durable_total + pending_count; + } + return result; + } + + if (offset_from_latest >= pending_count) + { + result = store_.loadPageFromLatest(conversation, + offset_from_latest - pending_count, + limit, + &durable_total); + } + else + { + const std::size_t pending_end = pending_count - offset_from_latest; + const std::size_t pending_start = + pending_end > limit ? pending_end - limit : 0; + const std::size_t pending_take = pending_end - pending_start; + const std::size_t durable_take = limit - pending_take; + if (durable_take > 0) + { + result = store_.loadPageFromLatest(conversation, + 0, + durable_take, + &durable_total); + } + else + { + (void)store_.loadPageFromLatest(conversation, + 0, + 1, + &durable_total); + } + result.reserve(result.size() + pending_take); + for (std::size_t index = pending_start; index < pending_end; ++index) + { + result.push_back(pending[index]->message); + } + } + + for (ChatMessage& message : result) + { + applyPendingStatus(message); + } + if (total) + { + *total = durable_total + pending_count; + } + return result; +} + +std::vector ChatMessageLedger::loadRecent( + const ConversationId& conversation, + std::size_t limit) const +{ + return loadPageFromLatest(conversation, 0, limit, nullptr); +} + +std::vector ChatMessageLedger::loadConversationPage( + MeshProtocol protocol, + std::size_t offset, + std::size_t limit, + std::size_t* total) const +{ + protocol = protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum + : protocol; + std::size_t durable_total = 0; + std::vector conversations = + store_.loadConversationPageForProtocol(protocol, + 0, + 0, + &durable_total); + + std::array + pending{}; + std::size_t pending_count = 0; + for (const PendingOutboundWrite& entry : pending_outbound_writes_) + { + const MeshProtocol entry_protocol = + entry.message.protocol == MeshProtocol::RNode + ? MeshProtocol::Reticulum + : entry.message.protocol; + if (entry.used && entry_protocol == protocol) + { + pending[pending_count++] = &entry; + } + } + std::sort(pending.begin(), + pending.begin() + static_cast(pending_count), + [](const PendingOutboundWrite* lhs, + const PendingOutboundWrite* rhs) + { + return lhs->sequence < rhs->sequence; + }); + + for (std::size_t pending_index = 0; + pending_index < pending_count; + ++pending_index) + { + const ChatMessage& message = pending[pending_index]->message; + const ConversationId conversation = conversationIdForMessage(message); + auto existing = std::find_if( + conversations.begin(), + conversations.end(), + [&](const ConversationMeta& meta) + { + return meta.id == conversation; + }); + if (existing == conversations.end()) + { + ConversationMeta meta{}; + meta.id = conversation; + if (conversation.peer == 0) + { + meta.name = "Broadcast"; + } + else + { + char name[16]{}; + std::snprintf(name, + sizeof(name), + "%04lX", + static_cast(conversation.peer & + 0xFFFFU)); + meta.name = name; + } + conversations.push_back(meta); + existing = conversations.end() - 1; + } + existing->preview = message.text; + existing->last_timestamp = message.timestamp; + existing->reticulum_identity = message.reticulum_identity; + } + + std::stable_sort(conversations.begin(), + conversations.end(), + [](const ConversationMeta& lhs, + const ConversationMeta& rhs) + { + return lhs.last_timestamp > rhs.last_timestamp; + }); + if (total) + { + *total = conversations.size(); + } + if (offset >= conversations.size()) + { + return {}; + } + const std::size_t end = + limit == 0 + ? conversations.size() + : std::min(conversations.size(), offset + limit); + return std::vector( + conversations.begin() + static_cast(offset), + conversations.begin() + static_cast(end)); +} + +bool ChatMessageLedger::findMessage(MessageId msg_id, ChatMessage& out) const +{ + return lookupMessage(msg_id, out); +} + +bool ChatMessageLedger::findMessageForProtocol(MessageId msg_id, + MeshProtocol protocol, + ChatMessage& out) const +{ + return lookupMessageForProtocol(msg_id, protocol, out); +} + +void ChatMessageLedger::clearConversation( + const ConversationId& conversation) +{ + for (PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (!pending.used || + !(conversationIdForMessage(pending.message) == conversation)) + { + continue; + } + removePendingStatus(pending.message.msg_id, + pending.message.protocol); + pending = PendingOutboundWrite{}; + } + + for (PendingStatusWrite& pending : pending_status_writes_) + { + if (!pending.used) + { + continue; + } + ChatMessage message{}; + if (lookupMessageForProtocol(pending.msg_id, + pending.protocol, + message) && + conversationIdForMessage(message) == conversation) + { + pending = PendingStatusWrite{}; + } + } +} + +void ChatMessageLedger::clear() +{ + for (PendingOutboundWrite& pending : pending_outbound_writes_) + { + pending = PendingOutboundWrite{}; + } + for (PendingStatusWrite& pending : pending_status_writes_) + { + pending = PendingStatusWrite{}; + } + next_pending_sequence_ = 1; +} + +ChatMessageLedger::PendingOutboundWrite* +ChatMessageLedger::findPendingOutbound(MessageId msg_id) +{ + if (msg_id == 0) + { + return nullptr; + } + for (PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (pending.used && pending.message.msg_id == msg_id) + { + return &pending; + } + } + return nullptr; +} + +ChatMessageLedger::PendingOutboundWrite* +ChatMessageLedger::findPendingOutboundForProtocol(MessageId msg_id, + MeshProtocol protocol) +{ + if (msg_id == 0) + { + return nullptr; + } + for (PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (pending.used && pending.message.msg_id == msg_id && + pending.message.protocol == protocol) + { + return &pending; + } + } + return nullptr; +} + +const ChatMessageLedger::PendingOutboundWrite* +ChatMessageLedger::findPendingOutbound(MessageId msg_id) const +{ + if (msg_id == 0) + { + return nullptr; + } + for (const PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (pending.used && pending.message.msg_id == msg_id) + { + return &pending; + } + } + return nullptr; +} + +const ChatMessageLedger::PendingOutboundWrite* +ChatMessageLedger::findPendingOutboundForProtocol( + MessageId msg_id, + MeshProtocol protocol) const +{ + if (msg_id == 0) + { + return nullptr; + } + for (const PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (pending.used && pending.message.msg_id == msg_id && + pending.message.protocol == protocol) + { + return &pending; + } + } + return nullptr; +} + +bool ChatMessageLedger::enqueuePendingOutbound(const ChatMessage& message) +{ + if (PendingOutboundWrite* existing = + findPendingOutboundForProtocol(message.msg_id, + message.protocol)) + { + existing->message = message; + return true; + } + for (PendingOutboundWrite& pending : pending_outbound_writes_) + { + if (pending.used) + { + continue; + } + pending.used = true; + pending.sequence = nextPendingSequence(); + pending.message = message; + return true; + } + return false; +} + +bool ChatMessageLedger::enqueuePendingStatus(MessageId msg_id, + MeshProtocol protocol, + MessageStatus status) +{ + for (PendingStatusWrite& pending : pending_status_writes_) + { + if (pending.used && pending.msg_id == msg_id && + pending.protocol == protocol) + { + pending.status = status; + return true; + } + } + for (PendingStatusWrite& pending : pending_status_writes_) + { + if (pending.used) + { + continue; + } + pending.used = true; + pending.sequence = nextPendingSequence(); + pending.msg_id = msg_id; + pending.protocol = protocol; + pending.status = status; + return true; + } + return false; +} + +void ChatMessageLedger::removePendingStatus(MessageId msg_id, + MeshProtocol protocol) +{ + for (PendingStatusWrite& pending : pending_status_writes_) + { + if (pending.used && pending.msg_id == msg_id && + pending.protocol == protocol) + { + pending = PendingStatusWrite{}; + } + } +} + +void ChatMessageLedger::applyPendingStatus(ChatMessage& message) const +{ + for (const PendingStatusWrite& pending : pending_status_writes_) + { + if (pending.used && pending.msg_id == message.msg_id && + pending.protocol == message.protocol) + { + message.status = pending.status; + } + } +} + +int ChatMessageLedger::oldestPendingOutboundIndex() const +{ + int oldest = -1; + for (std::size_t index = 0; index < pending_outbound_writes_.size(); ++index) + { + const PendingOutboundWrite& pending = pending_outbound_writes_[index]; + if (!pending.used || + (oldest >= 0 && + pending.sequence >= + pending_outbound_writes_[static_cast(oldest)] + .sequence)) + { + continue; + } + oldest = static_cast(index); + } + return oldest; +} + +int ChatMessageLedger::oldestPendingStatusIndex() const +{ + int oldest = -1; + for (std::size_t index = 0; index < pending_status_writes_.size(); ++index) + { + const PendingStatusWrite& pending = pending_status_writes_[index]; + if (!pending.used || + (oldest >= 0 && + pending.sequence >= + pending_status_writes_[static_cast(oldest)] + .sequence)) + { + continue; + } + oldest = static_cast(index); + } + return oldest; +} + +uint32_t ChatMessageLedger::nextPendingSequence() +{ + const uint32_t sequence = next_pending_sequence_++; + if (next_pending_sequence_ == 0) + { + next_pending_sequence_ = 1; + } + return sequence; } void ChatMessageLedger::publishDeliveryEvent(const ChatMessage& message, diff --git a/modules/core_chat/src/infra/mesh_peer_directory_core.cpp b/modules/core_chat/src/infra/mesh_peer_directory_core.cpp index a25ab104..ebaacfe1 100644 --- a/modules/core_chat/src/infra/mesh_peer_directory_core.cpp +++ b/modules/core_chat/src/infra/mesh_peer_directory_core.cpp @@ -355,6 +355,11 @@ void mergeMeshtasticFacts(MeshtasticPeerFacts& dst, const MeshtasticPeerFacts& src) { mergeNodeFacts(dst.node, src.node); + if (src.has_next_hop) + { + dst.has_next_hop = true; + dst.next_hop = src.next_hop; + } if (src.has_public_key && meshPeerHasNonZeroBytes(src.public_key, kMeshPeerMeshtasticPublicKeyLen)) @@ -364,6 +369,10 @@ void mergeMeshtasticFacts(MeshtasticPeerFacts& dst, std::memcmp(dst.public_key, src.public_key, kMeshPeerMeshtasticPublicKeyLen) != 0; + if (changed && dst.has_public_key && dst.key_manually_verified) + { + return; + } dst.has_public_key = true; std::memcpy(dst.public_key, src.public_key, @@ -387,6 +396,10 @@ void mergeMeshCoreFacts(MeshCorePeerFacts& dst, std::memcmp(dst.public_key, src.public_key, kMeshPeerMeshCorePublicKeyLen) != 0; + if (changed && dst.has_public_key && dst.public_key_verified) + { + return; + } dst.has_public_key = true; std::memcpy(dst.public_key, src.public_key, @@ -405,6 +418,10 @@ void mergeMeshCoreFacts(MeshCorePeerFacts& dst, dst.has_next_hop = true; dst.next_hop = src.next_hop; } + if (src.node_id_hint != 0) + { + dst.node_id_hint = src.node_id_hint; + } } void mergeReticulumFacts(ReticulumPeerFacts& dst, @@ -459,8 +476,8 @@ MeshPeerSource mergePeerSource(MeshPeerSource existing, return incoming; } -MeshPeerRecord mergePeerRecordFacts(const MeshPeerRecord& existing, - const MeshPeerRecord& incoming) +MeshPeerRecord mergePeerRecordFactsImpl(const MeshPeerRecord& existing, + const MeshPeerRecord& incoming) { MeshPeerRecord next = existing; next.valid = existing.valid || incoming.valid; @@ -483,7 +500,31 @@ MeshPeerRecord mergePeerRecordFacts(const MeshPeerRecord& existing, copyNonEmptyMeshPeerText(next.display_name, sizeof(next.display_name), incoming.display_name); + copyNonEmptyMeshPeerText(next.user_alias, + sizeof(next.user_alias), + incoming.user_alias); next.flags = existing.flags; + if (incoming.observations.has_snr) + { + next.observations.has_snr = true; + next.observations.snr = incoming.observations.snr; + } + if (incoming.observations.has_rssi) + { + next.observations.has_rssi = true; + next.observations.rssi = incoming.observations.rssi; + } + if (incoming.observations.has_device_metrics) + { + next.observations.has_device_metrics = true; + next.observations.device_metrics = + incoming.observations.device_metrics; + } + if (incoming.observations.has_position) + { + next.observations.has_position = true; + next.observations.position = incoming.observations.position; + } mergeMeshtasticFacts(next.meshtastic, incoming.meshtastic); mergeMeshCoreFacts(next.meshcore, incoming.meshcore); mergeReticulumFacts(next.reticulum, incoming.reticulum); @@ -504,6 +545,12 @@ NodeId reticulumNodeIdFromDestinationHash(const uint8_t* destination_hash) } // namespace +MeshPeerRecord mergeMeshPeerRecordFacts(const MeshPeerRecord& existing, + const MeshPeerRecord& incoming) +{ + return mergePeerRecordFactsImpl(existing, incoming); +} + MeshPeerDirectoryCore::MeshPeerDirectoryCore( IMeshPeerDirectoryBlobStore& blob_store) : blob_store_(blob_store) @@ -566,7 +613,7 @@ MeshPeerDirectoryStatus MeshPeerDirectoryCore::record( if (existing_index < records_.size()) { const MeshPeerRecord& existing = records_[existing_index]; - MeshPeerRecord next = mergePeerRecordFacts(existing, record); + MeshPeerRecord next = mergeMeshPeerRecordFacts(existing, record); records_[existing_index] = next; dirty_ = true; maybeSave(); diff --git a/modules/core_chat/src/usecase/chat_service.cpp b/modules/core_chat/src/usecase/chat_service.cpp index d548d1bf..07622c08 100644 --- a/modules/core_chat/src/usecase/chat_service.cpp +++ b/modules/core_chat/src/usecase/chat_service.cpp @@ -38,6 +38,20 @@ uint32_t persistable_incoming_timestamp(const MeshIncomingText& incoming) } return persistable_now_timestamp(); } + +const char* persistence_name(delivery::LedgerPersistence persistence) +{ + switch (persistence) + { + case delivery::LedgerPersistence::Durable: + return "durable"; + case delivery::LedgerPersistence::Deferred: + return "deferred"; + case delivery::LedgerPersistence::Rejected: + return "rejected"; + } + return "unknown"; +} } // namespace #ifndef CHAT_SERVICE_LOG_ENABLE @@ -51,7 +65,7 @@ uint32_t persistable_incoming_timestamp(const MeshIncomingText& incoming) #endif #ifndef CHAT_SERVICE_DIAG_LOG_ENABLE -#define CHAT_SERVICE_DIAG_LOG_ENABLE 1 +#define CHAT_SERVICE_DIAG_LOG_ENABLE 0 #endif #if CHAT_SERVICE_DIAG_LOG_ENABLE @@ -60,6 +74,16 @@ uint32_t persistable_incoming_timestamp(const MeshIncomingText& incoming) #define CHAT_SERVICE_DIAG_LOG(...) #endif +#ifndef CHAT_SERVICE_EVENT_LOG_ENABLE +#define CHAT_SERVICE_EVENT_LOG_ENABLE 1 +#endif + +#if CHAT_SERVICE_EVENT_LOG_ENABLE +#define CHAT_SERVICE_EVENT_LOG(...) std::printf(__VA_ARGS__) +#else +#define CHAT_SERVICE_EVENT_LOG(...) +#endif + const char* failure_name(MeshOperationFailure failure) { switch (failure) @@ -231,6 +255,110 @@ void ChatService::RecentIncomingWindow::remember(const IncomingIdentity& identit } } +void ChatService::DeferredIncomingQueue::clear() +{ + for (Slot& slot : slots) + { + slot = Slot{}; + } + head = 0; + count = 0; +} + +bool ChatService::DeferredIncomingQueue::contains( + const IncomingIdentity& identity) const +{ + for (std::size_t offset = 0; offset < count; ++offset) + { + const std::size_t index = (head + offset) % slots.size(); + if (slots[index].identity == identity) + { + return true; + } + } + return false; +} + +bool ChatService::DeferredIncomingQueue::push( + const MeshIncomingText& incoming, + MeshProtocol protocol, + const IncomingIdentity& identity) +{ + if (count >= slots.size() || + incoming.text.size() > kDeferredIncomingTextMaxLen) + { + return false; + } + + const std::size_t index = (head + count) % slots.size(); + Slot& slot = slots[index]; + slot = Slot{}; + slot.protocol = protocol; + slot.channel = incoming.channel; + slot.from = incoming.from; + slot.to = incoming.to; + slot.msg_id = incoming.msg_id; + slot.timestamp = incoming.timestamp; + slot.hop_limit = incoming.hop_limit; + slot.encrypted = incoming.encrypted; + slot.reticulum_identity = incoming.reticulum_identity; + slot.has_reticulum_lxmf_hash = incoming.has_reticulum_lxmf_hash; + std::memcpy(slot.reticulum_lxmf_hash, + incoming.reticulum_lxmf_hash, + sizeof(slot.reticulum_lxmf_hash)); + slot.source_unverified = incoming.source_unverified; + slot.rx_meta = incoming.rx_meta; + slot.identity = identity; + slot.text_len = static_cast(incoming.text.size()); + if (slot.text_len > 0) + { + std::memcpy(slot.text.data(), incoming.text.data(), slot.text_len); + } + slot.text[slot.text_len] = '\0'; + ++count; + return true; +} + +bool ChatService::DeferredIncomingQueue::peek( + MeshIncomingText& incoming, + MeshProtocol& protocol) const +{ + if (count == 0) + { + return false; + } + + const Slot& slot = slots[head]; + protocol = slot.protocol; + incoming.channel = slot.channel; + incoming.from = slot.from; + incoming.to = slot.to; + incoming.msg_id = slot.msg_id; + incoming.timestamp = slot.timestamp; + incoming.hop_limit = slot.hop_limit; + incoming.encrypted = slot.encrypted; + incoming.reticulum_identity = slot.reticulum_identity; + incoming.has_reticulum_lxmf_hash = slot.has_reticulum_lxmf_hash; + std::memcpy(incoming.reticulum_lxmf_hash, + slot.reticulum_lxmf_hash, + sizeof(incoming.reticulum_lxmf_hash)); + incoming.source_unverified = slot.source_unverified; + incoming.rx_meta = slot.rx_meta; + incoming.text.assign(slot.text.data(), slot.text_len); + return true; +} + +void ChatService::DeferredIncomingQueue::pop() +{ + if (count == 0) + { + return; + } + slots[head] = Slot{}; + head = (head + 1) % slots.size(); + --count; +} + MessageId ChatService::sendText(ChannelId channel, const std::string& text, NodeId peer) { return sendTextWithId(channel, text, 0, peer); @@ -354,17 +482,20 @@ MeshSendResult ChatService::sendTextResolvedDetailed( msg.reticulum_identity = result.reticulum_identity; msg.status = result.ok ? MessageStatus::Queued : MessageStatus::Failed; - message_ledger_.recordOutbound( - msg, - model_enabled_, - result.ok ? delivery::SendFailureKind::None - : delivery_failure_from_mesh_operation(result.failure)); - CHAT_SERVICE_DIAG_LOG("[ChatService][TX] stored msg=%lu status=%u peer=%08lX dest=%s text=\"%s\"\n", - static_cast(msg.msg_id), - static_cast(msg.status), - static_cast(msg.peer), - dest_hash, - text_preview); + const delivery::LedgerPersistence persistence = + message_ledger_.recordOutbound( + msg, + model_enabled_, + result.ok ? delivery::SendFailureKind::None + : delivery_failure_from_mesh_operation(result.failure)); + if (persistence != delivery::LedgerPersistence::Durable) + { + CHAT_SERVICE_EVENT_LOG("[ChatService][TX] ledger msg=%lu persistence=%s status=%u protocol=%s\n", + static_cast(msg.msg_id), + persistence_name(persistence), + static_cast(msg.status), + protocol_name(msg.protocol)); + } if (result.ok && result.msg_id != 0) { @@ -537,7 +668,7 @@ bool ChatService::resendFailedForProtocol(MessageId msg_id, std::vector ChatService::getRecentMessages(const ConversationId& conv, size_t limit) const { - return store_.loadRecent(conv, limit); + return message_ledger_.loadRecent(conv, limit); } std::vector ChatService::getMessagePageFromLatest( @@ -546,20 +677,29 @@ std::vector ChatService::getMessagePageFromLatest( size_t limit, size_t* total) const { - return store_.loadPageFromLatest(conv, offset_from_latest, limit, total); + return message_ledger_.loadPageFromLatest(conv, + offset_from_latest, + limit, + total); } std::vector ChatService::getConversations(size_t offset, size_t limit, size_t* total) const { - return store_.loadConversationPage(offset, limit, total); + return message_ledger_.loadConversationPage(active_protocol_, + offset, + limit, + total); } int ChatService::getTotalUnread() const { size_t total = 0; - auto convs = store_.loadConversationPage(0, 0, &total); + auto convs = store_.loadConversationPageForProtocol(active_protocol_, + 0, + 0, + &total); int sum = 0; for (const auto& conv : convs) { @@ -570,6 +710,8 @@ int ChatService::getTotalUnread() const void ChatService::clearAllMessages() { + message_ledger_.clear(); + deferred_incoming_.clear(); model_.clearAll(); store_.clearAll(); recent_incoming_.clear(); @@ -577,6 +719,8 @@ void ChatService::clearAllMessages() void ChatService::clearConversation(const ConversationId& conv) { + message_ledger_.clearConversation(conv); + deferred_incoming_.clear(); model_.clearConversation(conv); store_.clearConversation(conv); recent_incoming_.clear(); @@ -589,25 +733,19 @@ bool ChatService::markConversationRead(const ConversationId& conv) void ChatService::processIncoming() { + (void)message_ledger_.flushPendingWrites(1); + retryDeferredIncoming(); + + static constexpr std::size_t kIncomingDrainBudget = 4; + std::size_t drained = 0; MeshIncomingText incoming_text; - while (adapter_.pollIncomingText(&incoming_text)) + while (drained < kIncomingDrainBudget && + adapter_.pollIncomingText(&incoming_text)) { - ChatMessage msg; - msg.protocol = active_protocol_; - msg.channel = incoming_text.channel; - msg.from = incoming_text.from; - msg.peer = normalize_conversation_peer(incoming_text.to) == 0 ? 0 : incoming_text.from; - msg.msg_id = incoming_text.msg_id; - msg.timestamp = persistable_incoming_timestamp(incoming_text); - msg.text = incoming_text.text; - msg.reticulum_identity = incoming_text.reticulum_identity; - msg.has_reticulum_lxmf_hash = incoming_text.has_reticulum_lxmf_hash; - std::memcpy(msg.reticulum_lxmf_hash, - incoming_text.reticulum_lxmf_hash, - sizeof(msg.reticulum_lxmf_hash)); - msg.source_unverified = incoming_text.source_unverified; - msg.rx_origin = incoming_text.rx_meta.origin; - msg.status = MessageStatus::Incoming; + ++drained; + const MeshProtocol incoming_protocol = active_protocol_; + ChatMessage msg = incomingChatMessage(incoming_text, + incoming_protocol); const ReticulumPeerIdentity* incoming_identity = hasReticulumDestinationIdentity(incoming_text.reticulum_identity) @@ -618,7 +756,7 @@ void ChatService::processIncoming() incoming_dest_hash, sizeof(incoming_dest_hash)); CHAT_SERVICE_DIAG_LOG("[ChatService][RX] text protocol=%s msg=%lu from=%08lX to=%08lX peer=%08lX dest=%s len=%u encrypted=%u unverified=%u lxmf_hash=%u\n", - protocol_name(active_protocol_), + protocol_name(incoming_protocol), static_cast(msg.msg_id), static_cast(msg.from), static_cast(incoming_text.to), @@ -653,48 +791,43 @@ void ChatService::processIncoming() continue; } - if (!store_.appendIncomingDurably(msg)) + if (isDeferredIncoming(msg)) { - CHAT_SERVICE_DIAG_LOG("[ChatService][RX] durable append failed protocol=%s msg=%lu lxmf_hash=%u\n", - protocol_name(active_protocol_), - static_cast(msg.msg_id), - chat::hasReticulumLxmfMessageHash(msg) ? 1U : 0U); - if (IIncomingDeliveryCommitPort* commit_port = - adapter_.incomingDeliveryCommitPort()) - { - commit_port->commitIncomingText(incoming_text, false); - } + CHAT_SERVICE_DIAG_LOG("[ChatService][RX] pending duplicate held protocol=%s msg=%lu\n", + protocol_name(incoming_protocol), + static_cast(msg.msg_id)); continue; } - rememberIncoming(msg); - - if (model_enabled_) + if (message_ledger_.recordIncoming(msg) != + delivery::LedgerPersistence::Durable) { - model_.onIncoming(msg); - } - - if (IIncomingDeliveryCommitPort* commit_port = - adapter_.incomingDeliveryCommitPort()) - { - commit_port->commitIncomingText(incoming_text, true); - } - - for (auto* observer : incoming_message_observers_) - { - if (observer) + const IncomingIdentity identity = incomingIdentityForMessage(msg); + if (deferred_incoming_.push(incoming_text, + incoming_protocol, + identity)) { - observer->onIncomingMessage(msg, &incoming_text.rx_meta); + CHAT_SERVICE_EVENT_LOG("[ChatService][RX] deferred protocol=%s msg=%lu depth=%u lxmf_hash=%u\n", + protocol_name(incoming_protocol), + static_cast(msg.msg_id), + static_cast(deferred_incoming_.count), + chat::hasReticulumLxmfMessageHash(msg) ? 1U : 0U); } - } - - for (auto* observer : incoming_text_observers_) - { - if (observer) + else { - observer->onIncomingText(incoming_text); + CHAT_SERVICE_EVENT_LOG("[ChatService][RX] rejected reason=deferred_full protocol=%s msg=%lu depth=%u\n", + protocol_name(incoming_protocol), + static_cast(msg.msg_id), + static_cast(deferred_incoming_.count)); + if (IIncomingDeliveryCommitPort* commit_port = + adapter_.incomingDeliveryCommitPort()) + { + commit_port->commitIncomingText(incoming_text, false); + } } + continue; } + commitIncoming(incoming_text, msg); } if (incoming_data_observers_.empty()) @@ -741,6 +874,21 @@ bool ChatService::isDuplicateIncoming(const ChatMessage& msg) const return false; } + return recent_incoming_.contains(incomingIdentityForMessage(msg)); +} + +bool ChatService::isDeferredIncoming(const ChatMessage& msg) const +{ + if (msg.msg_id == 0 && !chat::hasReticulumLxmfMessageHash(msg)) + { + return false; + } + return deferred_incoming_.contains(incomingIdentityForMessage(msg)); +} + +ChatService::IncomingIdentity ChatService::incomingIdentityForMessage( + const ChatMessage& msg) +{ IncomingIdentity identity{}; identity.protocol = msg.protocol; identity.channel = msg.channel; @@ -761,7 +909,7 @@ bool ChatService::isDuplicateIncoming(const ChatMessage& msg) const identity.reticulum_destination_hash, msg.reticulum_identity); } - return recent_incoming_.contains(identity); + return identity; } void ChatService::rememberIncoming(const ChatMessage& msg) @@ -772,27 +920,84 @@ void ChatService::rememberIncoming(const ChatMessage& msg) return; } - IncomingIdentity identity{}; - identity.protocol = msg.protocol; - identity.channel = msg.channel; - identity.from = msg.from; - identity.peer = msg.peer; - identity.msg_id = msg.msg_id; - if (chat::hasReticulumLxmfMessageHash(msg)) + recent_incoming_.remember(incomingIdentityForMessage(msg)); +} + +ChatMessage ChatService::incomingChatMessage( + const MeshIncomingText& incoming, + MeshProtocol protocol) +{ + ChatMessage message; + message.protocol = protocol; + message.channel = incoming.channel; + message.from = incoming.from; + message.peer = normalize_conversation_peer(incoming.to) == 0 + ? 0 + : incoming.from; + message.msg_id = incoming.msg_id; + message.timestamp = persistable_incoming_timestamp(incoming); + message.text = incoming.text; + message.reticulum_identity = incoming.reticulum_identity; + message.has_reticulum_lxmf_hash = incoming.has_reticulum_lxmf_hash; + std::memcpy(message.reticulum_lxmf_hash, + incoming.reticulum_lxmf_hash, + sizeof(message.reticulum_lxmf_hash)); + message.source_unverified = incoming.source_unverified; + message.rx_origin = incoming.rx_meta.origin; + message.status = MessageStatus::Incoming; + return message; +} + +void ChatService::commitIncoming(const MeshIncomingText& incoming, + const ChatMessage& message) +{ + rememberIncoming(message); + if (model_enabled_) { - identity.has_reticulum_lxmf_hash = true; - std::memcpy(identity.reticulum_lxmf_hash, - msg.reticulum_lxmf_hash, - sizeof(identity.reticulum_lxmf_hash)); + model_.onIncoming(message); } - if (msg.protocol == MeshProtocol::Reticulum && - hasReticulumDestinationIdentity(msg.reticulum_identity)) + if (IIncomingDeliveryCommitPort* commit_port = + adapter_.incomingDeliveryCommitPort()) { - identity.has_reticulum_destination = copyReticulumDestinationHash( - identity.reticulum_destination_hash, - msg.reticulum_identity); + commit_port->commitIncomingText(incoming, true); } - recent_incoming_.remember(identity); + for (auto* observer : incoming_message_observers_) + { + if (observer) + { + observer->onIncomingMessage(message, &incoming.rx_meta); + } + } + for (auto* observer : incoming_text_observers_) + { + if (observer) + { + observer->onIncomingText(incoming); + } + } +} + +void ChatService::retryDeferredIncoming() +{ + MeshIncomingText incoming{}; + MeshProtocol protocol = MeshProtocol::Meshtastic; + if (!deferred_incoming_.peek(incoming, protocol)) + { + return; + } + + const ChatMessage message = incomingChatMessage(incoming, protocol); + if (message_ledger_.recordIncoming(message) != + delivery::LedgerPersistence::Durable) + { + return; + } + deferred_incoming_.pop(); + CHAT_SERVICE_EVENT_LOG("[ChatService][RX] deferred commit protocol=%s msg=%lu remaining=%u\n", + protocol_name(protocol), + static_cast(message.msg_id), + static_cast(deferred_incoming_.count)); + commitIncoming(incoming, message); } void ChatService::flushStore() @@ -929,11 +1134,7 @@ void ChatService::setDeliveryEventPort( const ChatMessage* ChatService::getMessage(MessageId msg_id) const { - if (const ChatMessage* msg = model_.getMessage(msg_id)) - { - return msg; - } - if (store_.getMessage(msg_id, &store_lookup_cache_)) + if (message_ledger_.findMessage(msg_id, store_lookup_cache_)) { return &store_lookup_cache_; } @@ -944,11 +1145,9 @@ const ChatMessage* ChatService::getMessageForProtocol( MessageId msg_id, MeshProtocol protocol) const { - if (const ChatMessage* msg = model_.getMessageForProtocol(msg_id, protocol)) - { - return msg; - } - if (store_.getMessageForProtocol(msg_id, protocol, &store_lookup_cache_)) + if (message_ledger_.findMessageForProtocol(msg_id, + protocol, + store_lookup_cache_)) { return &store_lookup_cache_; } diff --git a/modules/core_chat/src/usecase/contact_service.cpp b/modules/core_chat/src/usecase/contact_service.cpp index 28c73828..164c4f5c 100644 --- a/modules/core_chat/src/usecase/contact_service.cpp +++ b/modules/core_chat/src/usecase/contact_service.cpp @@ -92,6 +92,8 @@ ContactService::ContactService(INodeStore& node_store, IContactStore& contact_st void ContactService::begin() { + node_store_.setActiveProtocol(active_protocol_); + contact_store_.setActiveProtocol(active_protocol_); node_store_.begin(); contact_store_.begin(); @@ -104,6 +106,16 @@ void ContactService::begin() invalidateCache(); } +void ContactService::setActiveProtocol(MeshProtocol protocol) +{ + active_protocol_ = protocol == MeshProtocol::RNode + ? MeshProtocol::Reticulum + : protocol; + node_store_.setActiveProtocol(active_protocol_); + contact_store_.setActiveProtocol(active_protocol_); + invalidateCache(); +} + void ContactService::applyNodeUpdate(uint32_t node_id, const NodeUpdate& update) { node_store_.applyUpdate(node_id, update); @@ -435,6 +447,14 @@ void ContactService::buildCache() const const auto& node_entries = node_store_.getEntries(); for (const auto& entry : node_entries) { + const MeshProtocol entry_protocol = + entry.protocol == static_cast(MeshProtocol::RNode) + ? MeshProtocol::Reticulum + : static_cast(entry.protocol); + if (entry_protocol != active_protocol_) + { + continue; + } if (!isNodeVisible(entry.last_seen)) { continue; diff --git a/modules/core_chat/tests/test_chat_message_ledger.cpp b/modules/core_chat/tests/test_chat_message_ledger.cpp index 66625190..a1f423bc 100644 --- a/modules/core_chat/tests/test_chat_message_ledger.cpp +++ b/modules/core_chat/tests/test_chat_message_ledger.cpp @@ -21,6 +21,38 @@ class RecordingDeliveryEventPort final int count = 0; }; +class TransientStore final : public chat::RamStore +{ + public: + bool appendDurably(const chat::ChatMessage& message) override + { + if (append_failures > 0) + { + --append_failures; + return false; + } + append(message); + return true; + } + + bool updateMessageStatusForProtocol(chat::MessageId msg_id, + chat::MeshProtocol protocol, + chat::MessageStatus status) override + { + if (status_failures > 0) + { + --status_failures; + return false; + } + return RamStore::updateMessageStatusForProtocol(msg_id, + protocol, + status); + } + + int append_failures = 0; + int status_failures = 0; +}; + chat::ChatMessage outgoing(chat::MessageId id, chat::MessageStatus status) { chat::ChatMessage message; @@ -202,5 +234,81 @@ int main() &stored)); assert(stored.status == chat::MessageStatus::Delivered); + chat::ChatModel deferred_model; + TransientStore deferred_store; + deferred_store.append_failures = 1; + chat::delivery::ChatMessageLedger deferred_ledger(deferred_model, + deferred_store); + assert(deferred_ledger.recordOutbound( + outgoing(600, chat::MessageStatus::Queued), + true) == chat::delivery::LedgerPersistence::Deferred); + assert(!deferred_store.getMessage(600, &stored)); + std::size_t deferred_total = 0; + const chat::ConversationId deferred_conversation( + chat::ChannelId::PRIMARY, + 0xAABBCCDD, + chat::MeshProtocol::Meshtastic); + auto deferred_page = deferred_ledger.loadPageFromLatest( + deferred_conversation, + 0, + 10, + &deferred_total); + assert(deferred_total == 1); + assert(deferred_page.size() == 1); + assert(deferred_page.front().msg_id == 600); + auto deferred_conversations = deferred_ledger.loadConversationPage( + chat::MeshProtocol::Meshtastic, + 0, + 10, + &deferred_total); + assert(deferred_total == 1); + assert(deferred_conversations.size() == 1); + assert(deferred_conversations.front().id == deferred_conversation); + assert(deferred_conversations.front().preview == "ledger"); + assert(deferred_ledger.applyOutboundStatusForProtocol( + 600, + chat::MeshProtocol::Meshtastic, + chat::MessageStatus::Delivered, + true)); + assert(deferred_ledger.flushPendingWrites(1) == 1); + assert(deferred_store.getMessageForProtocol( + 600, + chat::MeshProtocol::Meshtastic, + &stored)); + assert(stored.status == chat::MessageStatus::Delivered); + deferred_page = deferred_ledger.loadPageFromLatest( + deferred_conversation, + 0, + 10, + &deferred_total); + assert(deferred_total == 1); + assert(deferred_page.size() == 1); + + deferred_store.status_failures = 1; + deferred_ledger.recordOutbound( + outgoing(601, chat::MessageStatus::Queued), + true); + assert(deferred_ledger.applyOutboundStatusForProtocol( + 601, + chat::MeshProtocol::Meshtastic, + chat::MessageStatus::Delivered, + true)); + assert(deferred_store.getMessageForProtocol( + 601, + chat::MeshProtocol::Meshtastic, + &stored)); + assert(stored.status == chat::MessageStatus::Queued); + assert(deferred_ledger.findMessageForProtocol( + 601, + chat::MeshProtocol::Meshtastic, + stored)); + assert(stored.status == chat::MessageStatus::Delivered); + assert(deferred_ledger.flushPendingWrites(1) == 1); + assert(deferred_store.getMessageForProtocol( + 601, + chat::MeshProtocol::Meshtastic, + &stored)); + assert(stored.status == chat::MessageStatus::Delivered); + return 0; } diff --git a/modules/core_chat/tests/test_chat_service_resend.cpp b/modules/core_chat/tests/test_chat_service_resend.cpp index df43c2b4..9f17d2b6 100644 --- a/modules/core_chat/tests/test_chat_service_resend.cpp +++ b/modules/core_chat/tests/test_chat_service_resend.cpp @@ -162,6 +162,23 @@ class CountingIncomingObserver final : public chat::ChatService::IncomingMessage chat::MessageId last_msg_id = 0; }; +class TransientIncomingStore final : public chat::RamStore +{ + public: + bool appendIncomingDurably(const chat::ChatMessage& message) override + { + if (failures > 0) + { + --failures; + return false; + } + append(message); + return true; + } + + int failures = 0; +}; + const chat::ChatMessage* onlyMessage(chat::ChatService& service, const chat::ConversationId& conv) { @@ -540,7 +557,10 @@ int main() { incoming_mesh.pushIncoming(0x1234ABCDU, 0x1000U + i, "window fill"); } - incoming_service.processIncoming(); + for (std::size_t tick = 0; tick < 64; ++tick) + { + incoming_service.processIncoming(); + } assert(incoming_observer.count == 257); incoming_mesh.pushIncoming(0x1234ABCDU, 0x10FFU, "recent duplicate"); @@ -550,6 +570,34 @@ int main() assert(incoming_observer.last_msg_id == 0x42U); } + { + chat::ChatModel incoming_model; + FakeMeshAdapter incoming_mesh; + TransientIncomingStore incoming_store; + incoming_store.failures = 1; + chat::ChatService incoming_service(incoming_model, + incoming_mesh, + incoming_store); + CountingIncomingObserver incoming_observer; + incoming_service.addIncomingMessageObserver(&incoming_observer); + const chat::ConversationId broadcast(chat::ChannelId::PRIMARY, + 0, + chat::MeshProtocol::Meshtastic); + + incoming_mesh.pushIncoming(0x1234ABCDU, 0x77U, "deferred"); + incoming_service.processIncoming(); + assert(incoming_store.loadRecent(broadcast, 10).empty()); + assert(incoming_observer.count == 0); + + incoming_mesh.pushIncoming(0x1234ABCDU, 0x77U, "deferred"); + incoming_service.processIncoming(); + const auto committed = incoming_store.loadRecent(broadcast, 10); + assert(committed.size() == 1); + assert(committed.front().msg_id == 0x77U); + assert(incoming_store.getUnread(broadcast) == 1); + assert(incoming_observer.count == 1); + } + { chat::ChatModel incoming_model; FakeMeshAdapter incoming_mesh; diff --git a/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp b/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp index c2a86317..ac5e8c55 100644 --- a/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp +++ b/modules/core_chat/tests/test_esp_sd_store_read_state_contract.cpp @@ -58,65 +58,125 @@ int main(int argc, char** argv) const std::string source = readFile( repo_root / "platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp"); + const std::string codec = readFile( + repo_root / + "platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp"); + const std::string journal = readFile( + repo_root / + "platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp"); + const std::string peer_header = readFile( + repo_root / + "platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h"); + const std::string peer_source = readFile( + repo_root / + "platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp"); + const std::string peer_codec = readFile( + repo_root / + "platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp"); + const std::string bindings = readFile( + repo_root / + "platform/esp/arduino_common/src/app_context_platform_bindings.cpp"); - assert(contains(header, "kReadStateFile = \"/chat/read_state.bin\"")); - assert(contains(header, "struct ReadStateEntry")); - assert(contains(header, "kReadStateMagic")); - assert(contains(header, "readStateUnreadOrLegacy")); - assert(contains(header, "writeReadStateUnread")); - assert(contains(header, "removeReadStateEntry")); + assert(contains(header, "kRoot = \"/data/v2\"")); + assert(contains(header, "kMeshtasticRoot = \"/data/v2/mt/chat\"")); + assert(contains(header, "kMeshCoreRoot = \"/data/v2/mc/chat\"")); + assert(contains(header, "kReticulumRoot = \"/data/v2/rt/chat\"")); + assert(contains(header, "PsramAllocator")); + assert(contains(header, "mutable ScratchBuffer scratch_")); + assert(!contains(header, "mutable uint8_t scratch_")); + assert(!contains(header, "RecordV2")); + assert(!contains(header, "RecordV3")); + assert(!contains(header, "RecordV4")); + assert(!contains(header, "kReadStateFile")); + assert(!contains(source, "\"/chat/")); + assert(!contains(source, "Legacy")); - const std::string ctor_body = - bodyBetween(source, "SdStore::SdStore()", "void SdStore::append"); - assert(contains(ctor_body, "reconcileIndexUnread(entries)")); - assert(positionOf(ctor_body, "reconcileIndexUnread(entries)") < - positionOf(ctor_body, "writeIndex(entries)")); + assert(contains(codec, "kMeshtasticTextMax")); + assert(contains(codec, "kMeshCoreTextMax")); + assert(contains(codec, "kReticulumTextMax")); + assert(contains(codec, "struct MeshtasticMessageSlot")); + assert(contains(codec, "struct MeshCoreMessageSlot")); + assert(contains(codec, "struct ReticulumMessageSlot")); + assert(contains(journal, "kStorageSchemaVersion")); + assert(contains(journal, "PartialTail")); + assert(contains(journal, "replaceFileAtomically")); + assert(contains(journal, "recoverAtomicFile")); const std::string append_body = bodyBetween( - source, "bool SdStore::appendInternal", "std::vector SdStore::loadRecent"); - assert(positionOf(append_body, "already_committed") < - positionOf(append_body, "readStateUnreadOrLegacy(conv, &committed_unread)")); - assert(positionOf(append_body, "readStateUnreadOrLegacy(conv, &committed_unread)") < - positionOf(append_body, "updateIndexForMessage(msg, committed_unread)")); - assert(positionOf(append_body, "writeReadStateUnread(conv, unread)") < - positionOf(append_body, "updateIndexForMessage(msg, unread)")); - assert(positionOf(append_body, "CHAT_STORE_LOG(\"[AppContext] chat unread persist failed stage=read_state") < - positionOf(append_body, "removeReadStateEntry(conv)")); + source, + "bool SdStore::appendInternal", + "std::vector SdStore::loadRecent"); + assert(positionOf(append_body, "appendMessageRecord(message, sequence)") < + positionOf(append_body, "rememberReticulumHash")); + assert(positionOf(append_body, "rememberReticulumHash") < + positionOf(append_body, "appendCatalogProjection")); + assert(contains(append_body, "projection_dirty_")); + assert(contains(append_body, "authoritative=1")); + assert(contains(append_body, "return true;")); - const std::string set_unread_body = + const std::string unread_body = bodyBetween(source, "bool SdStore::setUnread", "int SdStore::getUnread"); - assert(positionOf(set_unread_body, "writeReadStateUnread(conv, unread_count)") < - positionOf(set_unread_body, "writeConversationUnread(conv, unread_count)")); - assert(positionOf(set_unread_body, "writeConversationUnread(conv, unread_count)") < - positionOf(set_unread_body, "entries[index].unread = unread_count")); - assert(positionOf(set_unread_body, "entries[index].unread = unread_count") < - positionOf(set_unread_body, "writeIndex(entries)")); + assert(positionOf(unread_body, "appendReadProjection(projection)") < + positionOf(unread_body, "appendCatalogProjection(*catalog)")); + assert(contains(unread_body, "last_read_sequence")); - const std::string get_unread_body = - bodyBetween(source, "int SdStore::getUnread", "void SdStore::clearConversation"); - assert(contains(get_unread_body, "readStateUnreadOrLegacy(conv, &unread)")); + const std::string conversation_query = bodyBetween( + source, + "SdStore::loadConversationPageForProtocol", + "bool SdStore::setUnread"); + assert(contains(conversation_query, "sameProtocol")); - const std::string clear_conversation_body = - bodyBetween(source, "void SdStore::clearConversation", "void SdStore::clearAll"); - assert(contains(clear_conversation_body, "removeReadStateEntry(conv)")); + const std::string clear_all = bodyBetween( + source, + "void SdStore::clearAll", + "bool SdStore::updateMessageStatus"); + assert(contains(clear_all, "removeTree(protocolRoot(protocol))")); + assert(!contains(clear_all, "RecordV")); - const std::string clear_all_body = - bodyBetween(source, "void SdStore::clearAll", "bool SdStore::updateMessageStatus"); - assert(contains(clear_all_body, "sd_remove(kReadStateFile)")); - assert(contains(clear_all_body, "sd_remove(kTempReadStateFile)")); - assert(contains(clear_all_body, "sd_remove(kBackupReadStateFile)")); + assert(contains(peer_header, "IProtocolPeerRepository")); + assert(contains(peer_header, "PsramAllocator")); + assert(contains(peer_source, "\"peers.snapshot\"")); + assert(contains(peer_source, "\"peers.delta\"")); + assert(contains(peer_source, "\"contacts.snapshot\"")); + assert(contains(peer_source, "\"contacts.delta\"")); + assert(contains(peer_source, "kEphemeralPeerCapacity = 2048U")); + assert(contains(peer_source, "kProtectedContactCapacity = 4096U")); + assert(contains(peer_source, "peerIsProtected")); + assert(contains(peer_source, "peerReferencedByConversation")); + assert(contains(peer_source, "replaceFileAtomically")); + assert(contains(peer_codec, "validPeerIdentityForProtocol")); + assert(contains(peer_codec, "validContactIdentityForProtocol")); + const std::string peer_prefix = bodyBetween( + peer_codec, + "struct PeerPrefix", + "struct NodeFactsSlot"); + assert(!contains(peer_prefix, "user_alias")); + assert(!contains(bindings, "EspSdMeshPeerDirectoryBlobStore")); + assert(!contains(bindings, "chat::meshtastic::NodeStore")); + assert(!contains(bindings, "chat::contacts::ContactStore")); + assert(!contains(bindings, "\"/nodes.bin\"")); + assert(!contains(bindings, "\"/contacts.dat\"")); + assert(!contains(bindings, "\"/mesh/peers.bin\"")); + assert(!std::filesystem::exists( + repo_root / + "platform/esp/arduino_common/src/chat/infra/meshtastic/node_store.cpp")); + assert(!std::filesystem::exists( + repo_root / + "platform/esp/arduino_common/src/chat/infra/contact_store.cpp")); - const std::string reconcile_body = - bodyBetween(source, "bool SdStore::reconcileIndexUnread", "bool SdStore::readIndex"); - assert(positionOf(reconcile_body, "readStateUnreadOrLegacy(conv, &durable_unread)") < - positionOf(reconcile_body, "durable_unread = entry.unread")); - assert(positionOf(reconcile_body, "durable_unread = entry.unread") < - positionOf(reconcile_body, "writeReadStateUnread(conv, durable_unread)")); - - const std::string rebuild_body = - bodyBetween(source, "void SdStore::rebuildIndex", "bool SdStore::loadFileHeader"); - assert(contains(rebuild_body, "readStateUnreadOrLegacy(conv, &ledger_unread)")); - assert(contains(rebuild_body, "writeReadStateUnread(conv, unread)")); + const std::string seen_load = bodyBetween( + source, + "bool SdStore::loadSeenJournal", + "bool SdStore::reconcileProtocolCatalog"); + assert(contains(seen_load, "rebuildSeenJournalFromMessages")); + assert(contains(seen_load, "authoritative=messages")); + const std::string flush_body = bodyBetween( + source, + "void SdStore::flush", + "bool SdStore::ensureLayout"); + assert(contains(flush_body, "projection_dirty_")); + assert(contains(flush_body, "kProjectionRetryIntervalMs")); + assert(contains(flush_body, "BackgroundWorkerBounded")); return 0; } diff --git a/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp b/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp index 01616236..d4bd8a9e 100644 --- a/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp +++ b/modules/core_chat/tests/test_mesh_peer_directory_contract.cpp @@ -811,6 +811,40 @@ void core_merges_sparse_meshtastic_records_without_losing_key_or_node_facts() } } +void verified_keys_cannot_be_replaced_by_runtime_observations() +{ + const uint32_t node_id = 0x12345678U; + chat::MeshPeerRecord verified = + makeMeshtasticKeyOnlyPeer(node_id, 0x20U, 10U); + verified.meshtastic.key_manually_verified = true; + + chat::MeshPeerRecord observed = + makeMeshtasticKeyOnlyPeer(node_id, 0x80U, 20U); + observed.meshtastic.key_manually_verified = false; + chat::copyMeshPeerText(observed.display_name, + sizeof(observed.display_name), + "fresh name"); + + const chat::MeshPeerRecord merged = + chat::mergeMeshPeerRecordFacts(verified, observed); + assert(merged.meshtastic.key_manually_verified); + assert(std::memcmp(merged.meshtastic.public_key, + verified.meshtastic.public_key, + chat::kMeshPeerMeshtasticPublicKeyLen) == 0); + assert(std::strcmp(merged.display_name, "fresh name") == 0); + + chat::MeshPeerRecord meshcore = makeMeshCorePeer(0x11U, "trusted", 10U); + chat::MeshPeerRecord conflicting = meshcore; + conflicting.meshcore.public_key_verified = false; + conflicting.meshcore.public_key[0] ^= 0x7FU; + const chat::MeshPeerRecord meshcore_merged = + chat::mergeMeshPeerRecordFacts(meshcore, conflicting); + assert(meshcore_merged.meshcore.public_key_verified); + assert(std::memcmp(meshcore_merged.meshcore.public_key, + meshcore.meshcore.public_key, + chat::kMeshPeerMeshCorePublicKeyLen) == 0); +} + void remove_and_clear_protocol_are_directory_behaviors() { MemoryMeshPeerDirectory directory; @@ -842,6 +876,7 @@ int main() find_by_node_id_preserves_reticulum_destination_identity(); core_persists_records_and_preserves_user_flags(); core_merges_sparse_meshtastic_records_without_losing_key_or_node_facts(); + verified_keys_cannot_be_replaced_by_runtime_observations(); remove_and_clear_protocol_are_directory_behaviors(); return 0; } diff --git a/modules/core_chat/tests/test_protocol_chat_storage_v2_codec.cpp b/modules/core_chat/tests/test_protocol_chat_storage_v2_codec.cpp new file mode 100644 index 00000000..df5fd68b --- /dev/null +++ b/modules/core_chat/tests/test_protocol_chat_storage_v2_codec.cpp @@ -0,0 +1,104 @@ +#include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" + +#include +#include +#include +#include + +namespace storage = chat::storage::v2; + +namespace +{ + +chat::ChatMessage messageFor(chat::MeshProtocol protocol, + std::size_t text_length) +{ + chat::ChatMessage message{}; + message.protocol = protocol; + message.channel = chat::ChannelId::SECONDARY; + message.from = 0x11223344U; + message.peer = 0x55667788U; + message.msg_id = 0xAABBCCDDU; + message.timestamp = 123456U; + message.text.assign(text_length, 'x'); + message.has_geo = true; + message.geo_lat_e7 = 123; + message.geo_lon_e7 = -456; + message.source_unverified = true; + message.rx_origin = chat::RxOrigin::WiFi; + message.status = chat::MessageStatus::Incoming; + return message; +} + +void roundTrip(chat::MeshProtocol protocol, std::size_t text_length) +{ + chat::ChatMessage input = messageFor(protocol, text_length); + if (protocol == chat::MeshProtocol::Reticulum) + { + uint8_t destination[chat::kReticulumPeerHashSize]{}; + uint8_t identity[chat::kReticulumPeerHashSize]{}; + destination[0] = 0x42U; + identity[0] = 0x24U; + input.reticulum_identity = + chat::makeReticulumPeerIdentity(destination, identity); + input.has_reticulum_lxmf_hash = true; + input.reticulum_lxmf_hash[0] = 0x99U; + } + + std::vector slot(storage::messageSlotSize(protocol)); + assert(storage::encodeMessageSlot(input, 17U, slot.data(), slot.size())); + + chat::ChatMessage output{}; + uint32_t sequence = 0; + assert(storage::decodeMessageSlot(protocol, + slot.data(), + slot.size(), + output, + &sequence)); + assert(sequence == 17U); + assert(output.protocol == protocol); + assert(output.msg_id == input.msg_id); + assert(output.text == input.text); + assert(output.has_geo); + assert(output.geo_lon_e7 == input.geo_lon_e7); + assert(output.source_unverified); + if (protocol == chat::MeshProtocol::Reticulum) + { + assert(chat::sameReticulumDestinationHash(output.reticulum_identity, + input.reticulum_identity)); + assert(output.has_reticulum_lxmf_hash); + assert(std::memcmp(output.reticulum_lxmf_hash, + input.reticulum_lxmf_hash, + chat::kReticulumLxmfHashSize) == 0); + } + + slot.back() ^= 0x80U; + assert(!storage::decodeMessageSlot(protocol, + slot.data(), + slot.size(), + output)); +} + +} // namespace + +int main() +{ + roundTrip(chat::MeshProtocol::Meshtastic, + storage::kMeshtasticTextMax); + roundTrip(chat::MeshProtocol::MeshCore, storage::kMeshCoreTextMax); + roundTrip(chat::MeshProtocol::Reticulum, + storage::kReticulumTextMax); + + chat::ChatMessage too_long = messageFor( + chat::MeshProtocol::Meshtastic, + storage::kMeshtasticTextMax + 1U); + std::vector slot( + storage::messageSlotSize(chat::MeshProtocol::Meshtastic)); + assert(!storage::encodeMessageSlot(too_long, + 1U, + slot.data(), + slot.size())); + assert(storage::messageSlotSize(chat::MeshProtocol::Meshtastic) < + storage::messageSlotSize(chat::MeshProtocol::Reticulum)); + return 0; +} diff --git a/modules/core_chat/tests/test_protocol_peer_storage_v2_codec.cpp b/modules/core_chat/tests/test_protocol_peer_storage_v2_codec.cpp new file mode 100644 index 00000000..3918f1f4 --- /dev/null +++ b/modules/core_chat/tests/test_protocol_peer_storage_v2_codec.cpp @@ -0,0 +1,179 @@ +#include "platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h" + +#include +#include +#include + +namespace storage = chat::storage::v2; + +namespace +{ + +void fillKey(uint8_t* key, std::size_t len, uint8_t seed) +{ + for (std::size_t index = 0U; index < len; ++index) + { + key[index] = static_cast(seed + index); + } +} + +storage::PeerProjection meshtasticPeer() +{ + storage::PeerProjection projection{}; + chat::MeshPeerRecord& peer = projection.record; + peer.valid = true; + peer.identity = chat::makeMeshPeerNodeIdentity( + chat::MeshProtocol::Meshtastic, + 0x11223344U); + peer.source = chat::MeshPeerSource::RuntimeRx; + peer.first_seen_s = 10U; + peer.last_seen_s = 20U; + chat::copyMeshPeerText(peer.display_name, + sizeof(peer.display_name), + "Meshtastic peer"); + chat::copyMeshPeerText(peer.user_alias, + sizeof(peer.user_alias), + "not-in-peer"); + peer.flags.favorite = true; + chat::copyMeshPeerText(peer.meshtastic.node.short_name, + sizeof(peer.meshtastic.node.short_name), + "MT"); + chat::copyMeshPeerText(peer.meshtastic.node.long_name, + sizeof(peer.meshtastic.node.long_name), + "Meshtastic Node"); + peer.meshtastic.node.channel = 1U; + peer.meshtastic.node.hops_away = 2U; + peer.meshtastic.has_next_hop = true; + peer.meshtastic.next_hop = 7U; + peer.meshtastic.has_public_key = true; + peer.meshtastic.key_manually_verified = true; + fillKey(peer.meshtastic.public_key, + sizeof(peer.meshtastic.public_key), + 1U); + peer.observations.has_snr = true; + peer.observations.snr = 4.5F; + peer.observations.has_position = true; + peer.observations.position.valid = true; + peer.observations.position.latitude_i = 123; + return projection; +} + +void roundTripPeer(chat::MeshProtocol protocol, + const storage::PeerProjection& input) +{ + std::vector slot(storage::peerSlotSize(protocol)); + assert(storage::encodePeerSlot(protocol, + input, + slot.data(), + slot.size())); + storage::PeerProjection output{}; + assert(storage::decodePeerSlot(protocol, + slot.data(), + slot.size(), + output)); + assert(chat::sameMeshPeerIdentity(input.record.identity, + output.record.identity)); + assert(output.record.last_seen_s == input.record.last_seen_s); + assert(std::strcmp(output.record.display_name, + input.record.display_name) == 0); + assert(output.record.user_alias[0] == '\0'); + assert(!output.record.flags.favorite); + slot.back() ^= 0x80U; + assert(!storage::decodePeerSlot(protocol, + slot.data(), + slot.size(), + output)); +} + +void unresolvedIdentityRoundTrips() +{ + storage::PeerProjection mc{}; + mc.record.valid = true; + mc.record.identity = chat::makeMeshPeerNodeIdentity( + chat::MeshProtocol::MeshCore, + 0xAABBCCDDU); + mc.record.meshcore.node_id_hint = 0xAABBCCDDU; + roundTripPeer(chat::MeshProtocol::MeshCore, mc); + + storage::PeerProjection rt{}; + rt.record.valid = true; + rt.record.identity = chat::makeMeshPeerNodeIdentity( + chat::MeshProtocol::Reticulum, + 0x01020304U); + roundTripPeer(chat::MeshProtocol::Reticulum, rt); +} + +void stableIdentityAndContactRoundTrips() +{ + uint8_t public_key[chat::kMeshPeerMeshCorePublicKeyLen]{}; + fillKey(public_key, sizeof(public_key), 9U); + storage::PeerProjection mc{}; + mc.record.valid = true; + assert(chat::makeMeshPeerPublicKeyIdentity( + chat::MeshProtocol::MeshCore, + public_key, + sizeof(public_key), + mc.record.identity)); + mc.record.meshcore.has_public_key = true; + mc.record.meshcore.node_id_hint = 0x55667788U; + std::memcpy(mc.record.meshcore.public_key, + public_key, + sizeof(public_key)); + roundTripPeer(chat::MeshProtocol::MeshCore, mc); + + storage::ContactProjection contact{}; + contact.identity = mc.record.identity; + contact.node_id_hint = mc.record.meshcore.node_id_hint; + contact.flags.favorite = true; + contact.flags.trusted = true; + chat::copyMeshPeerText(contact.alias, + sizeof(contact.alias), + "base"); + std::vector slot( + storage::contactSlotSize(chat::MeshProtocol::MeshCore)); + assert(storage::encodeContactSlot(chat::MeshProtocol::MeshCore, + contact, + slot.data(), + slot.size())); + storage::ContactProjection decoded{}; + assert(storage::decodeContactSlot(chat::MeshProtocol::MeshCore, + slot.data(), + slot.size(), + decoded)); + assert(decoded.node_id_hint == contact.node_id_hint); + assert(decoded.flags.favorite); + assert(decoded.flags.trusted); + assert(std::strcmp(decoded.alias, contact.alias) == 0); + + uint8_t destination[chat::kReticulumPeerHashSize]{}; + uint8_t identity[chat::kReticulumPeerHashSize]{}; + fillKey(destination, sizeof(destination), 3U); + fillKey(identity, sizeof(identity), 33U); + storage::PeerProjection rt{}; + rt.record.valid = true; + rt.record.identity = chat::makeMeshPeerReticulumIdentity( + chat::makeReticulumPeerIdentity(destination, identity)); + rt.record.reticulum.identity = rt.record.identity.reticulum; + roundTripPeer(chat::MeshProtocol::Reticulum, rt); +} + +} // namespace + +int main() +{ + const storage::PeerProjection mt = meshtasticPeer(); + roundTripPeer(chat::MeshProtocol::Meshtastic, mt); + unresolvedIdentityRoundTrips(); + stableIdentityAndContactRoundTrips(); + + assert(storage::peerSlotSize(chat::MeshProtocol::Meshtastic) < + sizeof(chat::MeshPeerRecord)); + assert(storage::peerSlotSize(chat::MeshProtocol::MeshCore) < + sizeof(chat::MeshPeerRecord)); + assert(storage::peerSlotSize(chat::MeshProtocol::Reticulum) < + sizeof(chat::MeshPeerRecord)); + assert(storage::contactSlotSize(chat::MeshProtocol::Meshtastic) < 64U); + assert(storage::contactSlotSize(chat::MeshProtocol::MeshCore) < 80U); + assert(storage::contactSlotSize(chat::MeshProtocol::Reticulum) < 64U); + return 0; +} diff --git a/modules/core_sys/include/app/app_context_platform_bindings.h b/modules/core_sys/include/app/app_context_platform_bindings.h index 1ffdfb79..109b76d0 100644 --- a/modules/core_sys/include/app/app_context_platform_bindings.h +++ b/modules/core_sys/include/app/app_context_platform_bindings.h @@ -13,6 +13,7 @@ #include "chat/ports/i_mesh_adapter.h" #include "chat/ports/i_mesh_peer_directory.h" #include "chat/ports/i_node_store.h" +#include "chat/ports/i_protocol_peer_repository.h" #include "chat/usecase/chat_service.h" #include "chat/usecase/contact_service.h" #include "team/ports/i_team_crypto.h" @@ -40,7 +41,7 @@ struct ChatServicesBundle { std::unique_ptr model; std::unique_ptr store; - std::unique_ptr mesh_peer_directory; + std::unique_ptr mesh_peer_directory; std::unique_ptr mesh_runtime; std::unique_ptr service; std::unique_ptr incoming_message_observer; @@ -54,13 +55,11 @@ struct ChatServicesBundle struct ContactServicesBundle { - std::unique_ptr node_store; - std::unique_ptr contact_store; std::unique_ptr service; bool isValid() const { - return node_store && contact_store && service; + return service != nullptr; } }; @@ -106,7 +105,8 @@ struct AppContextPlatformBindings std::unique_ptr (*)(chat::MeshProtocol protocol, LoraBoard& lora_board, chat::IMeshPeerDirectory* peer_directory); - using ContactServicesFactory = ContactServicesBundle (*)(); + using ContactServicesFactory = + ContactServicesBundle (*)(chat::IProtocolPeerRepository& repository); using TeamServicesFactory = TeamServicesBundle (*)(chat::IMeshAdapter& mesh_adapter); using SelfNodeIdProvider = chat::NodeId (*)(); diff --git a/modules/ui_presentation/include/ui_presentation/chat/chat_workspace_snapshot.h b/modules/ui_presentation/include/ui_presentation/chat/chat_workspace_snapshot.h index efb44c50..016b8256 100644 --- a/modules/ui_presentation/include/ui_presentation/chat/chat_workspace_snapshot.h +++ b/modules/ui_presentation/include/ui_presentation/chat/chat_workspace_snapshot.h @@ -115,7 +115,7 @@ struct ChatWorkspaceSnapshot ui::SnapshotHeader header; static constexpr size_t kMaxLocationParticipants = 16; - static constexpr size_t kMaxMessages = 20; + static constexpr size_t kMaxMessages = 10; ConversationRow conversations[16]{}; size_t conversation_count = 0; diff --git a/modules/ui_presentation/src/chat/chat_workspace_model.cpp b/modules/ui_presentation/src/chat/chat_workspace_model.cpp index 4bae9d86..0d99aa13 100644 --- a/modules/ui_presentation/src/chat/chat_workspace_model.cpp +++ b/modules/ui_presentation/src/chat/chat_workspace_model.cpp @@ -5,7 +5,7 @@ #include #ifndef CHAT_UI_SEND_TRACE_ENABLE -#define CHAT_UI_SEND_TRACE_ENABLE 1 +#define CHAT_UI_SEND_TRACE_ENABLE 0 #endif #if CHAT_UI_SEND_TRACE_ENABLE diff --git a/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h b/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h index a48b6053..9817fa4f 100644 --- a/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h +++ b/modules/ui_shared/include/ui/screens/chat/chat_ui_controller.h @@ -152,7 +152,6 @@ class UiController : public IChatUiRefreshSink std::string resolveConversationDisplayName(const chat::ConversationId& conv) const; void applyConversationListToUi(); void updateConversationMetaForMessage(const chat::ChatMessage& msg, bool increment_unread); - bool updateConversationViewForIncoming(const chat::ChatMessage& msg); void reloadConversationView(); // Team text, read projection, and location action sends flow through // ChatTeamWorkflow. diff --git a/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp b/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp index 86018750..f480a6ca 100644 --- a/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp +++ b/modules/ui_shared/src/ui/presentation_sources/chat_presentation_source.cpp @@ -14,7 +14,7 @@ #include #ifndef CHAT_UI_SEND_TRACE_ENABLE -#define CHAT_UI_SEND_TRACE_ENABLE 1 +#define CHAT_UI_SEND_TRACE_ENABLE 0 #endif #if CHAT_UI_SEND_TRACE_ENABLE diff --git a/modules/ui_shared/src/ui/presentation_sources/runtime_chat_action_sink.cpp b/modules/ui_shared/src/ui/presentation_sources/runtime_chat_action_sink.cpp index b08d22d6..7cc58c8f 100644 --- a/modules/ui_shared/src/ui/presentation_sources/runtime_chat_action_sink.cpp +++ b/modules/ui_shared/src/ui/presentation_sources/runtime_chat_action_sink.cpp @@ -7,7 +7,7 @@ #include #ifndef CHAT_UI_SEND_TRACE_ENABLE -#define CHAT_UI_SEND_TRACE_ENABLE 1 +#define CHAT_UI_SEND_TRACE_ENABLE 0 #endif #if CHAT_UI_SEND_TRACE_ENABLE diff --git a/modules/ui_shared/src/ui/screens/chat/chat_conversation_components.cpp b/modules/ui_shared/src/ui/screens/chat/chat_conversation_components.cpp index 0ab9dd62..e9d3d8a5 100644 --- a/modules/ui_shared/src/ui/screens/chat/chat_conversation_components.cpp +++ b/modules/ui_shared/src/ui/screens/chat/chat_conversation_components.cpp @@ -29,7 +29,7 @@ #include #ifndef CHAT_CONVERSATION_LOG_ENABLE -#define CHAT_CONVERSATION_LOG_ENABLE 1 +#define CHAT_CONVERSATION_LOG_ENABLE 0 #endif #if CHAT_CONVERSATION_LOG_ENABLE diff --git a/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp b/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp index dbfedf34..12182793 100644 --- a/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp +++ b/modules/ui_shared/src/ui/screens/chat/chat_ui_controller.cpp @@ -35,7 +35,7 @@ #include #ifndef CHAT_UI_LOG_ENABLE -#define CHAT_UI_LOG_ENABLE 1 +#define CHAT_UI_LOG_ENABLE 0 #endif #if CHAT_UI_LOG_ENABLE @@ -883,7 +883,6 @@ void UiController::onRuntimeMessageArrived(chat::MessageId msg_id) updateConversationMetaForMessage(*latest, !is_current_conversation); if (is_current_conversation) { - (void)updateConversationViewForIncoming(*latest); reloadConversationView(); (void)chat_model_.markRead( chat_presentation_adapters::toUiConversationId(current_conv_)); @@ -1742,22 +1741,6 @@ void UiController::updateConversationMetaForMessage(const chat::ChatMessage& msg normalizeConversationNames(cached_conversations_); } -bool UiController::updateConversationViewForIncoming(const chat::ChatMessage& msg) -{ - if (!conversation_) - { - return false; - } - - if (!(current_conv_ == chat::conversationIdForMessage(msg))) - { - return false; - } - - reloadConversationView(); - return true; -} - void UiController::reloadConversationView() { if (!conversation_ || team_conv_active_) diff --git a/modules/ui_shared/tests/test_chat_presentation_source.cpp b/modules/ui_shared/tests/test_chat_presentation_source.cpp index 8d15ad87..dce72296 100644 --- a/modules/ui_shared/tests/test_chat_presentation_source.cpp +++ b/modules/ui_shared/tests/test_chat_presentation_source.cpp @@ -503,6 +503,7 @@ void setNodePosition(::chat::contacts::ContactService& contacts, int main() { + static_assert(::ui::chat::ChatWorkspaceSnapshot::kMaxMessages == 10); sys::set_epoch_seconds_provider([]() -> uint32_t { return 1700000000U; }); sys::set_millis_provider([]() -> uint32_t diff --git a/platform/esp/arduino_common/include/app/app_context.h b/platform/esp/arduino_common/include/app/app_context.h index d77f2e0b..c3248169 100644 --- a/platform/esp/arduino_common/include/app/app_context.h +++ b/platform/esp/arduino_common/include/app/app_context.h @@ -161,12 +161,12 @@ class AppContext final : public IAppBleFacade chat::contacts::INodeStore* getNodeStore() override { - return node_store_.get(); + return node_store_; } const chat::contacts::INodeStore* getNodeStore() const override { - return node_store_.get(); + return node_store_; } bool getDeviceMacAddress(uint8_t out_mac[6]) const override; @@ -175,12 +175,12 @@ class AppContext final : public IAppBleFacade chat::contacts::IContactStore* getContactStore() { - return contact_store_.get(); + return contact_store_; } const chat::contacts::IContactStore* getContactStore() const { - return contact_store_.get(); + return contact_store_; } void saveConfig() override; @@ -272,10 +272,10 @@ class AppContext final : public IAppBleFacade std::unique_ptr chat_model_; std::unique_ptr chat_store_; - std::unique_ptr mesh_peer_directory_; + std::unique_ptr mesh_peer_directory_; std::unique_ptr mesh_router_; - std::unique_ptr node_store_; - std::unique_ptr contact_store_; + chat::contacts::INodeStore* node_store_ = nullptr; + chat::contacts::IContactStore* contact_store_ = nullptr; std::unique_ptr chat_service_; std::unique_ptr contact_service_; diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/contact_store.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/contact_store.h deleted file mode 100644 index 3b04043d..00000000 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/contact_store.h +++ /dev/null @@ -1,68 +0,0 @@ -/** - * @file contact_store.h - * @brief Contact nickname storage shell backed by a single selected persistence backend - */ - -#pragma once - -#include "chat/infra/contact_store_core.h" -#include "chat/ports/i_contact_blob_store.h" - -#include - -namespace chat -{ -namespace contacts -{ - -class ContactStore : public IContactStore, - private chat::IContactBlobStore -{ - public: - static void* operator new(std::size_t size); - static void operator delete(void* ptr) noexcept; - static void operator delete(void* ptr, std::size_t) noexcept; - - ContactStore(); - - void begin() override; - std::string getNickname(uint32_t node_id) const override; - bool setNickname(uint32_t node_id, const char* nickname) override; - bool removeNickname(uint32_t node_id) override; - bool hasNickname(const char* nickname) const override; - std::vector getAllContactIds() const override; - size_t getCount() const override; - - private: - enum class StorageBackend : uint8_t - { - Sd, - Flash, - }; - - enum class LoadResult : uint8_t - { - Loaded, - MissingOrInvalid, - Busy, - }; - - static constexpr const char* kSdPath = "/contacts.dat"; - static constexpr const char* kPrefNs = "contacts"; - static constexpr const char* kPrefKey = "contact_blob"; - - bool loadBlob(std::vector& out) override; - bool saveBlob(const uint8_t* data, size_t len) override; - - LoadResult loadFromSD(std::vector& out) const; - bool saveToSD(const uint8_t* data, size_t len) const; - bool loadFromFlash(std::vector& out) const; - bool saveToFlash(const uint8_t* data, size_t len) const; - void clearFlash() const; - - ContactStoreCore core_; - StorageBackend backend_ = StorageBackend::Flash; -}; - -} // namespace contacts -} // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/lxmf/lxmf_memory.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/lxmf/lxmf_memory.h index 14b55eac..38803138 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/lxmf/lxmf_memory.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/lxmf/lxmf_memory.h @@ -6,101 +6,21 @@ #pragma once #include "chat/infra/lxmf/lxmf_wire.h" +#include "platform/esp/arduino_common/memory/psram_allocator.h" #include #include #include -#include -#include -#include #include #include -#if defined(ESP_PLATFORM) -#include -#endif - namespace chat::lxmf::runtime { -[[noreturn]] inline void allocation_failed() -{ -#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) - throw std::bad_alloc(); -#else - std::abort(); -#endif -} - +// LXMF keeps its public buffer aliases while sharing the platform allocator. template -class PsramAllocator -{ - public: - using value_type = T; - - PsramAllocator() noexcept = default; - - template - PsramAllocator(const PsramAllocator&) noexcept - { - } - - T* allocate(std::size_t count) - { - if (count > std::numeric_limits::max() / sizeof(T)) - { - allocation_failed(); - } - - const std::size_t bytes = count * sizeof(T); - if (bytes == 0) - { - return nullptr; - } - -#if defined(ESP_PLATFORM) - void* ptr = heap_caps_malloc(bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); -#else - void* ptr = std::malloc(bytes); -#endif - if (!ptr) - { - allocation_failed(); - } - return static_cast(ptr); - } - - void deallocate(T* ptr, std::size_t) noexcept - { - if (!ptr) - { - return; - } -#if defined(ESP_PLATFORM) - heap_caps_free(ptr); -#else - std::free(ptr); -#endif - } - - template - struct rebind - { - using other = PsramAllocator; - }; -}; - -template -bool operator==(const PsramAllocator&, const PsramAllocator&) noexcept -{ - return true; -} - -template -bool operator!=(const PsramAllocator&, const PsramAllocator&) noexcept -{ - return false; -} +using PsramAllocator = + ::platform::esp::arduino_common::memory::PsramAllocator; using RuntimeByteBuffer = std::vector>; using RuntimeByteBufferList = diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/meshtastic/node_store.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/meshtastic/node_store.h deleted file mode 100644 index 404819a9..00000000 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/meshtastic/node_store.h +++ /dev/null @@ -1,98 +0,0 @@ -/** - * @file node_store.h - * @brief Persisted NodeInfo store shell backed by a single selected persistence backend - */ - -#pragma once - -#include "chat/infra/node_store_core.h" -#include "chat/ports/i_node_blob_store.h" - -#include "freertos/FreeRTOS.h" -#include "freertos/queue.h" -#include "freertos/semphr.h" -#include "freertos/task.h" - -#include -#include - -namespace chat -{ -namespace meshtastic -{ - -class NodeStore : public contacts::INodeStore, - private contacts::INodeBlobStore -{ - public: - static void* operator new(std::size_t size); - static void operator delete(void* ptr) noexcept; - static void operator delete(void* ptr, std::size_t) noexcept; - - NodeStore(); - - void begin() override; - void applyUpdate(uint32_t node_id, const contacts::NodeUpdate& update) override; - void upsert(uint32_t node_id, const char* short_name, const char* long_name, - uint32_t now_secs, float snr = 0.0f, float rssi = 0.0f, uint8_t protocol = 0, - uint8_t role = contacts::kNodeRoleUnknown, uint8_t hops_away = 0xFF, - uint8_t hw_model = 0, uint8_t channel = 0xFF) override; - void updateProtocol(uint32_t node_id, uint8_t protocol, uint32_t now_secs) override; - void updatePosition(uint32_t node_id, const contacts::NodePosition& position) override; - bool remove(uint32_t node_id) override; - const std::vector& getEntries() const override; - void clear() override; - bool flush() override; - - private: - enum class StorageBackend : uint8_t - { - Sd, - Nvs, - }; - - enum class LoadResult : uint8_t - { - Loaded, - MissingOrInvalid, - Busy, - }; - - static constexpr const char* kPersistNodesFile = "/nodes.bin"; - static constexpr const char* kPersistNodesNs = "nodes"; - static constexpr const char* kPersistNodesKey = "node_blob"; - static constexpr const char* kPersistNodesKeyVer = "ver"; - static constexpr const char* kPersistNodesKeyCrc = "crc"; - - bool loadBlob(std::vector& out) override; - bool saveBlob(const uint8_t* data, size_t len) override; - void clearBlob() override; - - bool loadFromNvs(std::vector& out); - LoadResult loadFromSd(std::vector& out) const; - bool saveToNvs(const uint8_t* data, size_t len) const; - bool saveToSd(const uint8_t* data, size_t len) const; - bool saveToBackend(const uint8_t* data, size_t len, StorageBackend backend) const; - void ensureAsyncSaveWorker(); - bool enqueueAsyncSave(const uint8_t* data, size_t len); - bool waitForAsyncSave(TickType_t wait_ticks); - void asyncSaveLoop(); - static void asyncSaveTaskEntry(void* context); - void clearNvs() const; - - contacts::NodeStoreCore core_; - StorageBackend backend_ = StorageBackend::Nvs; - SemaphoreHandle_t async_save_mutex_ = nullptr; - QueueHandle_t async_save_queue_ = nullptr; - TaskHandle_t async_save_task_ = nullptr; - std::vector pending_save_blob_; - StorageBackend pending_save_backend_ = StorageBackend::Nvs; - uint32_t pending_save_generation_ = 0; - uint32_t completed_save_generation_ = 0; - bool async_save_pending_ = false; - bool async_save_busy_ = false; - bool async_save_failed_ = false; -}; - -} // namespace meshtastic -} // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h new file mode 100644 index 00000000..6274f7ed --- /dev/null +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h @@ -0,0 +1,92 @@ +#pragma once + +#include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" + +#include +#include + +namespace chat::storage::v2 +{ + +class FixedSlotJournalEngine final +{ + public: + enum class State : uint8_t + { + Missing = 0, + Ready, + Incompatible, + PartialTail, + IoError, + }; + + struct Inspection + { + State state = State::Missing; + uint32_t slot_count = 0; + }; + + Inspection inspect(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) const; + + bool create(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) const; + + bool append(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size, + const void* slot) const; + + bool read(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size, + uint32_t slot_index, + void* out_slot) const; + + static constexpr std::size_t headerSize(); + + private: + struct Header + { + uint32_t magic = 0; + uint16_t schema = 0; + uint8_t protocol = 0; + uint8_t kind = 0; + uint16_t slot_size = 0; + uint16_t reserved = 0; + uint32_t crc = 0; + } __attribute__((packed)); + + static constexpr uint32_t kMagic = 0x324C4E4AU; // JNL2 + + static bool validDescriptor(MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size); + static Header makeHeader(MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size); + static bool matches(const Header& header, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size); +}; + +bool replaceFileAtomically(const char* temp_path, + const char* final_path, + const char* backup_path); +bool recoverAtomicFile(const char* final_path, + const char* temp_path, + const char* backup_path); + +constexpr std::size_t FixedSlotJournalEngine::headerSize() +{ + return sizeof(Header); +} + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h new file mode 100644 index 00000000..05eb7cfb --- /dev/null +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h @@ -0,0 +1,118 @@ +#pragma once + +#include "chat/domain/chat_types.h" + +#include +#include + +namespace chat::storage::v2 +{ + +constexpr uint16_t kStorageSchemaVersion = 2; +constexpr std::size_t kMeshtasticTextMax = 233; +constexpr std::size_t kMeshCoreTextMax = 171; +constexpr std::size_t kReticulumTextMax = 255; +constexpr std::size_t kCatalogPreviewMax = 48; + +enum class JournalKind : uint8_t +{ + MessageSegment = 1, + CatalogSnapshot = 2, + CatalogDelta = 3, + ReadStateSnapshot = 4, + ReadStateDelta = 5, + StatusSnapshot = 6, + StatusDelta = 7, + ReticulumSeen = 8, + PeerSnapshot = 9, + PeerDelta = 10, + ContactSnapshot = 11, + ContactDelta = 12, +}; + +struct ChatCatalogProjection +{ + ConversationId conversation{}; + MessageId last_message_id = 0; + uint32_t message_count = 0; + uint32_t last_sequence = 0; + uint32_t last_timestamp = 0; + uint32_t unread = 0; + MessageStatus last_status = MessageStatus::Incoming; + bool deleted = false; + char preview[kCatalogPreviewMax + 1] = {}; +}; + +struct ChatReadProjection +{ + ConversationId conversation{}; + uint32_t last_read_sequence = 0; + bool deleted = false; +}; + +struct ChatStatusProjection +{ + MessageId message_id = 0; + MessageStatus status = MessageStatus::Queued; + uint32_t sequence = 0; +}; + +struct ReticulumSeenProjection +{ + uint8_t hash[kReticulumLxmfHashSize] = {}; +}; + +uint32_t crc32(const void* data, std::size_t len); + +MeshProtocol canonicalProtocol(MeshProtocol protocol); +bool supportedProtocol(MeshProtocol protocol); + +std::size_t messageSlotSize(MeshProtocol protocol); +std::size_t catalogSlotSize(MeshProtocol protocol); +std::size_t readStateSlotSize(MeshProtocol protocol); +std::size_t statusSlotSize(); +std::size_t reticulumSeenSlotSize(); + +bool encodeMessageSlot(const ChatMessage& message, + uint32_t sequence, + void* out, + std::size_t out_len); +bool decodeMessageSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatMessage& out_message, + uint32_t* out_sequence = nullptr); + +bool encodeCatalogSlot(MeshProtocol protocol, + const ChatCatalogProjection& projection, + void* out, + std::size_t out_len); +bool decodeCatalogSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatCatalogProjection& out_projection); + +bool encodeReadStateSlot(MeshProtocol protocol, + const ChatReadProjection& projection, + void* out, + std::size_t out_len); +bool decodeReadStateSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatReadProjection& out_projection); + +bool encodeStatusSlot(const ChatStatusProjection& projection, + void* out, + std::size_t out_len); +bool decodeStatusSlot(const void* data, + std::size_t len, + ChatStatusProjection& out_projection); + +bool encodeReticulumSeenSlot(const ReticulumSeenProjection& projection, + void* out, + std::size_t out_len); +bool decodeReticulumSeenSlot(const void* data, + std::size_t len, + ReticulumSeenProjection& out_projection); + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h new file mode 100644 index 00000000..8eb29343 --- /dev/null +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h @@ -0,0 +1,47 @@ +#pragma once + +#include "chat/domain/mesh_peer_directory.h" + +#include +#include + +namespace chat::storage::v2 +{ + +struct PeerProjection +{ + MeshPeerRecord record{}; + bool deleted = false; +}; + +struct ContactProjection +{ + MeshPeerIdentity identity{}; + NodeId node_id_hint = 0; + MeshPeerUserFlags flags{}; + char alias[kMeshPeerUserAliasMaxLen + 1] = {}; + bool deleted = false; +}; + +std::size_t peerSlotSize(MeshProtocol protocol); +std::size_t contactSlotSize(MeshProtocol protocol); + +bool encodePeerSlot(MeshProtocol protocol, + const PeerProjection& projection, + void* out, + std::size_t out_len); +bool decodePeerSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + PeerProjection& out_projection); + +bool encodeContactSlot(MeshProtocol protocol, + const ContactProjection& projection, + void* out, + std::size_t out_len); +bool decodeContactSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ContactProjection& out_projection); + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h new file mode 100644 index 00000000..d94c1369 --- /dev/null +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h @@ -0,0 +1,216 @@ +#pragma once + +#include "chat/infra/mesh_peer_directory_core.h" +#include "chat/ports/i_chat_store.h" +#include "chat/ports/i_protocol_peer_repository.h" +#include "platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h" +#include "platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h" +#include "platform/esp/arduino_common/memory/psram_allocator.h" + +#include +#include + +#include +#include +#include +#include + +namespace chat +{ + +class SdProtocolPeerRepository final : public IProtocolPeerRepository +{ + public: + explicit SdProtocolPeerRepository(IChatStore& chat_store); + ~SdProtocolPeerRepository() override; + + SdProtocolPeerRepository(const SdProtocolPeerRepository&) = delete; + SdProtocolPeerRepository& operator=(const SdProtocolPeerRepository&) = delete; + + MeshPeerDirectoryStatus begin() override; + MeshPeerDirectoryStatus record(const MeshPeerRecord& record) override; + MeshPeerDirectoryStatus find(const MeshPeerIdentity& identity, + MeshPeerRecord& out_record) override; + MeshPeerDirectoryStatus findByNodeId(MeshProtocol protocol, + NodeId node_id, + MeshPeerRecord& out_record) override; + MeshPeerDirectoryStatus loadRecent(MeshProtocol protocol, + MeshPeerRecord* out_records, + std::size_t max_records, + std::size_t* out_count) override; + MeshPeerDirectoryStatus search(MeshProtocol protocol, + const char* query, + MeshPeerRecord* out_records, + std::size_t max_records, + std::size_t* out_count) override; + MeshPeerDirectoryStatus setUserFlags( + const MeshPeerIdentity& identity, + const MeshPeerUserFlags& flags) override; + MeshPeerDirectoryStatus remove(const MeshPeerIdentity& identity) override; + MeshPeerDirectoryStatus clearProtocol(MeshProtocol protocol) override; + MeshPeerDirectoryCapacity capacityFor(MeshProtocol protocol) const override; + MeshPeerDirectoryStatus flush() override; + + contacts::INodeStore& nodeStoreView() override; + contacts::IContactStore& contactStoreView() override; + + private: + using PeerVector = std::vector< + MeshPeerRecord, + ::platform::esp::arduino_common::memory::PsramAllocator>; + using ContactVector = std::vector< + storage::v2::ContactProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::ContactProjection>>; + using PendingPeerVector = std::vector< + storage::v2::PeerProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::PeerProjection>>; + using PsramByteVector = std::vector< + uint8_t, + ::platform::esp::arduino_common::memory::PsramAllocator>; + + struct PartitionState + { + uint32_t peer_delta_count = 0; + uint32_t contact_delta_count = 0; + }; + + class NodeStoreView final : public contacts::INodeStore + { + public: + explicit NodeStoreView(SdProtocolPeerRepository& owner); + void setActiveProtocol(MeshProtocol protocol) override; + void begin() override; + void applyUpdate(uint32_t node_id, + const contacts::NodeUpdate& update) override; + void upsert(uint32_t node_id, + const char* short_name, + const char* long_name, + uint32_t now_secs, + float snr, + float rssi, + uint8_t protocol, + uint8_t role, + uint8_t hops_away, + uint8_t hw_model, + uint8_t channel) override; + void updateProtocol(uint32_t node_id, + uint8_t protocol, + uint32_t now_secs) override; + void updatePosition(uint32_t node_id, + const contacts::NodePosition& position) override; + bool remove(uint32_t node_id) override; + const std::vector& getEntries() const override; + void clear() override; + bool flush() override; + + private: + SdProtocolPeerRepository& owner_; + }; + + class ContactStoreView final : public contacts::IContactStore + { + public: + explicit ContactStoreView(SdProtocolPeerRepository& owner); + void setActiveProtocol(MeshProtocol protocol) override; + void begin() override; + std::string getNickname(uint32_t node_id) const override; + bool setNickname(uint32_t node_id, const char* nickname) override; + bool removeNickname(uint32_t node_id) override; + bool hasNickname(const char* nickname) const override; + std::vector getAllContactIds() const override; + size_t getCount() const override; + + private: + SdProtocolPeerRepository& owner_; + }; + + bool ensureLayout(); + bool ensureProtocolLayout(MeshProtocol protocol); + bool loadProtocol(MeshProtocol protocol); + bool loadPeerJournal(MeshProtocol protocol, const char* name); + bool loadContactJournal(MeshProtocol protocol, const char* name); + bool compactProtocolAtBoot(MeshProtocol protocol); + bool rewritePeerSnapshot(MeshProtocol protocol); + bool rewriteContactSnapshot(MeshProtocol protocol); + + bool appendPeerDelta(const storage::v2::PeerProjection& projection); + bool appendContactDelta( + const storage::v2::ContactProjection& projection); + bool queueOrAppendPeerDelta( + const storage::v2::PeerProjection& projection); + bool drainPendingPeerDeltas(std::size_t budget); + bool applyPeerProjection(const storage::v2::PeerProjection& projection); + bool applyContactProjection( + const storage::v2::ContactProjection& projection); + void overlayContactFacts(); + void overlayContactFactsForPeer(MeshPeerRecord& peer) const; + void reconcileStableIdentities(MeshProtocol protocol); + + std::size_t findPeerIndex(const MeshPeerIdentity& identity) const; + std::size_t findPeerIndexByNodeId(MeshProtocol protocol, + NodeId node_id) const; + std::size_t findContactIndex(const MeshPeerIdentity& identity) const; + bool stableContactIdentity(const MeshPeerRecord& peer, + MeshPeerIdentity& out_identity) const; + NodeId projectedNodeId(const MeshPeerRecord& peer) const; + bool peerIsProtected(const MeshPeerRecord& peer) const; + bool peerReferencedByConversation(const MeshPeerRecord& peer) const; + bool peerReferencedByConversations( + const MeshPeerRecord& peer, + const std::vector& conversations) const; + std::size_t ephemeralCount(MeshProtocol protocol) const; + bool evictOldestEphemeral(MeshProtocol protocol); + + void applyNodeUpdate(uint32_t node_id, + const contacts::NodeUpdate& update); + bool removeNode(uint32_t node_id); + void clearActiveNodes(); + const std::vector& activeNodeEntries() const; + void rebuildActiveNodeProjection() const; + bool toNodeEntry(const MeshPeerRecord& peer, + contacts::NodeEntry& out) const; + + std::string nicknameForNode(uint32_t node_id) const; + bool setNicknameForNode(uint32_t node_id, const char* nickname); + bool removeNicknameForNode(uint32_t node_id); + bool hasNicknameForActiveProtocol(const char* nickname) const; + std::vector activeContactIds() const; + size_t activeContactCount() const; + bool persistContactFacts(const MeshPeerIdentity& identity, + const MeshPeerUserFlags& flags, + const char* alias, + bool deleted); + + static MeshProtocol normalizeProtocol(MeshProtocol protocol); + static const char* protocolSlug(MeshProtocol protocol); + static std::size_t protocolIndex(MeshProtocol protocol); + static NodeId reticulumNodeId(const ReticulumPeerIdentity& identity); + static bool ensureDirectory(const char* path); + static void buildProtocolPath(MeshProtocol protocol, + const char* name, + char* out, + std::size_t out_len); + + bool lock() const; + void unlock() const; + + IChatStore& chat_store_; + storage::v2::FixedSlotJournalEngine journal_{}; + PeerVector peers_{}; + ContactVector contacts_{}; + PendingPeerVector pending_peer_deltas_{}; + std::size_t pending_peer_head_ = 0U; + PsramByteVector slot_scratch_{}; + PartitionState partitions_[3]{}; + mutable std::vector active_node_projection_{}; + mutable bool active_node_projection_dirty_ = true; + MeshProtocol active_protocol_ = MeshProtocol::Meshtastic; + bool begun_ = false; + mutable SemaphoreHandle_t mutex_ = nullptr; + NodeStoreView node_store_view_; + ContactStoreView contact_store_view_; +}; + +} // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h index d86e987c..16c68124 100644 --- a/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h @@ -1,33 +1,33 @@ /** * @file sd_store.h - * @brief ESP Arduino chat store backed by the mounted SD card. + * @brief Protocol-partitioned append-only ESP chat storage. */ #pragma once #include "chat/ports/i_chat_store.h" +#include "platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h" +#include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" +#include "platform/esp/arduino_common/memory/psram_allocator.h" +#include +#include #include -namespace platform::esp::arduino_common::storage -{ -class SdRuntimeFile; -} // namespace platform::esp::arduino_common::storage - namespace chat { class SdStore final : public IChatStore { public: - static constexpr const char* kDir = "/chat"; - static constexpr const char* kIndexFile = "/chat/index.bin"; - static constexpr const char* kReadStateFile = "/chat/read_state.bin"; - static constexpr const char* kReticulumLxmfSeenFile = "/chat/lxmf_seen.bin"; - static constexpr size_t kMaxMessagesPerConv = 1000; - static constexpr size_t kMaxReticulumLxmfSeen = 1024; - static constexpr size_t kMaxTextLen = 233; - static constexpr size_t kPreviewLen = 48; + static constexpr const char* kRoot = "/data/v2"; + static constexpr const char* kMeshtasticRoot = "/data/v2/mt/chat"; + static constexpr const char* kMeshCoreRoot = "/data/v2/mc/chat"; + static constexpr const char* kReticulumRoot = "/data/v2/rt/chat"; + static constexpr std::size_t kMessageSegmentBytes = 128U * 1024U; + static constexpr std::size_t kCatalogPreviewLen = + storage::v2::kCatalogPreviewMax; + static constexpr std::size_t kSeenHotCapacity = 4096; SdStore(); ~SdStore() override = default; @@ -35,20 +35,30 @@ class SdStore final : public IChatStore bool isReady() const { return ready_; } void append(const ChatMessage& msg) override; + bool appendDurably(const ChatMessage& msg) override; bool appendIncomingDurably(const ChatMessage& msg) override; - std::vector loadRecent(const ConversationId& conv, size_t n) override; - std::vector loadPageFromLatest(const ConversationId& conv, - size_t offset_from_latest, - size_t limit, - size_t* total) override; - std::vector loadConversationPage(size_t offset, - size_t limit, - size_t* total) override; + std::vector loadRecent(const ConversationId& conv, + std::size_t n) override; + std::vector loadPageFromLatest( + const ConversationId& conv, + std::size_t offset_from_latest, + std::size_t limit, + std::size_t* total) override; + std::vector loadConversationPage( + std::size_t offset, + std::size_t limit, + std::size_t* total) override; + std::vector loadConversationPageForProtocol( + MeshProtocol protocol, + std::size_t offset, + std::size_t limit, + std::size_t* total) override; bool setUnread(const ConversationId& conv, int unread) override; int getUnread(const ConversationId& conv) const override; void clearConversation(const ConversationId& conv) override; void clearAll() override; - bool updateMessageStatus(MessageId msg_id, MessageStatus status) override; + bool updateMessageStatus(MessageId msg_id, + MessageStatus status) override; bool updateMessageStatusForProtocol(MessageId msg_id, MeshProtocol protocol, MessageStatus status) override; @@ -56,245 +66,153 @@ class SdStore final : public IChatStore bool getMessageForProtocol(MessageId msg_id, MeshProtocol protocol, ChatMessage* out) const override; - bool hasReticulumLxmfMessageHash(const uint8_t* lxmf_hash) const override; + bool hasReticulumLxmfMessageHash( + const uint8_t* lxmf_hash) const override; void flush() override; private: - struct FileHeader + using CatalogList = std::vector< + storage::v2::ChatCatalogProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::ChatCatalogProjection>>; + using ReadStateList = std::vector< + storage::v2::ChatReadProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::ChatReadProjection>>; + struct ProtocolStatusProjection { - uint32_t magic = 0; - uint16_t version = 0; - uint16_t head = 0; - uint16_t count = 0; - uint16_t reserved = 0; - } __attribute__((packed)); + MeshProtocol protocol = MeshProtocol::Meshtastic; + storage::v2::ChatStatusProjection value{}; + }; - struct RecordV2 - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint16_t text_len = 0; - uint32_t from = 0; - uint32_t peer = 0; - uint32_t msg_id = 0; - uint32_t timestamp = 0; - char text[kMaxTextLen] = {}; - } __attribute__((packed)); + using StatusList = std::vector< + ProtocolStatusProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + ProtocolStatusProjection>>; + using SeenList = std::vector< + storage::v2::ReticulumSeenProjection, + ::platform::esp::arduino_common::memory::PsramAllocator< + storage::v2::ReticulumSeenProjection>>; + using ScratchBuffer = std::vector< + uint8_t, + ::platform::esp::arduino_common::memory::PsramAllocator>; - struct RecordV3 - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint8_t flags = 0; - uint16_t text_len = 0; - uint32_t from = 0; - uint32_t peer = 0; - uint32_t msg_id = 0; - uint32_t timestamp = 0; - uint8_t reticulum_destination_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_identity_hash[kReticulumPeerHashSize] = {}; - char text[kMaxTextLen] = {}; - } __attribute__((packed)); + static constexpr std::size_t kScratchCapacity = 512; + static constexpr std::size_t kCatalogCompactThreshold = 512; + static constexpr std::size_t kReadCompactThreshold = 256; + static constexpr std::size_t kStatusCompactThreshold = 2048; + static constexpr uint32_t kProjectionRetryIntervalMs = 5000U; - struct RecordV4 - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint8_t flags = 0; - uint8_t rx_origin = 0; - uint16_t text_len = 0; - uint32_t from = 0; - uint32_t peer = 0; - uint32_t msg_id = 0; - uint32_t timestamp = 0; - uint8_t reticulum_destination_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_identity_hash[kReticulumPeerHashSize] = {}; - char text[kMaxTextLen] = {}; - } __attribute__((packed)); + bool appendInternal(const ChatMessage& msg, bool incoming_commit); + bool ensureLayout() const; + bool ensureProtocolLayout(MeshProtocol protocol) const; + bool loadRuntimeState(); + bool loadProtocolState(MeshProtocol protocol); + bool loadCatalogJournal(MeshProtocol protocol, const char* name); + bool loadReadJournal(MeshProtocol protocol, const char* name); + bool loadStatusJournal(MeshProtocol protocol, const char* name); + bool loadSeenJournal(); + bool rebuildSeenJournalFromMessages(); + bool recoverProjectionSnapshot(MeshProtocol protocol, + const char* base_name); + bool reconcileProtocolCatalog(MeshProtocol protocol); + bool reconcileConversationDirectory(MeshProtocol protocol, + const char* directory_name); - struct Record - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint8_t flags = 0; - uint8_t rx_origin = 0; - uint16_t text_len = 0; - uint32_t from = 0; - uint32_t peer = 0; - uint32_t msg_id = 0; - uint32_t timestamp = 0; - uint8_t reticulum_destination_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_identity_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_lxmf_hash[kReticulumLxmfHashSize] = {}; - char text[kMaxTextLen] = {}; - } __attribute__((packed)); + std::size_t slotsPerMessageSegment(MeshProtocol protocol) const; + uint32_t messageCountOnDisk(const ConversationId& conv) const; + bool readMessageByOrdinal(const ConversationId& conv, + uint32_t ordinal, + ChatMessage& out_message, + uint32_t* out_sequence = nullptr) const; + bool latestStoredMessage(const ConversationId& conv, + ChatMessage& out_message, + uint32_t* out_count = nullptr) const; + bool storedMessageMatches(const ChatMessage& message, + bool* out_matches, + uint32_t* out_count) const; + bool appendMessageRecord(const ChatMessage& message, + uint32_t sequence); + bool repairPartialJournal(const char* path, + MeshProtocol protocol, + storage::v2::JournalKind kind, + std::size_t slot_size) const; - struct IndexHeader - { - uint32_t magic = 0; - uint16_t version = 0; - uint16_t count = 0; - } __attribute__((packed)); + bool appendCatalogProjection( + const storage::v2::ChatCatalogProjection& projection); + bool appendReadProjection( + const storage::v2::ChatReadProjection& projection); + bool appendStatusProjection(MeshProtocol protocol, + const storage::v2::ChatStatusProjection& projection); + bool appendSeenProjection( + const storage::v2::ReticulumSeenProjection& projection) const; + bool rememberReticulumHash(const uint8_t* hash); - struct IndexEntryV2 - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint16_t unread = 0; - uint32_t peer = 0; - uint32_t last_msg_id = 0; - uint32_t last_timestamp = 0; - uint32_t last_from = 0; - uint16_t preview_len = 0; - char preview[kPreviewLen] = {}; - } __attribute__((packed)); + storage::v2::ChatCatalogProjection* findCatalog( + const ConversationId& conversation); + const storage::v2::ChatCatalogProjection* findCatalog( + const ConversationId& conversation) const; + storage::v2::ChatReadProjection* findReadState( + const ConversationId& conversation); + const storage::v2::ChatReadProjection* findReadState( + const ConversationId& conversation) const; + storage::v2::ChatStatusProjection* findStatus(MessageId message_id, + MeshProtocol protocol); + const storage::v2::ChatStatusProjection* findStatus( + MessageId message_id, + MeshProtocol protocol) const; + void applyStoredStatus(ChatMessage& message) const; + uint32_t countUnreadAfter(const ConversationId& conversation, + uint32_t last_read_sequence) const; + uint32_t sequenceForUnread(const ConversationId& conversation, + uint32_t unread) const; - struct IndexEntry - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t status = 0; - uint8_t flags = 0; - uint16_t unread = 0; - uint32_t peer = 0; - uint32_t last_msg_id = 0; - uint32_t last_timestamp = 0; - uint32_t last_from = 0; - uint16_t preview_len = 0; - uint8_t reticulum_destination_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_identity_hash[kReticulumPeerHashSize] = {}; - char preview[kPreviewLen] = {}; - } __attribute__((packed)); + bool rewriteCatalogSnapshot(MeshProtocol protocol); + bool rewriteReadSnapshot(MeshProtocol protocol); + bool rewriteStatusSnapshot(MeshProtocol protocol); + bool compactProtocolProjections(MeshProtocol protocol); + bool rewriteJournalFromCatalog(MeshProtocol protocol, + const char* final_path, + const char* temp_path); + bool rewriteJournalFromReadState(MeshProtocol protocol, + const char* final_path, + const char* temp_path); + bool rewriteJournalFromStatus(MeshProtocol protocol, + const char* final_path, + const char* temp_path); - struct ReadStateHeader - { - uint32_t magic = 0; - uint16_t version = 0; - uint16_t count = 0; - } __attribute__((packed)); - - struct ReadStateEntry - { - uint8_t protocol = 0; - uint8_t channel = 0; - uint8_t flags = 0; - uint8_t reserved = 0; - uint16_t unread = 0; - uint16_t reserved2 = 0; - uint32_t peer = 0; - uint8_t reticulum_destination_hash[kReticulumPeerHashSize] = {}; - uint8_t reticulum_identity_hash[kReticulumPeerHashSize] = {}; - } __attribute__((packed)); - - struct LxmfSeenHeader - { - uint32_t magic = 0; - uint16_t version = 0; - uint16_t head = 0; - uint16_t count = 0; - } __attribute__((packed)); - - struct LxmfSeenRecord - { - uint8_t hash[kReticulumLxmfHashSize] = {}; - } __attribute__((packed)); - - static constexpr uint32_t kFileMagic = 0x474F4C43; // "CLOG" - static constexpr uint32_t kIndexMagic = 0x54414843; // "CHAT" - static constexpr uint32_t kReadStateMagic = 0x44525343; // "CSRD" - static constexpr uint32_t kLxmfSeenMagic = 0x4E45584C; // "LXEN" - static constexpr uint16_t kLegacyVersion = 2; - static constexpr uint16_t kReticulumIdentityVersion = 3; - static constexpr uint16_t kRxOriginVersion = 4; - static constexpr uint16_t kFileVersion = 5; - static constexpr uint16_t kIndexVersion = 3; - static constexpr uint16_t kReadStateVersion = 1; - static constexpr uint16_t kLxmfSeenVersion = 1; - - bool ensureFs() const; - bool ensureDir() const; - bool recoverIndex() const; - bool reconcileIndexUnread(std::vector& entries) const; - bool readIndex(std::vector& entries) const; - bool writeIndex(const std::vector& entries) const; - bool ensureIndex(std::vector& entries); - bool readReadState(std::vector& entries) const; - bool writeReadState(const std::vector& entries) const; - bool findReadStateEntry(const ConversationId& conv, - std::vector& entries, - size_t* out_idx) const; - bool findReadStateEntry(const ConversationId& conv, - const std::vector& entries, - size_t* out_idx) const; - bool readStateUnreadOnly(const ConversationId& conv, uint16_t* unread) const; - bool readStateUnreadOrLegacy(const ConversationId& conv, uint16_t* unread) const; - bool writeReadStateUnread(const ConversationId& conv, uint16_t unread) const; - bool removeReadStateEntry(const ConversationId& conv) const; - bool findIndexEntry(const ConversationId& conv, - std::vector& entries, - size_t* out_idx) const; - bool findIndexEntry(const ConversationId& conv, - const std::vector& entries, - size_t* out_idx) const; - void updateIndexForMessage(const ChatMessage& msg, uint16_t unread); - void rebuildIndex(); - bool loadFileHeader(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - FileHeader& header) const; - bool initFileHeader(::platform::esp::arduino_common::storage::SdRuntimeFile& file) const; - bool upgradeConversationFile(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - const char* path, - FileHeader& header) const; - bool readRecord(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - const FileHeader& header, - uint16_t slot, - Record& rec) const; - bool writeRecord(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - uint16_t slot, - const Record& rec) const; - bool initLxmfSeenFile(::platform::esp::arduino_common::storage::SdRuntimeFile& file) const; - bool loadLxmfSeenHeader(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - LxmfSeenHeader& header) const; - bool readLxmfSeenRecord(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - uint16_t slot, - LxmfSeenRecord& rec) const; - bool writeLxmfSeenRecord(::platform::esp::arduino_common::storage::SdRuntimeFile& file, - uint16_t slot, - const LxmfSeenRecord& rec) const; - bool rememberReticulumLxmfMessageHash(const uint8_t* lxmf_hash) const; - bool conversationContainsReticulumLxmfHash( - ::platform::esp::arduino_common::storage::SdRuntimeFile& file, - const FileHeader& header, - const uint8_t* lxmf_hash, - bool* found) const; - bool appendInternal(const ChatMessage& msg); - bool openConversationForUpdate(const ConversationId& conv, - ::platform::esp::arduino_common::storage::SdRuntimeFile& file, - FileHeader& header) const; - bool readConversationUnread(const ConversationId& conv, uint16_t* unread) const; - bool writeConversationUnread(const ConversationId& conv, uint16_t unread) const; - void buildConversationPath(const ConversationId& conv, char* out, size_t out_len) const; - const char* channelName(ChannelId channel) const; - static ChatMessage messageFromRecord(const Record& rec); - static Record recordFromMessage(const ChatMessage& msg); - static bool indexEntryHasReticulumIdentity(const IndexEntry& entry); - static bool indexEntryMatchesConversation(const IndexEntry& entry, - const ConversationId& conv); - static ConversationId conversationFromIndexEntry(const IndexEntry& entry); - static ConversationMeta metaFromIndexEntry(const IndexEntry& entry); - static bool readStateEntryHasReticulumIdentity(const ReadStateEntry& entry); - static bool readStateEntryMatchesConversation(const ReadStateEntry& entry, - const ConversationId& conv); - static ReadStateEntry readStateEntryFromConversation(const ConversationId& conv, - uint16_t unread); - static bool hasLogSuffix(const char* name); + static MeshProtocol normalizeProtocol(MeshProtocol protocol); + static const char* protocolRoot(MeshProtocol protocol); + static const char* protocolSlug(MeshProtocol protocol); + static bool sameProtocol(MeshProtocol lhs, MeshProtocol rhs); + static bool sameStoredMessage(const ChatMessage& lhs, + const ChatMessage& rhs); + static ConversationMeta makeMeta( + const storage::v2::ChatCatalogProjection& projection); + static void buildConversationDirectory(const ConversationId& conversation, + char* out, + std::size_t out_len); + static void buildMessageSegmentPath(const ConversationId& conversation, + uint32_t segment, + char* out, + std::size_t out_len); + static void buildProjectionPath(MeshProtocol protocol, + const char* name, + char* out, + std::size_t out_len); + static bool ensureDirectory(const char* path); + static bool removeTree(const char* path); + storage::v2::FixedSlotJournalEngine journal_{}; + CatalogList catalog_{}; + ReadStateList read_state_{}; + StatusList statuses_{}; + mutable SeenList seen_hot_{}; + mutable ScratchBuffer scratch_{}; + bool projection_dirty_[3] = {}; + uint8_t flush_protocol_cursor_ = 0; + uint32_t last_projection_retry_ms_ = 0; bool ready_ = false; - bool unread_reconcile_pending_ = false; }; } // namespace chat diff --git a/platform/esp/arduino_common/include/platform/esp/arduino_common/memory/psram_allocator.h b/platform/esp/arduino_common/include/platform/esp/arduino_common/memory/psram_allocator.h new file mode 100644 index 00000000..8e3cd2c6 --- /dev/null +++ b/platform/esp/arduino_common/include/platform/esp/arduino_common/memory/psram_allocator.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include + +#if defined(ESP_PLATFORM) +#include +#endif + +namespace platform::esp::arduino_common::memory +{ + +[[noreturn]] inline void psramAllocationFailed() +{ +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) + throw std::bad_alloc(); +#else + std::abort(); +#endif +} + +template +class PsramAllocator +{ + public: + using value_type = T; + using is_always_equal = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + + PsramAllocator() noexcept = default; + + template + PsramAllocator(const PsramAllocator&) noexcept + { + } + + T* allocate(std::size_t count) + { + if (count > std::numeric_limits::max() / sizeof(T)) + { + psramAllocationFailed(); + } + const std::size_t bytes = count * sizeof(T); + if (bytes == 0) + { + return nullptr; + } +#if defined(ESP_PLATFORM) + void* ptr = heap_caps_malloc(bytes, + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); +#else + void* ptr = std::malloc(bytes); +#endif + if (!ptr) + { + psramAllocationFailed(); + } + return static_cast(ptr); + } + + void deallocate(T* ptr, std::size_t) noexcept + { + if (!ptr) + { + return; + } +#if defined(ESP_PLATFORM) + heap_caps_free(ptr); +#else + std::free(ptr); +#endif + } + + template + struct rebind + { + using other = PsramAllocator; + }; +}; + +template +bool operator==(const PsramAllocator&, const PsramAllocator&) noexcept +{ + return true; +} + +template +bool operator!=(const PsramAllocator&, const PsramAllocator&) noexcept +{ + return false; +} + +} // namespace platform::esp::arduino_common::memory diff --git a/platform/esp/arduino_common/src/app_context.cpp b/platform/esp/arduino_common/src/app_context.cpp index cf934a94..7981b9c5 100644 --- a/platform/esp/arduino_common/src/app_context.cpp +++ b/platform/esp/arduino_common/src/app_context.cpp @@ -16,6 +16,7 @@ #include "chat/runtime/self_identity_policy.h" #include "platform/esp/arduino_common/app_tasks.h" #include "platform/esp/arduino_common/memory_diag.h" +#include "platform/ui/reticulum_call_runtime.h" #include "platform/ui/reticulum_directory_runtime.h" #include "platform/ui/reticulum_group_config_runtime.h" #include "sys/event_bus.h" @@ -222,16 +223,27 @@ void AppContext::initContactServices() return; } - auto contact_services = platform_bindings_.create_contact_services(); + if (!mesh_peer_directory_) + { + Serial.printf("[AppContext] protocol peer repository unavailable\n"); + return; + } + + auto contact_services = + platform_bindings_.create_contact_services(*mesh_peer_directory_); if (!contact_services.isValid()) { Serial.printf("[AppContext] contact service bundle invalid\n"); return; } - node_store_ = std::move(contact_services.node_store); - contact_store_ = std::move(contact_services.contact_store); + node_store_ = &mesh_peer_directory_->nodeStoreView(); + contact_store_ = &mesh_peer_directory_->contactStoreView(); contact_service_ = std::move(contact_services.service); + if (contact_service_) + { + contact_service_->setActiveProtocol(config_.mesh_protocol); + } } chat::IMeshAdapter* AppContext::getMeshAdapter() @@ -582,6 +594,10 @@ bool AppContext::switchMeshProtocol(chat::MeshProtocol protocol, bool persist) { chat_service_->setActiveProtocol(normalized); } + if (contact_service_) + { + contact_service_->setActiveProtocol(normalized); + } if (persist) { @@ -629,6 +645,11 @@ void AppContext::updateCoreServices() { event_runtime_hooks_.update_core_services(*this); } + if (mesh_peer_directory_ && + !::platform::ui::reticulum_call::resource_preempt_active()) + { + (void)mesh_peer_directory_->flush(); + } } void AppContext::tickEventRuntime() diff --git a/platform/esp/arduino_common/src/app_context_platform_bindings.cpp b/platform/esp/arduino_common/src/app_context_platform_bindings.cpp index 98231170..2ed44502 100644 --- a/platform/esp/arduino_common/src/app_context_platform_bindings.cpp +++ b/platform/esp/arduino_common/src/app_context_platform_bindings.cpp @@ -6,26 +6,22 @@ #include "app/app_facades.h" #include "board/GpsBoard.h" #include "board/MotionBoard.h" -#include "chat/infra/mesh_peer_directory_core.h" #include "chat/infra/store/ram_store.h" #include "chat/usecase/contact_service.h" #include "platform/esp/arduino_common/chat/infra/chat_event_bus_bridge.h" -#include "platform/esp/arduino_common/chat/infra/contact_store.h" #include "platform/esp/arduino_common/chat/infra/mesh_adapter_router.h" -#include "platform/esp/arduino_common/chat/infra/meshtastic/node_store.h" #include "platform/esp/arduino_common/chat/infra/protocol_factory.h" +#include "platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h" #include "platform/esp/arduino_common/chat/infra/store/sd_store.h" #include "platform/esp/arduino_common/device_identity.h" #include "platform/esp/arduino_common/gps/gps_service.h" #include "platform/esp/arduino_common/gps/track_recorder.h" -#include "platform/esp/arduino_common/storage/persistence_bus_gate.h" #include "platform/esp/arduino_common/storage/sd_card_runtime.h" #include "platform/esp/arduino_common/team/crypto/team_crypto.h" #include "platform/esp/arduino_common/team/event/team_app_data_event_bus_bridge.h" #include "platform/esp/arduino_common/team/event/team_event_bus_sink.h" #include "platform/esp/arduino_common/team/event/team_pairing_event_bus_sink.h" #include "platform/esp/arduino_common/team_platform_bundle.h" -#include "platform/esp/common/shared_spi_bus_arbiter.h" #include "platform/ui/team_ui_store_runtime.h" #include "team/usecase/team_controller.h" #include "team/usecase/team_track_sampler.h" @@ -34,37 +30,10 @@ #include "freertos/FreeRTOS.h" #include -#include namespace { -constexpr uint32_t kMeshPeerDirectorySdWaitMs = 50; -constexpr const char* kMeshPeerDirectoryDir = "/mesh"; -constexpr const char* kMeshPeerDirectoryPath = "/mesh/peers.bin"; -constexpr const char* kMeshPeerDirectoryTempPath = "/mesh/peers.tmp"; -constexpr std::size_t kMeshPeerDirectoryMaxBlobBytes = 768U * 1024U; -constexpr uint32_t kMeshPeerDirectoryBusResource = 5; -constexpr uint32_t kMeshPeerDirectoryBusOwnerId = 0x4D504452u; // 'MPDR' - -::platform::esp::common::SharedSpiBusAdapter s_mesh_peer_directory_load_adapter( - "mesh_peer_dir_load", - kMeshPeerDirectoryBusOwnerId); -::platform::esp::common::SharedSpiBusAdapter s_mesh_peer_directory_save_adapter( - "mesh_peer_dir_save", - kMeshPeerDirectoryBusOwnerId); -::platform::esp::common::FixedSharedSpiBusPolicyStrategy s_mesh_peer_directory_bus_policy( - kMeshPeerDirectorySdWaitMs, - kMeshPeerDirectorySdWaitMs, - kMeshPeerDirectorySdWaitMs, - kMeshPeerDirectorySdWaitMs); -sys::runtime::StorageBusArbiter s_mesh_peer_directory_load_arbiter( - s_mesh_peer_directory_load_adapter, - s_mesh_peer_directory_bus_policy); -sys::runtime::StorageBusArbiter s_mesh_peer_directory_save_arbiter( - s_mesh_peer_directory_save_adapter, - s_mesh_peer_directory_bus_policy); - gps::GpsReceiverInitConfig make_receiver_init_config(const app::AppConfig& config) { gps::GpsReceiverInitConfig init{}; @@ -151,7 +120,7 @@ std::unique_ptr create_chat_store() std::unique_ptr sd_store(new chat::SdStore()); if (sd_store && sd_store->isReady()) { - Serial.printf("[AppContext] chat store=SdStore backend=%s layout=/chat/index.bin+/chat/*.log\n", + Serial.printf("[AppContext] chat store=SdStore backend=%s layout=/data/v2/{mt,mc,rt}/chat\n", ::platform::esp::arduino_common::storage::sd_card_backend_name()); return std::unique_ptr(sd_store.release()); } @@ -163,237 +132,19 @@ std::unique_ptr create_chat_store() return std::unique_ptr(new chat::RamStore()); } -bool ensure_mesh_peer_directory_dir() +std::unique_ptr create_mesh_peer_directory( + chat::IChatStore& chat_store) { - using namespace ::platform::esp::arduino_common::storage; - return sd_exists(kMeshPeerDirectoryDir) || sd_mkdir(kMeshPeerDirectoryDir); -} - -class EspSdMeshPeerDirectoryBlobStore final - : public chat::IMeshPeerDirectoryBlobStore -{ - public: - chat::MeshPeerDirectoryBlobLoadResult loadBlob( - std::vector& out) override + std::unique_ptr repository( + new (std::nothrow) chat::SdProtocolPeerRepository(chat_store)); + if (!repository) { - using namespace ::platform::esp::arduino_common::storage; - out.clear(); - if (!sd_card_ready()) - { - return chat::MeshPeerDirectoryBlobLoadResult::Unavailable; - } - if (!sd_exists(kMeshPeerDirectoryPath)) - { - return chat::MeshPeerDirectoryBlobLoadResult::Missing; - } - - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_mesh_peer_directory_load_arbiter, - sys::runtime::BusAccessPolicy::BackgroundWorkerBounded, - kMeshPeerDirectorySdWaitMs, - kMeshPeerDirectoryBusResource, - kMeshPeerDirectoryBusOwnerId + 1, - kMeshPeerDirectoryBusOwnerId); - if (!bus_gate.locked()) - { - return chat::MeshPeerDirectoryBlobLoadResult::Unavailable; - } - - SdRuntimeFile file; - if (!file.open(kMeshPeerDirectoryPath, "r")) - { - return chat::MeshPeerDirectoryBlobLoadResult::IoError; - } - const uint64_t size = file.size(); - if (size == 0 || size > kMeshPeerDirectoryMaxBlobBytes) - { - file.close(); - return chat::MeshPeerDirectoryBlobLoadResult::IoError; - } - out.resize(static_cast(size)); - const int read = file.read(out.data(), out.size()); - file.close(); - if (read < 0 || static_cast(read) != out.size()) - { - out.clear(); - return chat::MeshPeerDirectoryBlobLoadResult::IoError; - } - return chat::MeshPeerDirectoryBlobLoadResult::Loaded; + return repository; } - - bool saveBlob(const uint8_t* data, std::size_t len) override - { - using namespace ::platform::esp::arduino_common::storage; - if (!sd_card_ready() || (!data && len > 0) || - len > kMeshPeerDirectoryMaxBlobBytes || !ensure_mesh_peer_directory_dir()) - { - return false; - } - - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_mesh_peer_directory_save_arbiter, - sys::runtime::BusAccessPolicy::DurableCommit, - kMeshPeerDirectorySdWaitMs, - kMeshPeerDirectoryBusResource, - kMeshPeerDirectoryBusOwnerId + 2, - kMeshPeerDirectoryBusOwnerId); - if (!bus_gate.locked()) - { - return false; - } - - if (sd_exists(kMeshPeerDirectoryTempPath)) - { - sd_remove(kMeshPeerDirectoryTempPath); - } - - SdRuntimeFile file; - if (!file.open(kMeshPeerDirectoryTempPath, "w")) - { - return false; - } - bool ok = true; - if (len > 0) - { - ok = file.write(data, len) == len; - } - file.close(); - if (!ok) - { - sd_remove(kMeshPeerDirectoryTempPath); - return false; - } - - if (sd_exists(kMeshPeerDirectoryPath)) - { - sd_remove(kMeshPeerDirectoryPath); - } - if (!sd_rename(kMeshPeerDirectoryTempPath, kMeshPeerDirectoryPath)) - { - sd_remove(kMeshPeerDirectoryTempPath); - return false; - } - return true; - } - - void clearBlob() override - { - using namespace ::platform::esp::arduino_common::storage; - if (!sd_card_ready()) - { - return; - } - if (sd_exists(kMeshPeerDirectoryTempPath)) - { - sd_remove(kMeshPeerDirectoryTempPath); - } - if (sd_exists(kMeshPeerDirectoryPath)) - { - sd_remove(kMeshPeerDirectoryPath); - } - } -}; - -class EspSdMeshPeerDirectory final : public chat::IMeshPeerDirectory -{ - public: - EspSdMeshPeerDirectory() - : core_(blob_store_) - { - } - - chat::MeshPeerDirectoryStatus begin() override - { - return core_.begin(); - } - - chat::MeshPeerDirectoryStatus record( - const chat::MeshPeerRecord& record) override - { - return core_.record(record); - } - - chat::MeshPeerDirectoryStatus find( - const chat::MeshPeerIdentity& identity, - chat::MeshPeerRecord& out_record) override - { - return core_.find(identity, out_record); - } - - chat::MeshPeerDirectoryStatus findByNodeId( - chat::MeshProtocol protocol, - chat::NodeId node_id, - chat::MeshPeerRecord& out_record) override - { - return core_.findByNodeId(protocol, node_id, out_record); - } - - chat::MeshPeerDirectoryStatus loadRecent( - chat::MeshProtocol protocol, - chat::MeshPeerRecord* out_records, - std::size_t max_records, - std::size_t* out_count) override - { - return core_.loadRecent(protocol, out_records, max_records, out_count); - } - - chat::MeshPeerDirectoryStatus search(chat::MeshProtocol protocol, - const char* query, - chat::MeshPeerRecord* out_records, - std::size_t max_records, - std::size_t* out_count) override - { - return core_.search(protocol, query, out_records, max_records, out_count); - } - - chat::MeshPeerDirectoryStatus setUserFlags( - const chat::MeshPeerIdentity& identity, - const chat::MeshPeerUserFlags& flags) override - { - return core_.setUserFlags(identity, flags); - } - - chat::MeshPeerDirectoryStatus remove( - const chat::MeshPeerIdentity& identity) override - { - return core_.remove(identity); - } - - chat::MeshPeerDirectoryStatus clearProtocol( - chat::MeshProtocol protocol) override - { - return core_.clearProtocol(protocol); - } - - chat::MeshPeerDirectoryCapacity capacityFor( - chat::MeshProtocol protocol) const override - { - return core_.capacityFor(protocol); - } - - chat::MeshPeerDirectoryStatus flush() override - { - return core_.flush(); - } - - private: - EspSdMeshPeerDirectoryBlobStore blob_store_; - chat::MeshPeerDirectoryCore core_; -}; - -std::unique_ptr create_mesh_peer_directory() -{ - std::unique_ptr directory( - new (std::nothrow) EspSdMeshPeerDirectory()); - if (!directory) - { - return directory; - } - const auto status = directory->begin(); - Serial.printf("[MeshPeerDirectory] backend=sd path=%s status=%u\n", - kMeshPeerDirectoryPath, + const auto status = repository->begin(); + Serial.printf("[PeerStoreV2] backend=sd root=/data/v2 status=%u\n", static_cast(status.code)); - return directory; + return repository; } std::unique_ptr create_mesh_backend(chat::MeshProtocol protocol, @@ -403,24 +154,21 @@ std::unique_ptr create_mesh_backend(chat::MeshProtocol proto return chat::ProtocolFactory::createAdapter(protocol, lora_board, peer_directory); } -app::ContactServicesBundle create_contact_services() +app::ContactServicesBundle create_contact_services( + chat::IProtocolPeerRepository& repository) { app::ContactServicesBundle bundle; - bundle.node_store = std::unique_ptr(new chat::meshtastic::NodeStore()); - bundle.contact_store = std::unique_ptr(new chat::contacts::ContactStore()); - if (!bundle.node_store || !bundle.contact_store) - { - return bundle; - } - bundle.service = std::unique_ptr( - new chat::contacts::ContactService(*bundle.node_store, *bundle.contact_store)); + new chat::contacts::ContactService(repository.nodeStoreView(), + repository.contactStoreView())); if (bundle.service) { bundle.service->begin(); Serial.printf("[ContactService] startup nodes=%u nicknames=%u\n", - static_cast(bundle.node_store->getEntries().size()), - static_cast(bundle.contact_store->getCount())); + static_cast( + repository.nodeStoreView().getEntries().size()), + static_cast( + repository.contactStoreView().getCount())); } return bundle; } @@ -445,7 +193,11 @@ app::ChatServicesBundle create_chat_services(const app::AppConfig& config, bundle.model->setPolicy(config.chat_policy); bundle.store = create_chat_store(); - bundle.mesh_peer_directory = create_mesh_peer_directory(); + if (!bundle.store) + { + return bundle; + } + bundle.mesh_peer_directory = create_mesh_peer_directory(*bundle.store); bundle.mesh_runtime = create_mesh_runtime(); if (!bundle.store || !bundle.mesh_peer_directory || !bundle.mesh_runtime) { diff --git a/platform/esp/arduino_common/src/chat/infra/contact_store.cpp b/platform/esp/arduino_common/src/chat/infra/contact_store.cpp deleted file mode 100644 index 7443472b..00000000 --- a/platform/esp/arduino_common/src/chat/infra/contact_store.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/** - * @file contact_store.cpp - * @brief Contact nickname storage shell implementation - */ - -#include "platform/esp/arduino_common/chat/infra/contact_store.h" -#include "../internal/blob_store_io.h" -#include "platform/esp/arduino_common/storage/persistence_bus_gate.h" -#include "platform/esp/arduino_common/storage/sd_card_runtime.h" -#include "platform/esp/common/shared_spi_bus_arbiter.h" - -#include -#include -#include - -namespace chat -{ -namespace contacts -{ - -#ifndef CONTACT_STORE_LOG_ENABLE -#define CONTACT_STORE_LOG_ENABLE 1 -#endif - -#if CONTACT_STORE_LOG_ENABLE -#define CONTACT_STORE_LOG(...) Serial.printf(__VA_ARGS__) -#else -#define CONTACT_STORE_LOG(...) -#endif - -void* ContactStore::operator new(std::size_t size) -{ - void* ptr = heap_caps_malloc_prefer(size, - 2, - MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT, - MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - return ptr != nullptr ? ptr : ::operator new(size); -} - -void ContactStore::operator delete(void* ptr) noexcept -{ - heap_caps_free(ptr); -} - -void ContactStore::operator delete(void* ptr, std::size_t) noexcept -{ - operator delete(ptr); -} - -namespace -{ -constexpr uint32_t kSdLoadWaitMs = 250; -constexpr uint32_t kSdPersistWaitMs = 750; -constexpr uint32_t kContactStoreBusResource = 5; -constexpr uint32_t kContactStoreBusOwnerId = 0x434F4E54u; // 'CONT' -constexpr const char* kContactStoreBusOwner = "contact_store_sd"; - -::platform::esp::common::SharedSpiBusAdapter s_contact_store_bus_adapter( - kContactStoreBusOwner, - kContactStoreBusOwnerId); -::platform::esp::common::FixedSharedSpiBusPolicyStrategy s_contact_store_bus_policy( - kSdLoadWaitMs, - kSdLoadWaitMs, - kSdLoadWaitMs, - kSdLoadWaitMs); -sys::runtime::StorageBusArbiter s_contact_store_bus_arbiter( - s_contact_store_bus_adapter, - s_contact_store_bus_policy); -} // namespace - -ContactStore::ContactStore() - : core_(*this) -{ -} - -void ContactStore::begin() -{ - backend_ = ::platform::esp::arduino_common::storage::sd_card_ready() - ? StorageBackend::Sd - : StorageBackend::Flash; - CONTACT_STORE_LOG("[ContactStore] backend=%s\n", - backend_ == StorageBackend::Sd ? "sd" : "flash"); - core_.begin(); -} - -std::string ContactStore::getNickname(uint32_t node_id) const -{ - return core_.getNickname(node_id); -} - -bool ContactStore::setNickname(uint32_t node_id, const char* nickname) -{ - return core_.setNickname(node_id, nickname); -} - -bool ContactStore::removeNickname(uint32_t node_id) -{ - return core_.removeNickname(node_id); -} - -bool ContactStore::hasNickname(const char* nickname) const -{ - return core_.hasNickname(nickname); -} - -std::vector ContactStore::getAllContactIds() const -{ - return core_.getAllContactIds(); -} - -size_t ContactStore::getCount() const -{ - return core_.getCount(); -} - -bool ContactStore::loadBlob(std::vector& out) -{ - if (backend_ == StorageBackend::Sd) - { - const LoadResult sd_result = loadFromSD(out); - if (sd_result == LoadResult::Loaded) - { - CONTACT_STORE_LOG("[ContactStore] load source=sd path=%s len=%u\n", - kSdPath, - static_cast(out.size())); - return true; - } - if (sd_result == LoadResult::Busy) - { - CONTACT_STORE_LOG("[ContactStore] load source=none reason=sd_busy\n"); - out.clear(); - return false; - } - - std::vector fallback; - if (loadFromFlash(fallback)) - { - CONTACT_STORE_LOG("[ContactStore] load source=flash fallback=sd_miss ns=%s len=%u\n", - kPrefNs, - static_cast(fallback.size())); - const bool migrated = saveToSD(fallback.data(), fallback.size()); - CONTACT_STORE_LOG("[ContactStore] migrate source=flash target=sd path=%s len=%u ok=%u\n", - kSdPath, - static_cast(fallback.size()), - migrated ? 1U : 0U); - if (migrated) - { - clearFlash(); - } - out.swap(fallback); - return true; - } - } - else - { - if (loadFromFlash(out)) - { - CONTACT_STORE_LOG("[ContactStore] load source=flash ns=%s len=%u\n", - kPrefNs, - static_cast(out.size())); - return true; - } - } - out.clear(); - CONTACT_STORE_LOG("[ContactStore] load source=none\n"); - return false; -} - -bool ContactStore::saveBlob(const uint8_t* data, size_t len) -{ - if (backend_ == StorageBackend::Sd) - { - const bool ok = saveToSD(data, len); - CONTACT_STORE_LOG("[ContactStore] save target=sd path=%s len=%u ok=%u\n", - kSdPath, - static_cast(len), - ok ? 1U : 0U); - return ok; - } - - const bool ok = saveToFlash(data, len); - CONTACT_STORE_LOG("[ContactStore] save target=flash ns=%s len=%u ok=%u\n", - kPrefNs, - static_cast(len), - ok ? 1U : 0U); - return ok; -} - -ContactStore::LoadResult ContactStore::loadFromSD(std::vector& out) const -{ - if (!::platform::esp::arduino_common::storage::sd_card_ready()) - { - out.clear(); - return LoadResult::MissingOrInvalid; - } - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_contact_store_bus_arbiter, - sys::runtime::BusAccessPolicy::BackgroundWorkerBounded, - kSdLoadWaitMs, - kContactStoreBusResource, - kContactStoreBusOwnerId + 1, - kContactStoreBusOwnerId); - if (!bus_gate.locked()) - { - out.clear(); - return LoadResult::Busy; - } - constexpr size_t kMaxContactBlobBytes = - ContactStoreCore::kMaxContacts * ContactStoreCore::kSerializedEntrySize; - return chat::infra::loadRawBlobFromSd(kSdPath, out, kMaxContactBlobBytes) - ? LoadResult::Loaded - : LoadResult::MissingOrInvalid; -} - -bool ContactStore::saveToSD(const uint8_t* data, size_t len) const -{ - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_contact_store_bus_arbiter, - sys::runtime::BusAccessPolicy::DurableCommit, - kSdPersistWaitMs, - kContactStoreBusResource, - kContactStoreBusOwnerId + 2, - kContactStoreBusOwnerId); - if (!bus_gate.locked()) - { - CONTACT_STORE_LOG("[ContactStore] save SD skipped: spi busy len=%u\n", - static_cast(len)); - return false; - } - return chat::infra::saveRawBlobToSd(kSdPath, data, len); -} - -bool ContactStore::loadFromFlash(std::vector& out) const -{ - return chat::infra::loadRawBlobFromPreferences(kPrefNs, kPrefKey, out); -} - -bool ContactStore::saveToFlash(const uint8_t* data, size_t len) const { return chat::infra::saveRawBlobToPreferences(kPrefNs, kPrefKey, data, len); } - -void ContactStore::clearFlash() const -{ - chat::infra::clearRawBlobFromPreferences(kPrefNs, kPrefKey); -} - -} // namespace contacts -} // namespace chat diff --git a/platform/esp/arduino_common/src/chat/infra/meshcore/meshcore_adapter.cpp b/platform/esp/arduino_common/src/chat/infra/meshcore/meshcore_adapter.cpp index b57ea66e..b8ebbcee 100644 --- a/platform/esp/arduino_common/src/chat/infra/meshcore/meshcore_adapter.cpp +++ b/platform/esp/arduino_common/src/chat/infra/meshcore/meshcore_adapter.cpp @@ -1677,6 +1677,8 @@ void MeshCoreAdapter::savePeerPubKeyToDirectory(const PeerRouteEntry& route) record.meshcore.public_key_verified = route.pubkey_verified; record.meshcore.has_peer_hash = true; record.meshcore.peer_hash = route.peer_hash; + record.meshcore.node_id_hint = + deriveNodeIdFromPubkey(route.pubkey, sizeof(route.pubkey)); const MeshPeerDirectoryStatus status = peer_directory_->record(record); if (!status.succeeded()) diff --git a/platform/esp/arduino_common/src/chat/infra/meshtastic/node_store.cpp b/platform/esp/arduino_common/src/chat/infra/meshtastic/node_store.cpp deleted file mode 100644 index 057fc25c..00000000 --- a/platform/esp/arduino_common/src/chat/infra/meshtastic/node_store.cpp +++ /dev/null @@ -1,853 +0,0 @@ -/** - * @file node_store.cpp - * @brief Persisted NodeInfo store shell backed by a single selected persistence backend - */ - -#include "platform/esp/arduino_common/chat/infra/meshtastic/node_store.h" -#include "../../internal/blob_store_io.h" -#include "chat/infra/node_store_blob_format.h" -#include "platform/esp/arduino_common/storage/persistence_bus_gate.h" -#include "platform/esp/arduino_common/storage/sd_card_runtime.h" -#include "platform/esp/common/shared_spi_bus_arbiter.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace chat -{ -namespace meshtastic -{ - -#ifndef NODE_STORE_LOG_ENABLE -#define NODE_STORE_LOG_ENABLE 1 -#endif - -#if NODE_STORE_LOG_ENABLE -#define NODE_STORE_LOG(...) Serial.printf(__VA_ARGS__) -#else -#define NODE_STORE_LOG(...) -#endif - -void* NodeStore::operator new(std::size_t size) -{ - void* ptr = heap_caps_malloc_prefer(size, - 2, - MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT, - MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - return ptr != nullptr ? ptr : ::operator new(size); -} - -void NodeStore::operator delete(void* ptr) noexcept -{ - heap_caps_free(ptr); -} - -void NodeStore::operator delete(void* ptr, std::size_t) noexcept -{ - operator delete(ptr); -} - -namespace -{ -constexpr TickType_t kAsyncSaveMutexWait = pdMS_TO_TICKS(20); -constexpr TickType_t kAsyncSavePollInterval = pdMS_TO_TICKS(10); -constexpr TickType_t kAsyncSaveRetryDelay = pdMS_TO_TICKS(500); -constexpr uint32_t kSdLoadWaitMs = 250; -constexpr uint32_t kSdPersistWaitMs = 100; -constexpr uint32_t kAsyncSaveTaskStackBytes = 4 * 1024; -constexpr UBaseType_t kAsyncSaveTaskPriority = 2; -constexpr size_t kSdReadChunkBytes = 256; -constexpr uint32_t kNodeStoreBusResource = 5; -constexpr uint32_t kNodeStoreBusOwnerId = 0x4E4F4445u; // 'NODE' -constexpr const char* kNodeStoreBusOwner = "node_store_sd"; - -::platform::esp::common::SharedSpiBusAdapter s_node_store_bus_adapter( - kNodeStoreBusOwner, - kNodeStoreBusOwnerId); -::platform::esp::common::FixedSharedSpiBusPolicyStrategy s_node_store_bus_policy( - kSdLoadWaitMs, - kSdLoadWaitMs, - kSdLoadWaitMs, - kSdLoadWaitMs); -sys::runtime::StorageBusArbiter s_node_store_bus_arbiter(s_node_store_bus_adapter, - s_node_store_bus_policy); - -void logNvsStats(const char* tag, const char* ns) -{ - nvs_stats_t stats{}; - esp_err_t err = nvs_get_stats(nullptr, &stats); - if (err == ESP_OK) - { - NODE_STORE_LOG("[NodeStore] NVS stats(%s): used=%u free=%u total=%u namespaces=%u\n", - tag ? tag : "?", - static_cast(stats.used_entries), - static_cast(stats.free_entries), - static_cast(stats.total_entries), - static_cast(stats.namespace_count)); - } - else - { - NODE_STORE_LOG("[NodeStore] NVS stats(%s) err=%s\n", - tag ? tag : "?", - esp_err_to_name(err)); - } - - if (ns && ns[0]) - { - nvs_handle_t handle; - err = nvs_open(ns, NVS_READONLY, &handle); - if (err == ESP_OK) - { - NODE_STORE_LOG("[NodeStore] NVS open ns=%s ok\n", ns); - nvs_close(handle); - } - else - { - NODE_STORE_LOG("[NodeStore] NVS open ns=%s err=%s\n", ns, esp_err_to_name(err)); - } - } -} - -bool canonicalizeNodePayload(const uint8_t* data, size_t len, uint8_t version, std::vector& out) -{ - std::vector entries; - if (!contacts::NodeStoreCore::decodeBlob(entries, data, len, version)) - { - out.clear(); - return false; - } - - contacts::NodeStoreCore::encodeBlob(out, entries); - return true; -} - -} // namespace - -NodeStore::NodeStore() - : core_(*this) -{ -} - -void NodeStore::begin() -{ - backend_ = ::platform::esp::arduino_common::storage::sd_card_ready() - ? StorageBackend::Sd - : StorageBackend::Nvs; - NODE_STORE_LOG("[NodeStore] backend=%s\n", - backend_ == StorageBackend::Sd ? "sd" : "nvs"); - core_.begin(); -} - -void NodeStore::applyUpdate(uint32_t node_id, const contacts::NodeUpdate& update) -{ - core_.applyUpdate(node_id, update); -} - -void NodeStore::upsert(uint32_t node_id, const char* short_name, const char* long_name, - uint32_t now_secs, float snr, float rssi, uint8_t protocol, - uint8_t role, uint8_t hops_away, uint8_t hw_model, uint8_t channel) -{ - NODE_STORE_LOG("[NodeStore] upsert node=%08lX ts=%lu snr=%.1f rssi=%.1f\n", - static_cast(node_id), - static_cast(now_secs), - snr, - rssi); - core_.upsert(node_id, short_name, long_name, now_secs, snr, rssi, - protocol, role, hops_away, hw_model, channel); -} - -void NodeStore::updateProtocol(uint32_t node_id, uint8_t protocol, uint32_t now_secs) -{ - core_.updateProtocol(node_id, protocol, now_secs); -} - -void NodeStore::updatePosition(uint32_t node_id, const contacts::NodePosition& position) -{ - core_.updatePosition(node_id, position); -} - -bool NodeStore::remove(uint32_t node_id) -{ - const bool removed = core_.remove(node_id); - if (removed) - { - NODE_STORE_LOG("[NodeStore] remove node=%08lX remaining=%u\n", - static_cast(node_id), - static_cast(core_.getEntries().size())); - } - return removed; -} - -const std::vector& NodeStore::getEntries() const -{ - return core_.getEntries(); -} - -void NodeStore::clear() -{ - core_.clear(); -} - -bool NodeStore::flush() -{ - const bool accepted = core_.flush(); - const bool persisted = waitForAsyncSave(pdMS_TO_TICKS(1500)); - return accepted && persisted; -} - -bool NodeStore::loadBlob(std::vector& out) -{ - out.clear(); - - if (backend_ == StorageBackend::Sd) - { - const LoadResult sd_result = loadFromSd(out); - if (sd_result == LoadResult::Loaded) - { - NODE_STORE_LOG("[NodeStore] load source=sd path=%s len=%u\n", - kPersistNodesFile, - static_cast(out.size())); - return true; - } - if (sd_result == LoadResult::Busy) - { - NODE_STORE_LOG("[NodeStore] load source=none reason=sd_busy\n"); - return false; - } - - std::vector fallback; - if (loadFromNvs(fallback)) - { - NODE_STORE_LOG("[NodeStore] load source=nvs fallback=sd_miss ns=%s len=%u\n", - kPersistNodesNs, - static_cast(fallback.size())); - const bool migrated = saveToSd(fallback.data(), fallback.size()); - NODE_STORE_LOG("[NodeStore] migrate source=nvs target=sd path=%s len=%u ok=%u\n", - kPersistNodesFile, - static_cast(fallback.size()), - migrated ? 1U : 0U); - if (migrated) - { - clearNvs(); - } - out.swap(fallback); - return true; - } - } - else - { - if (loadFromNvs(out)) - { - NODE_STORE_LOG("[NodeStore] load source=nvs ns=%s len=%u\n", - kPersistNodesNs, - static_cast(out.size())); - return true; - } - } - - NODE_STORE_LOG("[NodeStore] load source=none\n"); - return false; -} - -bool NodeStore::saveBlob(const uint8_t* data, size_t len) -{ - return enqueueAsyncSave(data, len); -} - -void NodeStore::clearBlob() -{ - if (async_save_mutex_ != nullptr && - xSemaphoreTake(async_save_mutex_, kAsyncSaveMutexWait) == pdTRUE) - { - pending_save_blob_.clear(); - async_save_pending_ = false; - async_save_failed_ = false; - xSemaphoreGive(async_save_mutex_); - } - - if (::platform::esp::arduino_common::storage::sd_card_ready()) - { - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_node_store_bus_arbiter, - sys::runtime::BusAccessPolicy::DurableCommit, - kSdPersistWaitMs, - kNodeStoreBusResource, - kNodeStoreBusOwnerId + 3, - kNodeStoreBusOwnerId); - if (bus_gate.locked() && - ::platform::esp::arduino_common::storage::sd_exists(kPersistNodesFile)) - { - ::platform::esp::arduino_common::storage::sd_remove(kPersistNodesFile); - } - } - clearNvs(); -} - -bool NodeStore::loadFromNvs(std::vector& out) -{ - out.clear(); - - chat::infra::PreferencesBlobMetadata meta; - if (!chat::infra::loadRawBlobFromPreferencesWithMetadata(kPersistNodesNs, - kPersistNodesKey, - kPersistNodesKeyVer, - kPersistNodesKeyCrc, - out, - &meta)) - { - NODE_STORE_LOG("[NodeStore] NVS blob missing/unreadable ns=%s\n", kPersistNodesNs); - logNvsStats("begin", kPersistNodesNs); - return false; - } - - NODE_STORE_LOG("[NodeStore] blob len=%u ver=%u crc=%08lX has_crc=%u\n", - static_cast(meta.len), - static_cast(meta.version), - static_cast(meta.crc), - meta.has_crc ? 1U : 0U); - - if (meta.len == 0) - { - if (meta.has_crc || meta.has_version) - { - NODE_STORE_LOG("[NodeStore] stale meta detected ver=%u has_crc=%u\n", - static_cast(meta.version), - meta.has_crc ? 1U : 0U); - } - NODE_STORE_LOG("[NodeStore] NVS backup empty\n"); - return false; - } - - if (!meta.has_version) - { - NODE_STORE_LOG("[NodeStore] missing version len=%u\n", - static_cast(meta.len), - static_cast(meta.len)); - out.clear(); - return false; - } - - const size_t entry_size = contacts::nodeBlobEntrySizeForVersion(meta.version); - if (entry_size == 0 || - meta.len == 0 || - (meta.len % entry_size) != 0 || - (meta.len / entry_size) > contacts::NodeStoreCore::kMaxNodes) - { - NODE_STORE_LOG("[NodeStore] invalid blob shape len=%u ver=%u\n", - static_cast(meta.len), - static_cast(meta.version)); - out.clear(); - return false; - } - - if (!meta.has_crc) - { - NODE_STORE_LOG("[NodeStore] missing crc\n"); - out.clear(); - return false; - } - - const uint32_t calc_crc = contacts::NodeStoreCore::computeBlobCrc(out.data(), out.size()); - if (calc_crc != meta.crc) - { - NODE_STORE_LOG("[NodeStore] crc mismatch stored=%08lX calc=%08lX\n", - static_cast(meta.crc), - static_cast(calc_crc)); - out.clear(); - return false; - } - - std::vector normalized; - if (!canonicalizeNodePayload(out.data(), out.size(), meta.version, normalized)) - { - NODE_STORE_LOG("[NodeStore] decode failed ver=%u len=%u\n", - static_cast(meta.version), - static_cast(out.size())); - out.clear(); - return false; - } - - out.swap(normalized); - NODE_STORE_LOG("[NodeStore] loaded=%u\n", - static_cast(out.size() / contacts::NodeStoreCore::kSerializedEntrySize)); - return true; -} - -NodeStore::LoadResult NodeStore::loadFromSd(std::vector& out) const -{ - if (!::platform::esp::arduino_common::storage::sd_card_ready()) - { - NODE_STORE_LOG("[NodeStore] load SD skipped: card none\n"); - return LoadResult::MissingOrInvalid; - } - - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_node_store_bus_arbiter, - sys::runtime::BusAccessPolicy::BackgroundWorkerBounded, - kSdLoadWaitMs, - kNodeStoreBusResource, - kNodeStoreBusOwnerId + 1, - kNodeStoreBusOwnerId); - if (!bus_gate.locked()) - { - NODE_STORE_LOG("[NodeStore] load SD skipped: spi busy\n"); - return LoadResult::Busy; - } - - ::platform::esp::arduino_common::storage::SdRuntimeFile file; - if (!file.open(kPersistNodesFile, "r")) - { - NODE_STORE_LOG("[NodeStore] load SD open failed path=%s\n", kPersistNodesFile); - return LoadResult::MissingOrInvalid; - } - - const size_t file_size = static_cast(file.size()); - if (file_size < sizeof(contacts::NodeStoreSdHeader)) - { - NODE_STORE_LOG("[NodeStore] load SD short file path=%s size=%u\n", - kPersistNodesFile, - static_cast(file_size)); - file.close(); - return LoadResult::MissingOrInvalid; - } - - contacts::NodeStoreSdHeader header{}; - if (file.read(reinterpret_cast(&header), sizeof(header)) != sizeof(header)) - { - NODE_STORE_LOG("[NodeStore] load SD header read failed path=%s\n", kPersistNodesFile); - file.close(); - return LoadResult::MissingOrInvalid; - } - - if (header.count == 0 || header.count > contacts::NodeStoreCore::kMaxNodes) - { - NODE_STORE_LOG("[NodeStore] load SD header invalid path=%s ver=%u count=%u\n", - kPersistNodesFile, - static_cast(header.ver), - static_cast(header.count)); - file.close(); - return LoadResult::MissingOrInvalid; - } - - const size_t payload_len = file_size - sizeof(header); - if (payload_len == 0) - { - NODE_STORE_LOG("[NodeStore] load SD empty payload path=%s ver=%u count=%u\n", - kPersistNodesFile, - static_cast(header.ver), - static_cast(header.count)); - file.close(); - return LoadResult::MissingOrInvalid; - } - - const size_t entry_size = contacts::nodeBlobEntrySizeForVersion(header.ver); - if (entry_size == 0) - { - NODE_STORE_LOG("[NodeStore] load SD unsupported version path=%s ver=%u\n", - kPersistNodesFile, - static_cast(header.ver)); - file.close(); - return LoadResult::MissingOrInvalid; - } - - const size_t expected_len = static_cast(header.count) * entry_size; - if (expected_len != payload_len) - { - NODE_STORE_LOG("[NodeStore] load SD blob size mismatch path=%s len=%u ver=%u count=%u expected=%u\n", - kPersistNodesFile, - static_cast(payload_len), - static_cast(header.ver), - static_cast(header.count), - static_cast(expected_len)); - file.close(); - return LoadResult::MissingOrInvalid; - } - - out.resize(expected_len); - size_t total_read = 0; - while (total_read < expected_len) - { - const size_t chunk = std::min(kSdReadChunkBytes, expected_len - total_read); - const size_t read_bytes = file.read(out.data() + total_read, chunk); - if (read_bytes != chunk) - { - NODE_STORE_LOG("[NodeStore] load SD payload read mismatch path=%s want=%u got=%u\n", - kPersistNodesFile, - static_cast(chunk), - static_cast(read_bytes)); - file.close(); - out.clear(); - return LoadResult::MissingOrInvalid; - } - total_read += read_bytes; - } - file.close(); - - if (out.size() != expected_len) - { - NODE_STORE_LOG("[NodeStore] load SD blob size mismatch path=%s len=%u ver=%u count=%u expected=%u\n", - kPersistNodesFile, - static_cast(out.size()), - static_cast(header.ver), - static_cast(header.count), - static_cast(expected_len)); - out.clear(); - return LoadResult::MissingOrInvalid; - } - - const uint32_t calc_crc = contacts::NodeStoreCore::computeBlobCrc(out.data(), out.size()); - if (calc_crc != header.crc) - { - NODE_STORE_LOG("[NodeStore] load SD crc mismatch path=%s stored=%08lX calc=%08lX\n", - kPersistNodesFile, - static_cast(header.crc), - static_cast(calc_crc)); - out.clear(); - return LoadResult::MissingOrInvalid; - } - - std::vector normalized; - if (!canonicalizeNodePayload(out.data(), out.size(), header.ver, normalized)) - { - NODE_STORE_LOG("[NodeStore] load SD decode failed path=%s ver=%u len=%u\n", - kPersistNodesFile, - static_cast(header.ver), - static_cast(out.size())); - out.clear(); - return LoadResult::MissingOrInvalid; - } - - out.swap(normalized); - NODE_STORE_LOG("[NodeStore] loaded=%u (SD)\n", - static_cast(out.size() / contacts::NodeStoreCore::kSerializedEntrySize)); - return LoadResult::Loaded; -} - -bool NodeStore::saveToNvs(const uint8_t* data, size_t len) const -{ - chat::infra::PreferencesBlobMetadata meta; - if (data && len > 0) - { - if (!contacts::isValidNodeBlobSize(len) || - contacts::nodeBlobEntryCount(len) > contacts::NodeStoreCore::kMaxNodes) - { - return false; - } - meta.len = len; - meta.has_version = true; - meta.version = contacts::NodeStoreCore::kPersistVersion; - meta.has_crc = true; - meta.crc = contacts::NodeStoreCore::computeBlobCrc(data, len); - } - - const bool ok = chat::infra::saveRawBlobToPreferencesWithMetadata(kPersistNodesNs, - kPersistNodesKey, - kPersistNodesKeyVer, - kPersistNodesKeyCrc, - data, - len, - &meta, - true); - if (!ok) - { - NODE_STORE_LOG("[NodeStore] save failed ns=%s\n", kPersistNodesNs); - logNvsStats("save-write", kPersistNodesNs); - } - return ok; -} - -bool NodeStore::saveToSd(const uint8_t* data, size_t len) const -{ - if (!::platform::esp::arduino_common::storage::sd_card_ready()) - { - return false; - } - if (!contacts::isValidNodeBlobSize(len) || - contacts::nodeBlobEntryCount(len) > contacts::NodeStoreCore::kMaxNodes) - { - return false; - } - - ::platform::esp::arduino_common::storage::PersistenceBusGate bus_gate( - s_node_store_bus_arbiter, - sys::runtime::BusAccessPolicy::DurableCommit, - kSdPersistWaitMs, - kNodeStoreBusResource, - kNodeStoreBusOwnerId + 2, - kNodeStoreBusOwnerId); - if (!bus_gate.locked()) - { - NODE_STORE_LOG("[NodeStore] save SD skipped: spi busy len=%u\n", - static_cast(len)); - return false; - } - - const std::string temp_path = std::string(kPersistNodesFile) + ".tmp"; - if (::platform::esp::arduino_common::storage::sd_exists(temp_path.c_str())) - { - ::platform::esp::arduino_common::storage::sd_remove(temp_path.c_str()); - } - - ::platform::esp::arduino_common::storage::SdRuntimeFile file; - if (!file.open(temp_path.c_str(), "w")) - { - return false; - } - - contacts::NodeStoreSdHeader header = contacts::makeNodeStoreSdHeader(data, len); - - bool ok = (file.write(reinterpret_cast(&header), sizeof(header)) == sizeof(header)); - if (ok && data && len > 0) - { - ok = (file.write(data, len) == len); - } - - file.close(); - if (!ok) - { - ::platform::esp::arduino_common::storage::sd_remove(temp_path.c_str()); - return false; - } - - if (::platform::esp::arduino_common::storage::sd_exists(kPersistNodesFile)) - { - ::platform::esp::arduino_common::storage::sd_remove(kPersistNodesFile); - } - - if (!::platform::esp::arduino_common::storage::sd_rename(temp_path.c_str(), kPersistNodesFile)) - { - ::platform::esp::arduino_common::storage::sd_remove(temp_path.c_str()); - return false; - } - - NODE_STORE_LOG("[NodeStore] save SD path=%s len=%u ver=%u count=%u\n", - kPersistNodesFile, - static_cast(len), - static_cast(header.ver), - static_cast(header.count)); - return true; -} - -bool NodeStore::saveToBackend(const uint8_t* data, size_t len, StorageBackend backend) const -{ - if (backend == StorageBackend::Sd) - { - const bool ok = saveToSd(data, len); - NODE_STORE_LOG("[NodeStore] save target=sd len=%u count=%u ok=%u\n", - static_cast(len), - static_cast(contacts::nodeBlobEntryCount(len)), - ok ? 1U : 0U); - return ok; - } - - const bool ok = saveToNvs(data, len); - NODE_STORE_LOG("[NodeStore] save target=nvs len=%u count=%u ok=%u\n", - static_cast(len), - static_cast(contacts::nodeBlobEntryCount(len)), - ok ? 1U : 0U); - return ok; -} - -void NodeStore::ensureAsyncSaveWorker() -{ - if (async_save_mutex_ == nullptr) - { - async_save_mutex_ = xSemaphoreCreateMutex(); - } - if (async_save_queue_ == nullptr) - { - async_save_queue_ = xQueueCreate(1, sizeof(uint8_t)); - } - if (async_save_task_ == nullptr && async_save_mutex_ != nullptr && async_save_queue_ != nullptr) - { - BaseType_t ok = xTaskCreate(asyncSaveTaskEntry, - "node_store_io", - kAsyncSaveTaskStackBytes, - this, - kAsyncSaveTaskPriority, - &async_save_task_); - if (ok != pdPASS) - { - NODE_STORE_LOG("[NodeStore] async save task create failed\n"); - async_save_task_ = nullptr; - } - } -} - -bool NodeStore::enqueueAsyncSave(const uint8_t* data, size_t len) -{ - if (len > 0 && data == nullptr) - { - return false; - } - if (!contacts::isValidNodeBlobSize(len) || - contacts::nodeBlobEntryCount(len) > contacts::NodeStoreCore::kMaxNodes) - { - return false; - } - - ensureAsyncSaveWorker(); - if (async_save_mutex_ == nullptr || async_save_queue_ == nullptr || async_save_task_ == nullptr) - { - return false; - } - - if (xSemaphoreTake(async_save_mutex_, kAsyncSaveMutexWait) != pdTRUE) - { - NODE_STORE_LOG("[NodeStore] async save enqueue busy len=%u\n", - static_cast(len)); - return false; - } - - pending_save_blob_.assign(data, data + len); - pending_save_backend_ = backend_; - ++pending_save_generation_; - async_save_pending_ = true; - async_save_failed_ = false; - const uint32_t generation = pending_save_generation_; - const StorageBackend target = pending_save_backend_; - xSemaphoreGive(async_save_mutex_); - - const uint8_t signal = 1; - if (xQueueOverwrite(async_save_queue_, &signal) != pdTRUE) - { - NODE_STORE_LOG("[NodeStore] async save signal failed gen=%lu\n", - static_cast(generation)); - return false; - } - - NODE_STORE_LOG("[NodeStore] save queued gen=%lu target=%s len=%u count=%u\n", - static_cast(generation), - target == StorageBackend::Sd ? "sd" : "nvs", - static_cast(len), - static_cast(contacts::nodeBlobEntryCount(len))); - return true; -} - -bool NodeStore::waitForAsyncSave(TickType_t wait_ticks) -{ - ensureAsyncSaveWorker(); - if (async_save_mutex_ == nullptr || async_save_queue_ == nullptr || async_save_task_ == nullptr) - { - return false; - } - - const TickType_t start = xTaskGetTickCount(); - while (true) - { - bool done = false; - bool failed = false; - if (xSemaphoreTake(async_save_mutex_, kAsyncSaveMutexWait) == pdTRUE) - { - done = !async_save_pending_ && !async_save_busy_; - failed = async_save_failed_; - xSemaphoreGive(async_save_mutex_); - } - if (done) - { - return !failed; - } - if ((xTaskGetTickCount() - start) >= wait_ticks) - { - return false; - } - vTaskDelay(kAsyncSavePollInterval); - } -} - -void NodeStore::asyncSaveLoop() -{ - uint8_t signal = 0; - for (;;) - { - if (xQueueReceive(async_save_queue_, &signal, portMAX_DELAY) != pdTRUE) - { - continue; - } - - for (;;) - { - std::vector blob; - StorageBackend backend = StorageBackend::Nvs; - uint32_t generation = 0; - - if (xSemaphoreTake(async_save_mutex_, portMAX_DELAY) != pdTRUE) - { - break; - } - if (!async_save_pending_) - { - async_save_busy_ = false; - xSemaphoreGive(async_save_mutex_); - break; - } - blob = pending_save_blob_; - backend = pending_save_backend_; - generation = pending_save_generation_; - async_save_pending_ = false; - async_save_busy_ = true; - xSemaphoreGive(async_save_mutex_); - - const bool ok = saveToBackend(blob.data(), blob.size(), backend); - - bool has_more = false; - if (xSemaphoreTake(async_save_mutex_, portMAX_DELAY) == pdTRUE) - { - async_save_busy_ = false; - async_save_failed_ = !ok; - if (ok) - { - completed_save_generation_ = generation; - } - else if (!async_save_pending_) - { - pending_save_blob_ = blob; - pending_save_backend_ = backend; - async_save_pending_ = true; - } - has_more = async_save_pending_; - xSemaphoreGive(async_save_mutex_); - } - - NODE_STORE_LOG("[NodeStore] save done gen=%lu ok=%u more=%u\n", - static_cast(generation), - ok ? 1U : 0U, - has_more ? 1U : 0U); - if (!ok) - { - vTaskDelay(kAsyncSaveRetryDelay); - } - if (!has_more) - { - break; - } - } - } -} - -void NodeStore::asyncSaveTaskEntry(void* context) -{ - auto* self = static_cast(context); - if (self) - { - self->asyncSaveLoop(); - } - vTaskDelete(nullptr); -} - -void NodeStore::clearNvs() const -{ - chat::infra::clearPreferencesKeys(kPersistNodesNs, - kPersistNodesKey, - kPersistNodesKeyVer, - kPersistNodesKeyCrc); -} - -} // namespace meshtastic -} // namespace chat diff --git a/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp b/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp new file mode 100644 index 00000000..ffbc1a5b --- /dev/null +++ b/platform/esp/arduino_common/src/chat/infra/store/fixed_slot_journal.cpp @@ -0,0 +1,276 @@ +#include "platform/esp/arduino_common/chat/infra/store/fixed_slot_journal.h" + +#include "platform/esp/arduino_common/storage/sd_card_runtime.h" + +#include +#include + +namespace chat::storage::v2 +{ +namespace +{ +namespace storage = ::platform::esp::arduino_common::storage; + +bool readExact(storage::SdRuntimeFile& file, void* out, std::size_t len) +{ + return out && len != 0U && + file.read(out, len) == static_cast(len); +} + +bool writeExact(storage::SdRuntimeFile& file, + const void* data, + std::size_t len) +{ + return data && len != 0U && file.write(data, len) == len; +} + +} // namespace + +FixedSlotJournalEngine::Inspection FixedSlotJournalEngine::inspect( + const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) const +{ + Inspection result{}; + if (!path || path[0] == '\0' || + !validDescriptor(protocol, kind, slot_size)) + { + result.state = State::Incompatible; + return result; + } + if (!storage::sd_exists(path)) + { + result.state = State::Missing; + return result; + } + + storage::SdRuntimeFile file; + if (!file.open(path, "r")) + { + result.state = State::IoError; + return result; + } + Header header{}; + if (file.size() < sizeof(header) || !readExact(file, &header, sizeof(header))) + { + result.state = State::Incompatible; + return result; + } + if (!matches(header, protocol, kind, slot_size)) + { + result.state = State::Incompatible; + return result; + } + + const uint64_t payload_size = file.size() - sizeof(header); + if (payload_size % slot_size != 0U) + { + result.state = State::PartialTail; + result.slot_count = static_cast(payload_size / slot_size); + return result; + } + const uint64_t count = payload_size / slot_size; + if (count > std::numeric_limits::max()) + { + result.state = State::Incompatible; + return result; + } + result.state = State::Ready; + result.slot_count = static_cast(count); + return result; +} + +bool FixedSlotJournalEngine::create(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) const +{ + if (!path || path[0] == '\0' || + !validDescriptor(protocol, kind, slot_size)) + { + return false; + } + storage::SdRuntimeFile file; + if (!file.open(path, "w+")) + { + return false; + } + const Header header = makeHeader(protocol, kind, slot_size); + return writeExact(file, &header, sizeof(header)) && file.flush(); +} + +bool FixedSlotJournalEngine::append(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size, + const void* slot) const +{ + if (!slot) + { + return false; + } + Inspection inspection = inspect(path, protocol, kind, slot_size); + if (inspection.state == State::Missing) + { + if (!create(path, protocol, kind, slot_size)) + { + return false; + } + inspection = inspect(path, protocol, kind, slot_size); + } + if (inspection.state != State::Ready) + { + return false; + } + + storage::SdRuntimeFile file; + if (!file.open(path, "r+") || !file.seek(file.size())) + { + return false; + } + return writeExact(file, slot, slot_size) && file.flush(); +} + +bool FixedSlotJournalEngine::read(const char* path, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size, + uint32_t slot_index, + void* out_slot) const +{ + if (!out_slot) + { + return false; + } + const Inspection inspection = inspect(path, protocol, kind, slot_size); + if ((inspection.state != State::Ready && + inspection.state != State::PartialTail) || + slot_index >= inspection.slot_count) + { + return false; + } + + storage::SdRuntimeFile file; + const uint64_t offset = sizeof(Header) + + static_cast(slot_index) * slot_size; + return file.open(path, "r") && file.seek(offset) && + readExact(file, out_slot, slot_size); +} + +bool FixedSlotJournalEngine::validDescriptor(MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) +{ + return supportedProtocol(protocol) && + static_cast(kind) >= + static_cast(JournalKind::MessageSegment) && + static_cast(kind) <= + static_cast(JournalKind::ContactDelta) && + slot_size > 0U && + slot_size <= std::numeric_limits::max(); +} + +FixedSlotJournalEngine::Header FixedSlotJournalEngine::makeHeader( + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) +{ + Header header{}; + header.magic = kMagic; + header.schema = kStorageSchemaVersion; + header.protocol = static_cast(canonicalProtocol(protocol)); + header.kind = static_cast(kind); + header.slot_size = static_cast(slot_size); + header.crc = crc32(&header, sizeof(header) - sizeof(header.crc)); + return header; +} + +bool FixedSlotJournalEngine::matches(const Header& header, + MeshProtocol protocol, + JournalKind kind, + std::size_t slot_size) +{ + return header.magic == kMagic && + header.schema == kStorageSchemaVersion && + header.protocol == + static_cast(canonicalProtocol(protocol)) && + header.kind == static_cast(kind) && + header.slot_size == slot_size && + header.crc == crc32(&header, sizeof(header) - sizeof(header.crc)); +} + +bool replaceFileAtomically(const char* temp_path, + const char* final_path, + const char* backup_path) +{ + if (!temp_path || !final_path || !backup_path || + !storage::sd_exists(temp_path)) + { + return false; + } + if (storage::sd_exists(backup_path) && + !storage::sd_remove(backup_path)) + { + return false; + } + const bool had_final = storage::sd_exists(final_path); + if (had_final && !storage::sd_rename(final_path, backup_path)) + { + return false; + } + if (!storage::sd_rename(temp_path, final_path)) + { + if (had_final) + { + (void)storage::sd_rename(backup_path, final_path); + } + return false; + } + if (had_final) + { + (void)storage::sd_remove(backup_path); + } + return true; +} + +bool recoverAtomicFile(const char* final_path, + const char* temp_path, + const char* backup_path) +{ + if (!final_path || !temp_path || !backup_path) + { + return false; + } + if (storage::sd_exists(final_path)) + { + if (storage::sd_exists(temp_path)) + { + (void)storage::sd_remove(temp_path); + } + if (storage::sd_exists(backup_path)) + { + (void)storage::sd_remove(backup_path); + } + return true; + } + if (storage::sd_exists(backup_path)) + { + if (!storage::sd_rename(backup_path, final_path)) + { + return false; + } + if (storage::sd_exists(temp_path)) + { + (void)storage::sd_remove(temp_path); + } + return true; + } + if (storage::sd_exists(temp_path)) + { + (void)storage::sd_remove(temp_path); + } + return true; +} + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp b/platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp new file mode 100644 index 00000000..901526cf --- /dev/null +++ b/platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp @@ -0,0 +1,789 @@ +#include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" + +#include +#include + +namespace chat::storage::v2 +{ +namespace +{ + +constexpr uint32_t kMessageMagic = 0x3247534DU; // MSG2 +constexpr uint32_t kCatalogMagic = 0x32544143U; // CAT2 +constexpr uint32_t kReadMagic = 0x32444152U; // RAD2 +constexpr uint32_t kStatusMagic = 0x32535453U; // STS2 +constexpr uint32_t kSeenMagic = 0x324E4553U; // SEN2 + +constexpr uint8_t kMessageSourceUnverified = 0x01U; +constexpr uint8_t kMessageHasGeo = 0x02U; +constexpr uint8_t kMessageHasReticulumIdentity = 0x04U; +constexpr uint8_t kMessageHasLxmfHash = 0x08U; +constexpr uint8_t kProjectionDeleted = 0x01U; +constexpr uint8_t kProjectionHasReticulumIdentity = 0x02U; + +struct MessagePrefix +{ + uint32_t magic = kMessageMagic; + uint16_t schema = kStorageSchemaVersion; + uint8_t flags = 0; + uint8_t reserved = 0; + uint32_t sequence = 0; + uint32_t message_id = 0; + uint32_t timestamp = 0; + uint32_t from = 0; + uint32_t peer = 0; + int32_t geo_lat_e7 = 0; + int32_t geo_lon_e7 = 0; + uint16_t text_len = 0; + uint8_t status = 0; + uint8_t channel = 0; + uint8_t rx_origin = 0; + uint8_t team_location_icon = 0; +} __attribute__((packed)); + +struct MeshtasticMessageSlot +{ + MessagePrefix prefix{}; + char text[kMeshtasticTextMax] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct MeshCoreMessageSlot +{ + MessagePrefix prefix{}; + char text[kMeshCoreTextMax] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReticulumMessageSlot +{ + MessagePrefix prefix{}; + uint8_t destination_hash[kReticulumPeerHashSize] = {}; + uint8_t identity_hash[kReticulumPeerHashSize] = {}; + uint8_t lxmf_hash[kReticulumLxmfHashSize] = {}; + char text[kReticulumTextMax] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct CatalogPrefix +{ + uint32_t magic = kCatalogMagic; + uint16_t schema = kStorageSchemaVersion; + uint8_t flags = 0; + uint8_t channel = 0; + uint32_t peer = 0; + uint32_t last_message_id = 0; + uint32_t message_count = 0; + uint32_t last_sequence = 0; + uint32_t last_timestamp = 0; + uint32_t unread = 0; + uint8_t last_status = 0; + uint8_t preview_len = 0; + uint16_t reserved = 0; + char preview[kCatalogPreviewMax] = {}; +} __attribute__((packed)); + +struct NodeCatalogSlot +{ + CatalogPrefix prefix{}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReticulumCatalogSlot +{ + CatalogPrefix prefix{}; + uint8_t destination_hash[kReticulumPeerHashSize] = {}; + uint8_t identity_hash[kReticulumPeerHashSize] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReadPrefix +{ + uint32_t magic = kReadMagic; + uint16_t schema = kStorageSchemaVersion; + uint8_t flags = 0; + uint8_t channel = 0; + uint32_t peer = 0; + uint32_t last_read_sequence = 0; +} __attribute__((packed)); + +struct NodeReadSlot +{ + ReadPrefix prefix{}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReticulumReadSlot +{ + ReadPrefix prefix{}; + uint8_t destination_hash[kReticulumPeerHashSize] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct StatusSlot +{ + uint32_t magic = kStatusMagic; + uint16_t schema = kStorageSchemaVersion; + uint8_t status = 0; + uint8_t reserved = 0; + uint32_t message_id = 0; + uint32_t sequence = 0; + uint32_t crc = 0; +} __attribute__((packed)); + +struct SeenSlot +{ + uint32_t magic = kSeenMagic; + uint16_t schema = kStorageSchemaVersion; + uint16_t reserved = 0; + uint8_t hash[kReticulumLxmfHashSize] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +static_assert(sizeof(MessagePrefix) == 42, + "v2 message prefix size changed"); +static_assert(sizeof(MeshtasticMessageSlot) == 279, + "v2 Meshtastic message slot size changed"); +static_assert(sizeof(MeshCoreMessageSlot) == 217, + "v2 MeshCore message slot size changed"); +static_assert(sizeof(ReticulumMessageSlot) == 365, + "v2 Reticulum message slot size changed"); + +bool validStatus(uint8_t raw) +{ + return raw <= static_cast(MessageStatus::Delivered); +} + +template +bool validSlot(const T& slot, uint32_t expected_magic) +{ + return slot.crc == crc32(&slot, sizeof(T) - sizeof(slot.crc)) && + slot.prefix.magic == expected_magic && + slot.prefix.schema == kStorageSchemaVersion; +} + +bool validStatusSlot(const StatusSlot& slot) +{ + return slot.magic == kStatusMagic && + slot.schema == kStorageSchemaVersion && + validStatus(slot.status) && + slot.crc == crc32(&slot, sizeof(slot) - sizeof(slot.crc)); +} + +bool validSeenSlot(const SeenSlot& slot) +{ + return slot.magic == kSeenMagic && + slot.schema == kStorageSchemaVersion && + slot.crc == crc32(&slot, sizeof(slot) - sizeof(slot.crc)); +} + +void encodeMessagePrefix(MessagePrefix& prefix, + const ChatMessage& message, + uint32_t sequence, + std::size_t text_len) +{ + prefix = MessagePrefix{}; + prefix.sequence = sequence; + prefix.message_id = message.msg_id; + prefix.timestamp = message.timestamp; + prefix.from = message.from; + prefix.peer = message.peer; + prefix.geo_lat_e7 = message.geo_lat_e7; + prefix.geo_lon_e7 = message.geo_lon_e7; + prefix.text_len = static_cast(text_len); + prefix.status = static_cast(message.status); + prefix.channel = static_cast(message.channel); + prefix.rx_origin = static_cast(message.rx_origin); + prefix.team_location_icon = message.team_location_icon; + if (message.source_unverified) + { + prefix.flags |= kMessageSourceUnverified; + } + if (message.has_geo) + { + prefix.flags |= kMessageHasGeo; + } +} + +bool decodeMessagePrefix(const MessagePrefix& prefix, + std::size_t text_capacity, + ChatMessage& message, + uint32_t* out_sequence) +{ + if (prefix.magic != kMessageMagic || + prefix.schema != kStorageSchemaVersion || + prefix.text_len > text_capacity || !validStatus(prefix.status)) + { + return false; + } + message.channel = static_cast(prefix.channel); + message.from = prefix.from; + message.peer = prefix.peer; + message.msg_id = prefix.message_id; + message.timestamp = prefix.timestamp; + message.team_location_icon = prefix.team_location_icon; + message.has_geo = (prefix.flags & kMessageHasGeo) != 0; + message.geo_lat_e7 = prefix.geo_lat_e7; + message.geo_lon_e7 = prefix.geo_lon_e7; + message.source_unverified = + (prefix.flags & kMessageSourceUnverified) != 0; + message.rx_origin = static_cast(prefix.rx_origin); + message.status = static_cast(prefix.status); + if (out_sequence) + { + *out_sequence = prefix.sequence; + } + return true; +} + +template +bool encodeNodeMessage(const ChatMessage& message, + uint32_t sequence, + void* out, + std::size_t out_len) +{ + T slot{}; + if (!out || out_len != sizeof(T) || + message.text.size() > sizeof(slot.text)) + { + return false; + } + encodeMessagePrefix(slot.prefix, message, sequence, message.text.size()); + if (!message.text.empty()) + { + std::memcpy(slot.text, message.text.data(), message.text.size()); + } + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; +} + +template +bool decodeNodeMessage(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatMessage& message, + uint32_t* out_sequence) +{ + if (!data || len != sizeof(T)) + { + return false; + } + T slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kMessageMagic) || + !decodeMessagePrefix(slot.prefix, + sizeof(slot.text), + message, + out_sequence)) + { + return false; + } + message.protocol = protocol; + message.text.assign(slot.text, slot.prefix.text_len); + return true; +} + +void encodeCatalogPrefix(CatalogPrefix& prefix, + const ChatCatalogProjection& projection) +{ + prefix = CatalogPrefix{}; + prefix.channel = static_cast(projection.conversation.channel); + prefix.peer = projection.conversation.peer; + prefix.last_message_id = projection.last_message_id; + prefix.message_count = projection.message_count; + prefix.last_sequence = projection.last_sequence; + prefix.last_timestamp = projection.last_timestamp; + prefix.unread = projection.unread; + prefix.last_status = static_cast(projection.last_status); + if (projection.deleted) + { + prefix.flags |= kProjectionDeleted; + } + const std::size_t preview_len = + std::min(std::strlen(projection.preview), + kCatalogPreviewMax); + prefix.preview_len = static_cast(preview_len); + if (preview_len > 0) + { + std::memcpy(prefix.preview, projection.preview, preview_len); + } +} + +bool decodeCatalogPrefix(MeshProtocol protocol, + const CatalogPrefix& prefix, + ChatCatalogProjection& projection) +{ + if (prefix.magic != kCatalogMagic || + prefix.schema != kStorageSchemaVersion || + prefix.preview_len > kCatalogPreviewMax || + !validStatus(prefix.last_status)) + { + return false; + } + projection = ChatCatalogProjection{}; + projection.conversation.protocol = protocol; + projection.conversation.channel = static_cast(prefix.channel); + projection.conversation.peer = prefix.peer; + projection.last_message_id = prefix.last_message_id; + projection.message_count = prefix.message_count; + projection.last_sequence = prefix.last_sequence; + projection.last_timestamp = prefix.last_timestamp; + projection.unread = prefix.unread; + projection.last_status = static_cast(prefix.last_status); + projection.deleted = (prefix.flags & kProjectionDeleted) != 0; + std::memcpy(projection.preview, prefix.preview, prefix.preview_len); + projection.preview[prefix.preview_len] = '\0'; + return true; +} + +void encodeReadPrefix(ReadPrefix& prefix, + const ChatReadProjection& projection) +{ + prefix = ReadPrefix{}; + prefix.channel = static_cast(projection.conversation.channel); + prefix.peer = projection.conversation.peer; + prefix.last_read_sequence = projection.last_read_sequence; + if (projection.deleted) + { + prefix.flags |= kProjectionDeleted; + } +} + +bool decodeReadPrefix(MeshProtocol protocol, + const ReadPrefix& prefix, + ChatReadProjection& projection) +{ + if (prefix.magic != kReadMagic || prefix.schema != kStorageSchemaVersion) + { + return false; + } + projection = ChatReadProjection{}; + projection.conversation.protocol = protocol; + projection.conversation.channel = static_cast(prefix.channel); + projection.conversation.peer = prefix.peer; + projection.last_read_sequence = prefix.last_read_sequence; + projection.deleted = (prefix.flags & kProjectionDeleted) != 0; + return true; +} + +} // namespace + +uint32_t crc32(const void* data, std::size_t len) +{ + const auto* bytes = static_cast(data); + uint32_t crc = 0xFFFFFFFFU; + for (std::size_t index = 0; index < len; ++index) + { + crc ^= bytes[index]; + for (uint8_t bit = 0; bit < 8; ++bit) + { + crc = (crc & 1U) != 0U ? (crc >> 1U) ^ 0xEDB88320U + : crc >> 1U; + } + } + return ~crc; +} + +MeshProtocol canonicalProtocol(MeshProtocol protocol) +{ + return protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum : protocol; +} + +bool supportedProtocol(MeshProtocol protocol) +{ + switch (canonicalProtocol(protocol)) + { + case MeshProtocol::Meshtastic: + case MeshProtocol::MeshCore: + case MeshProtocol::Reticulum: + return true; + default: + return false; + } +} + +std::size_t messageSlotSize(MeshProtocol protocol) +{ + switch (canonicalProtocol(protocol)) + { + case MeshProtocol::Meshtastic: + return sizeof(MeshtasticMessageSlot); + case MeshProtocol::MeshCore: + return sizeof(MeshCoreMessageSlot); + case MeshProtocol::Reticulum: + return sizeof(ReticulumMessageSlot); + default: + return 0; + } +} + +std::size_t catalogSlotSize(MeshProtocol protocol) +{ + return canonicalProtocol(protocol) == MeshProtocol::Reticulum + ? sizeof(ReticulumCatalogSlot) + : supportedProtocol(protocol) ? sizeof(NodeCatalogSlot) + : 0; +} + +std::size_t readStateSlotSize(MeshProtocol protocol) +{ + return canonicalProtocol(protocol) == MeshProtocol::Reticulum + ? sizeof(ReticulumReadSlot) + : supportedProtocol(protocol) ? sizeof(NodeReadSlot) + : 0; +} + +std::size_t statusSlotSize() +{ + return sizeof(StatusSlot); +} + +std::size_t reticulumSeenSlotSize() +{ + return sizeof(SeenSlot); +} + +bool encodeMessageSlot(const ChatMessage& message, + uint32_t sequence, + void* out, + std::size_t out_len) +{ + switch (canonicalProtocol(message.protocol)) + { + case MeshProtocol::Meshtastic: + return encodeNodeMessage(message, + sequence, + out, + out_len); + case MeshProtocol::MeshCore: + return encodeNodeMessage(message, + sequence, + out, + out_len); + case MeshProtocol::Reticulum: + { + if (!out || out_len != sizeof(ReticulumMessageSlot) || + message.text.size() > kReticulumTextMax) + { + return false; + } + ReticulumMessageSlot slot{}; + encodeMessagePrefix(slot.prefix, + message, + sequence, + message.text.size()); + if (hasReticulumDestinationIdentity(message.reticulum_identity)) + { + slot.prefix.flags |= kMessageHasReticulumIdentity; + std::memcpy(slot.destination_hash, + message.reticulum_identity.destination_hash, + sizeof(slot.destination_hash)); + std::memcpy(slot.identity_hash, + message.reticulum_identity.identity_hash, + sizeof(slot.identity_hash)); + } + if (hasReticulumLxmfMessageHash(message)) + { + slot.prefix.flags |= kMessageHasLxmfHash; + std::memcpy(slot.lxmf_hash, + message.reticulum_lxmf_hash, + sizeof(slot.lxmf_hash)); + } + if (!message.text.empty()) + { + std::memcpy(slot.text, + message.text.data(), + message.text.size()); + } + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + default: + return false; + } +} + +bool decodeMessageSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatMessage& out_message, + uint32_t* out_sequence) +{ + protocol = canonicalProtocol(protocol); + if (protocol == MeshProtocol::Meshtastic) + { + return decodeNodeMessage(protocol, + data, + len, + out_message, + out_sequence); + } + if (protocol == MeshProtocol::MeshCore) + { + return decodeNodeMessage(protocol, + data, + len, + out_message, + out_sequence); + } + if (protocol != MeshProtocol::Reticulum || !data || + len != sizeof(ReticulumMessageSlot)) + { + return false; + } + ReticulumMessageSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kMessageMagic) || + !decodeMessagePrefix(slot.prefix, + sizeof(slot.text), + out_message, + out_sequence)) + { + return false; + } + out_message.protocol = protocol; + out_message.text.assign(slot.text, slot.prefix.text_len); + if ((slot.prefix.flags & kMessageHasReticulumIdentity) != 0) + { + out_message.reticulum_identity = + makeReticulumPeerIdentity(slot.destination_hash, + slot.identity_hash); + } + if ((slot.prefix.flags & kMessageHasLxmfHash) != 0) + { + out_message.has_reticulum_lxmf_hash = true; + std::memcpy(out_message.reticulum_lxmf_hash, + slot.lxmf_hash, + sizeof(slot.lxmf_hash)); + } + return true; +} + +bool encodeCatalogSlot(MeshProtocol protocol, + const ChatCatalogProjection& projection, + void* out, + std::size_t out_len) +{ + protocol = canonicalProtocol(protocol); + if (protocol == MeshProtocol::Reticulum) + { + if (!out || out_len != sizeof(ReticulumCatalogSlot)) + { + return false; + } + ReticulumCatalogSlot slot{}; + encodeCatalogPrefix(slot.prefix, projection); + if (hasReticulumDestinationIdentity( + projection.conversation.reticulum_identity)) + { + slot.prefix.flags |= kProjectionHasReticulumIdentity; + std::memcpy(slot.destination_hash, + projection.conversation.reticulum_identity + .destination_hash, + sizeof(slot.destination_hash)); + std::memcpy(slot.identity_hash, + projection.conversation.reticulum_identity + .identity_hash, + sizeof(slot.identity_hash)); + } + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (!supportedProtocol(protocol) || !out || + out_len != sizeof(NodeCatalogSlot)) + { + return false; + } + NodeCatalogSlot slot{}; + encodeCatalogPrefix(slot.prefix, projection); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; +} + +bool decodeCatalogSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatCatalogProjection& out_projection) +{ + protocol = canonicalProtocol(protocol); + if (protocol == MeshProtocol::Reticulum) + { + if (!data || len != sizeof(ReticulumCatalogSlot)) + { + return false; + } + ReticulumCatalogSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kCatalogMagic) || + !decodeCatalogPrefix(protocol, slot.prefix, out_projection)) + { + return false; + } + if ((slot.prefix.flags & kProjectionHasReticulumIdentity) != 0) + { + out_projection.conversation.reticulum_identity = + makeReticulumPeerIdentity(slot.destination_hash, + slot.identity_hash); + } + return true; + } + if (!data || len != sizeof(NodeCatalogSlot) || + !supportedProtocol(protocol)) + { + return false; + } + NodeCatalogSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + return validSlot(slot, kCatalogMagic) && + decodeCatalogPrefix(protocol, slot.prefix, out_projection); +} + +bool encodeReadStateSlot(MeshProtocol protocol, + const ChatReadProjection& projection, + void* out, + std::size_t out_len) +{ + protocol = canonicalProtocol(protocol); + if (protocol == MeshProtocol::Reticulum) + { + if (!out || out_len != sizeof(ReticulumReadSlot) || + !hasReticulumDestinationIdentity( + projection.conversation.reticulum_identity)) + { + return false; + } + ReticulumReadSlot slot{}; + encodeReadPrefix(slot.prefix, projection); + slot.prefix.flags |= kProjectionHasReticulumIdentity; + std::memcpy(slot.destination_hash, + projection.conversation.reticulum_identity + .destination_hash, + sizeof(slot.destination_hash)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (!supportedProtocol(protocol) || !out || + out_len != sizeof(NodeReadSlot)) + { + return false; + } + NodeReadSlot slot{}; + encodeReadPrefix(slot.prefix, projection); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; +} + +bool decodeReadStateSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ChatReadProjection& out_projection) +{ + protocol = canonicalProtocol(protocol); + if (protocol == MeshProtocol::Reticulum) + { + if (!data || len != sizeof(ReticulumReadSlot)) + { + return false; + } + ReticulumReadSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kReadMagic) || + !decodeReadPrefix(protocol, slot.prefix, out_projection) || + (slot.prefix.flags & kProjectionHasReticulumIdentity) == 0) + { + return false; + } + out_projection.conversation.reticulum_identity = + makeReticulumDestinationIdentity(slot.destination_hash); + return true; + } + if (!data || len != sizeof(NodeReadSlot) || + !supportedProtocol(protocol)) + { + return false; + } + NodeReadSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + return validSlot(slot, kReadMagic) && + decodeReadPrefix(protocol, slot.prefix, out_projection); +} + +bool encodeStatusSlot(const ChatStatusProjection& projection, + void* out, + std::size_t out_len) +{ + if (!out || out_len != sizeof(StatusSlot) || + projection.message_id == 0) + { + return false; + } + StatusSlot slot{}; + slot.status = static_cast(projection.status); + slot.message_id = projection.message_id; + slot.sequence = projection.sequence; + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; +} + +bool decodeStatusSlot(const void* data, + std::size_t len, + ChatStatusProjection& out_projection) +{ + if (!data || len != sizeof(StatusSlot)) + { + return false; + } + StatusSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validStatusSlot(slot)) + { + return false; + } + out_projection.message_id = slot.message_id; + out_projection.status = static_cast(slot.status); + out_projection.sequence = slot.sequence; + return true; +} + +bool encodeReticulumSeenSlot(const ReticulumSeenProjection& projection, + void* out, + std::size_t out_len) +{ + if (!out || out_len != sizeof(SeenSlot) || + isAllZeroKeyBytes(projection.hash, sizeof(projection.hash))) + { + return false; + } + SeenSlot slot{}; + std::memcpy(slot.hash, projection.hash, sizeof(slot.hash)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; +} + +bool decodeReticulumSeenSlot(const void* data, + std::size_t len, + ReticulumSeenProjection& out_projection) +{ + if (!data || len != sizeof(SeenSlot)) + { + return false; + } + SeenSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSeenSlot(slot)) + { + return false; + } + std::memcpy(out_projection.hash, slot.hash, sizeof(slot.hash)); + return true; +} + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp b/platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp new file mode 100644 index 00000000..aace1631 --- /dev/null +++ b/platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp @@ -0,0 +1,724 @@ +#include "platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h" + +#include "platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h" + +#include +#include + +namespace chat::storage::v2 +{ +namespace +{ + +constexpr uint32_t kPeerMagic = 0x32524550U; // PER2 +constexpr uint32_t kContactMagic = 0x32544E43U; // CNT2 + +constexpr uint16_t kDeleted = 0x0001U; +constexpr uint16_t kFavorite = 0x0002U; +constexpr uint16_t kIgnored = 0x0004U; +constexpr uint16_t kTrusted = 0x0008U; +constexpr uint16_t kHasSnr = 0x0010U; +constexpr uint16_t kHasRssi = 0x0020U; +constexpr uint16_t kHasMetrics = 0x0040U; +constexpr uint16_t kHasPosition = 0x0080U; + +constexpr uint8_t kHasBattery = 0x01U; +constexpr uint8_t kHasVoltage = 0x02U; +constexpr uint8_t kHasChannelUtilization = 0x04U; +constexpr uint8_t kHasAirUtilization = 0x08U; +constexpr uint8_t kHasUptime = 0x10U; +constexpr uint8_t kPositionValid = 0x01U; +constexpr uint8_t kPositionHasAltitude = 0x02U; + +struct PeerPrefix +{ + uint32_t magic = kPeerMagic; + uint16_t schema = kStorageSchemaVersion; + uint16_t flags = 0; + uint8_t source = 0; + uint8_t identity_kind = 0; + uint16_t reserved = 0; + uint32_t node_id = 0; + uint32_t first_seen_s = 0; + uint32_t last_seen_s = 0; + char display_name[kMeshPeerDisplayNameMaxLen] = {}; + float snr = 0.0f; + float rssi = 0.0f; + uint8_t metrics_flags = 0; + uint8_t position_flags = 0; + uint16_t reserved2 = 0; + uint32_t battery_level = 0; + float voltage = 0.0f; + float channel_utilization = 0.0f; + float air_util_tx = 0.0f; + uint32_t uptime_seconds = 0; + int32_t latitude_i = 0; + int32_t longitude_i = 0; + int32_t altitude = 0; + uint32_t position_timestamp = 0; + uint32_t precision_bits = 0; + uint32_t pdop = 0; + uint32_t hdop = 0; + uint32_t vdop = 0; + uint32_t gps_accuracy_mm = 0; +} __attribute__((packed)); + +struct NodeFactsSlot +{ + char short_name[kMeshPeerShortNameMaxLen] = {}; + char long_name[kMeshPeerDisplayNameMaxLen] = {}; + uint8_t role = 0xFF; + uint8_t hw_model = 0; + uint8_t channel = 0xFF; + uint8_t hops_away = 0xFF; + uint8_t flags = 0; + uint8_t macaddr[kMeshPeerMacAddrLen] = {}; +} __attribute__((packed)); + +struct MeshtasticPeerSlot +{ + PeerPrefix prefix{}; + NodeFactsSlot node{}; + uint8_t has_public_key = 0; + uint8_t key_verified = 0; + uint8_t has_next_hop = 0; + uint8_t next_hop = 0; + uint8_t public_key[kMeshPeerMeshtasticPublicKeyLen] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct MeshCorePeerSlot +{ + PeerPrefix prefix{}; + NodeFactsSlot node{}; + uint8_t has_public_key = 0; + uint8_t key_verified = 0; + uint8_t has_peer_hash = 0; + uint8_t peer_hash = 0; + uint8_t has_next_hop = 0; + uint8_t next_hop = 0; + uint16_t reserved = 0; + uint32_t node_id_hint = 0; + uint8_t public_key[kMeshPeerMeshCorePublicKeyLen] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReticulumPeerSlot +{ + PeerPrefix prefix{}; + uint8_t destination_hash[kReticulumPeerHashSize] = {}; + uint8_t identity_hash[kReticulumPeerHashSize] = {}; + uint8_t has_public_keys = 0; + uint8_t has_ratchet = 0; + uint8_t delivery = 0; + uint8_t propagation = 0; + uint8_t enc_pub[kMeshPeerReticulumPublicKeyLen] = {}; + uint8_t sig_pub[kMeshPeerReticulumPublicKeyLen] = {}; + uint8_t ratchet_pub[kMeshPeerReticulumRatchetLen] = {}; + uint32_t ratchet_seen_s = 0; + uint32_t crc = 0; +} __attribute__((packed)); + +struct NodeContactSlot +{ + uint32_t magic = kContactMagic; + uint16_t schema = kStorageSchemaVersion; + uint16_t flags = 0; + uint32_t node_id = 0; + char alias[kMeshPeerUserAliasMaxLen + 1] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct MeshCoreContactSlot +{ + uint32_t magic = kContactMagic; + uint16_t schema = kStorageSchemaVersion; + uint16_t flags = 0; + uint32_t node_id_hint = 0; + uint8_t public_key[kMeshPeerMeshCorePublicKeyLen] = {}; + char alias[kMeshPeerUserAliasMaxLen + 1] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +struct ReticulumContactSlot +{ + uint32_t magic = kContactMagic; + uint16_t schema = kStorageSchemaVersion; + uint16_t flags = 0; + uint8_t destination_hash[kReticulumPeerHashSize] = {}; + char alias[kMeshPeerUserAliasMaxLen + 1] = {}; + uint32_t crc = 0; +} __attribute__((packed)); + +template +bool validSlot(const T& slot, uint32_t magic) +{ + return slot.prefix.magic == magic && + slot.prefix.schema == kStorageSchemaVersion && + slot.crc == crc32(&slot, sizeof(slot) - sizeof(slot.crc)); +} + +template +bool validContactSlot(const T& slot) +{ + return slot.magic == kContactMagic && + slot.schema == kStorageSchemaVersion && + slot.crc == crc32(&slot, sizeof(slot) - sizeof(slot.crc)); +} + +uint16_t userFlags(const MeshPeerUserFlags& flags, bool deleted) +{ + uint16_t result = deleted ? kDeleted : 0U; + result |= flags.favorite ? kFavorite : 0U; + result |= flags.ignored ? kIgnored : 0U; + result |= flags.trusted ? kTrusted : 0U; + return result; +} + +MeshPeerUserFlags decodeUserFlags(uint16_t flags) +{ + MeshPeerUserFlags result{}; + result.favorite = (flags & kFavorite) != 0U; + result.ignored = (flags & kIgnored) != 0U; + result.trusted = (flags & kTrusted) != 0U; + return result; +} + +void encodeNodeFacts(NodeFactsSlot& out, const MeshPeerNodeFacts& facts) +{ + std::memcpy(out.short_name, facts.short_name, sizeof(out.short_name)); + std::memcpy(out.long_name, facts.long_name, sizeof(out.long_name)); + out.role = facts.role; + out.hw_model = facts.hw_model; + out.channel = facts.channel; + out.hops_away = facts.hops_away; + out.flags = facts.has_macaddr ? 0x01U : 0U; + out.flags |= facts.via_mqtt ? 0x02U : 0U; + std::memcpy(out.macaddr, facts.macaddr, sizeof(out.macaddr)); +} + +void decodeNodeFacts(MeshPeerNodeFacts& out, const NodeFactsSlot& facts) +{ + std::memcpy(out.short_name, facts.short_name, sizeof(out.short_name)); + out.short_name[sizeof(out.short_name) - 1U] = '\0'; + std::memcpy(out.long_name, facts.long_name, sizeof(out.long_name)); + out.long_name[sizeof(out.long_name) - 1U] = '\0'; + out.role = facts.role; + out.hw_model = facts.hw_model; + out.channel = facts.channel; + out.hops_away = facts.hops_away; + out.has_macaddr = (facts.flags & 0x01U) != 0U; + out.via_mqtt = (facts.flags & 0x02U) != 0U; + std::memcpy(out.macaddr, facts.macaddr, sizeof(out.macaddr)); +} + +void encodePrefix(PeerPrefix& out, const PeerProjection& projection) +{ + const MeshPeerRecord& record = projection.record; + out.flags = projection.deleted ? kDeleted : 0U; + out.source = static_cast(record.source); + out.identity_kind = static_cast(record.identity.kind); + out.node_id = record.identity.node_id; + out.first_seen_s = record.first_seen_s; + out.last_seen_s = record.last_seen_s; + std::memcpy(out.display_name, + record.display_name, + sizeof(out.display_name)); + const MeshPeerObservations& observations = record.observations; + if (observations.has_snr) + { + out.flags |= kHasSnr; + out.snr = observations.snr; + } + if (observations.has_rssi) + { + out.flags |= kHasRssi; + out.rssi = observations.rssi; + } + if (observations.has_device_metrics) + { + out.flags |= kHasMetrics; + const contacts::NodeDeviceMetrics& metrics = + observations.device_metrics; + out.metrics_flags |= metrics.has_battery_level ? kHasBattery : 0U; + out.metrics_flags |= metrics.has_voltage ? kHasVoltage : 0U; + out.metrics_flags |= metrics.has_channel_utilization + ? kHasChannelUtilization + : 0U; + out.metrics_flags |= metrics.has_air_util_tx ? kHasAirUtilization : 0U; + out.metrics_flags |= metrics.has_uptime_seconds ? kHasUptime : 0U; + out.battery_level = metrics.battery_level; + out.voltage = metrics.voltage; + out.channel_utilization = metrics.channel_utilization; + out.air_util_tx = metrics.air_util_tx; + out.uptime_seconds = metrics.uptime_seconds; + } + if (observations.has_position) + { + out.flags |= kHasPosition; + const contacts::NodePosition& position = observations.position; + out.position_flags = position.valid ? kPositionValid : 0U; + out.position_flags |= position.has_altitude ? kPositionHasAltitude : 0U; + out.latitude_i = position.latitude_i; + out.longitude_i = position.longitude_i; + out.altitude = position.altitude; + out.position_timestamp = position.timestamp; + out.precision_bits = position.precision_bits; + out.pdop = position.pdop; + out.hdop = position.hdop; + out.vdop = position.vdop; + out.gps_accuracy_mm = position.gps_accuracy_mm; + } +} + +void decodePrefix(MeshProtocol protocol, + const PeerPrefix& prefix, + PeerProjection& projection) +{ + projection = PeerProjection{}; + projection.deleted = (prefix.flags & kDeleted) != 0U; + MeshPeerRecord& record = projection.record; + record.valid = true; + record.identity.protocol = canonicalProtocol(protocol); + record.identity.kind = + static_cast(prefix.identity_kind); + record.identity.node_id = prefix.node_id; + record.source = static_cast(prefix.source); + record.first_seen_s = prefix.first_seen_s; + record.last_seen_s = prefix.last_seen_s; + std::memcpy(record.display_name, + prefix.display_name, + sizeof(record.display_name)); + record.display_name[sizeof(record.display_name) - 1U] = '\0'; + record.observations.has_snr = (prefix.flags & kHasSnr) != 0U; + record.observations.snr = prefix.snr; + record.observations.has_rssi = (prefix.flags & kHasRssi) != 0U; + record.observations.rssi = prefix.rssi; + record.observations.has_device_metrics = + (prefix.flags & kHasMetrics) != 0U; + contacts::NodeDeviceMetrics& metrics = + record.observations.device_metrics; + metrics.has_battery_level = (prefix.metrics_flags & kHasBattery) != 0U; + metrics.battery_level = prefix.battery_level; + metrics.has_voltage = (prefix.metrics_flags & kHasVoltage) != 0U; + metrics.voltage = prefix.voltage; + metrics.has_channel_utilization = + (prefix.metrics_flags & kHasChannelUtilization) != 0U; + metrics.channel_utilization = prefix.channel_utilization; + metrics.has_air_util_tx = + (prefix.metrics_flags & kHasAirUtilization) != 0U; + metrics.air_util_tx = prefix.air_util_tx; + metrics.has_uptime_seconds = (prefix.metrics_flags & kHasUptime) != 0U; + metrics.uptime_seconds = prefix.uptime_seconds; + record.observations.has_position = + (prefix.flags & kHasPosition) != 0U; + contacts::NodePosition& position = record.observations.position; + position.valid = (prefix.position_flags & kPositionValid) != 0U; + position.has_altitude = + (prefix.position_flags & kPositionHasAltitude) != 0U; + position.latitude_i = prefix.latitude_i; + position.longitude_i = prefix.longitude_i; + position.altitude = prefix.altitude; + position.timestamp = prefix.position_timestamp; + position.precision_bits = prefix.precision_bits; + position.pdop = prefix.pdop; + position.hdop = prefix.hdop; + position.vdop = prefix.vdop; + position.gps_accuracy_mm = prefix.gps_accuracy_mm; +} + +bool validPeerIdentityForProtocol(MeshProtocol protocol, + const MeshPeerIdentity& identity) +{ + protocol = canonicalProtocol(protocol); + return meshPeerSameProtocol(protocol, identity.protocol) && + ((protocol == MeshProtocol::Meshtastic && + identity.kind == MeshPeerIdentityKind::NodeId) || + (protocol == MeshProtocol::MeshCore && + ((identity.kind == MeshPeerIdentityKind::NodeId && + identity.node_id != 0U) || + (identity.kind == MeshPeerIdentityKind::PublicKey && + identity.public_key_len == kMeshPeerMeshCorePublicKeyLen))) || + (protocol == MeshProtocol::Reticulum && + ((identity.kind == MeshPeerIdentityKind::NodeId && + identity.node_id != 0U) || + identity.kind == MeshPeerIdentityKind::ReticulumDestination))); +} + +bool validContactIdentityForProtocol(MeshProtocol protocol, + const MeshPeerIdentity& identity) +{ + protocol = canonicalProtocol(protocol); + return meshPeerSameProtocol(protocol, identity.protocol) && + ((protocol == MeshProtocol::Meshtastic && + identity.kind == MeshPeerIdentityKind::NodeId) || + (protocol == MeshProtocol::MeshCore && + identity.kind == MeshPeerIdentityKind::PublicKey && + identity.public_key_len == kMeshPeerMeshCorePublicKeyLen) || + (protocol == MeshProtocol::Reticulum && + identity.kind == MeshPeerIdentityKind::ReticulumDestination)); +} + +} // namespace + +std::size_t peerSlotSize(MeshProtocol protocol) +{ + switch (canonicalProtocol(protocol)) + { + case MeshProtocol::Meshtastic: + return sizeof(MeshtasticPeerSlot); + case MeshProtocol::MeshCore: + return sizeof(MeshCorePeerSlot); + case MeshProtocol::Reticulum: + return sizeof(ReticulumPeerSlot); + default: + return 0U; + } +} + +std::size_t contactSlotSize(MeshProtocol protocol) +{ + switch (canonicalProtocol(protocol)) + { + case MeshProtocol::Meshtastic: + return sizeof(NodeContactSlot); + case MeshProtocol::MeshCore: + return sizeof(MeshCoreContactSlot); + case MeshProtocol::Reticulum: + return sizeof(ReticulumContactSlot); + default: + return 0U; + } +} + +bool encodePeerSlot(MeshProtocol protocol, + const PeerProjection& projection, + void* out, + std::size_t out_len) +{ + protocol = canonicalProtocol(protocol); + if (!out || !meshPeerRecordIsValid(projection.record) || + !validPeerIdentityForProtocol(protocol, projection.record.identity)) + { + return false; + } + if (protocol == MeshProtocol::Meshtastic && + out_len == sizeof(MeshtasticPeerSlot)) + { + MeshtasticPeerSlot slot{}; + encodePrefix(slot.prefix, projection); + encodeNodeFacts(slot.node, projection.record.meshtastic.node); + slot.has_public_key = projection.record.meshtastic.has_public_key ? 1U : 0U; + slot.key_verified = + projection.record.meshtastic.key_manually_verified ? 1U : 0U; + slot.has_next_hop = + projection.record.meshtastic.has_next_hop ? 1U : 0U; + slot.next_hop = projection.record.meshtastic.next_hop; + std::memcpy(slot.public_key, + projection.record.meshtastic.public_key, + sizeof(slot.public_key)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (protocol == MeshProtocol::MeshCore && + out_len == sizeof(MeshCorePeerSlot)) + { + MeshCorePeerSlot slot{}; + encodePrefix(slot.prefix, projection); + encodeNodeFacts(slot.node, projection.record.meshcore.node); + slot.has_public_key = projection.record.meshcore.has_public_key ? 1U : 0U; + slot.key_verified = + projection.record.meshcore.public_key_verified ? 1U : 0U; + slot.has_peer_hash = projection.record.meshcore.has_peer_hash ? 1U : 0U; + slot.peer_hash = projection.record.meshcore.peer_hash; + slot.has_next_hop = projection.record.meshcore.has_next_hop ? 1U : 0U; + slot.next_hop = projection.record.meshcore.next_hop; + slot.node_id_hint = projection.record.meshcore.node_id_hint; + if (projection.record.identity.kind == + MeshPeerIdentityKind::PublicKey) + { + std::memcpy(slot.public_key, + projection.record.identity.public_key, + sizeof(slot.public_key)); + } + else if (projection.record.meshcore.has_public_key) + { + std::memcpy(slot.public_key, + projection.record.meshcore.public_key, + sizeof(slot.public_key)); + } + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (protocol == MeshProtocol::Reticulum && + out_len == sizeof(ReticulumPeerSlot)) + { + ReticulumPeerSlot slot{}; + encodePrefix(slot.prefix, projection); + const ReticulumPeerFacts& facts = projection.record.reticulum; + if (projection.record.identity.kind == + MeshPeerIdentityKind::ReticulumDestination) + { + (void)copyReticulumIdentityHashes( + slot.destination_hash, + slot.identity_hash, + projection.record.identity.reticulum); + } + slot.has_public_keys = facts.has_public_keys ? 1U : 0U; + slot.has_ratchet = facts.has_ratchet ? 1U : 0U; + slot.delivery = facts.delivery ? 1U : 0U; + slot.propagation = facts.propagation ? 1U : 0U; + std::memcpy(slot.enc_pub, facts.enc_pub, sizeof(slot.enc_pub)); + std::memcpy(slot.sig_pub, facts.sig_pub, sizeof(slot.sig_pub)); + std::memcpy(slot.ratchet_pub, + facts.ratchet_pub, + sizeof(slot.ratchet_pub)); + slot.ratchet_seen_s = facts.ratchet_seen_s; + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + return false; +} + +bool decodePeerSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + PeerProjection& out_projection) +{ + protocol = canonicalProtocol(protocol); + if (!data) + { + return false; + } + if (protocol == MeshProtocol::Meshtastic && + len == sizeof(MeshtasticPeerSlot)) + { + MeshtasticPeerSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kPeerMagic)) + { + return false; + } + decodePrefix(protocol, slot.prefix, out_projection); + out_projection.record.identity = makeMeshPeerNodeIdentity( + protocol, + slot.prefix.node_id); + decodeNodeFacts(out_projection.record.meshtastic.node, slot.node); + out_projection.record.meshtastic.has_public_key = slot.has_public_key != 0U; + out_projection.record.meshtastic.key_manually_verified = + slot.key_verified != 0U; + out_projection.record.meshtastic.has_next_hop = + slot.has_next_hop != 0U; + out_projection.record.meshtastic.next_hop = slot.next_hop; + std::memcpy(out_projection.record.meshtastic.public_key, + slot.public_key, + sizeof(slot.public_key)); + return meshPeerRecordIsValid(out_projection.record); + } + if (protocol == MeshProtocol::MeshCore && + len == sizeof(MeshCorePeerSlot)) + { + MeshCorePeerSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kPeerMagic)) + { + return false; + } + MeshPeerIdentity identity{}; + if (static_cast(slot.prefix.identity_kind) == + MeshPeerIdentityKind::NodeId) + { + identity = makeMeshPeerNodeIdentity(protocol, + slot.prefix.node_id); + } + else if (!makeMeshPeerPublicKeyIdentity(protocol, + slot.public_key, + sizeof(slot.public_key), + identity)) + { + return false; + } + decodePrefix(protocol, slot.prefix, out_projection); + out_projection.record.identity = identity; + decodeNodeFacts(out_projection.record.meshcore.node, slot.node); + out_projection.record.meshcore.has_public_key = slot.has_public_key != 0U; + out_projection.record.meshcore.public_key_verified = + slot.key_verified != 0U; + out_projection.record.meshcore.has_peer_hash = slot.has_peer_hash != 0U; + out_projection.record.meshcore.peer_hash = slot.peer_hash; + out_projection.record.meshcore.has_next_hop = slot.has_next_hop != 0U; + out_projection.record.meshcore.next_hop = slot.next_hop; + out_projection.record.meshcore.node_id_hint = slot.node_id_hint; + std::memcpy(out_projection.record.meshcore.public_key, + slot.public_key, + sizeof(slot.public_key)); + return meshPeerRecordIsValid(out_projection.record); + } + if (protocol == MeshProtocol::Reticulum && + len == sizeof(ReticulumPeerSlot)) + { + ReticulumPeerSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validSlot(slot, kPeerMagic)) + { + return false; + } + decodePrefix(protocol, slot.prefix, out_projection); + if (static_cast(slot.prefix.identity_kind) == + MeshPeerIdentityKind::NodeId) + { + out_projection.record.identity = makeMeshPeerNodeIdentity( + protocol, + slot.prefix.node_id); + } + else + { + out_projection.record.identity = makeMeshPeerReticulumIdentity( + makeReticulumPeerIdentity(slot.destination_hash, + slot.identity_hash)); + } + ReticulumPeerFacts& facts = out_projection.record.reticulum; + if (out_projection.record.identity.kind == + MeshPeerIdentityKind::ReticulumDestination) + { + facts.identity = out_projection.record.identity.reticulum; + } + facts.has_public_keys = slot.has_public_keys != 0U; + facts.has_ratchet = slot.has_ratchet != 0U; + facts.delivery = slot.delivery != 0U; + facts.propagation = slot.propagation != 0U; + std::memcpy(facts.enc_pub, slot.enc_pub, sizeof(facts.enc_pub)); + std::memcpy(facts.sig_pub, slot.sig_pub, sizeof(facts.sig_pub)); + std::memcpy(facts.ratchet_pub, + slot.ratchet_pub, + sizeof(facts.ratchet_pub)); + facts.ratchet_seen_s = slot.ratchet_seen_s; + return meshPeerRecordIsValid(out_projection.record); + } + return false; +} + +bool encodeContactSlot(MeshProtocol protocol, + const ContactProjection& projection, + void* out, + std::size_t out_len) +{ + protocol = canonicalProtocol(protocol); + if (!out || + !validContactIdentityForProtocol(protocol, projection.identity)) + { + return false; + } + const uint16_t flags = userFlags(projection.flags, projection.deleted); + if (protocol == MeshProtocol::Meshtastic && + out_len == sizeof(NodeContactSlot)) + { + NodeContactSlot slot{}; + slot.flags = flags; + slot.node_id = projection.identity.node_id; + std::memcpy(slot.alias, projection.alias, sizeof(slot.alias)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (protocol == MeshProtocol::MeshCore && + out_len == sizeof(MeshCoreContactSlot)) + { + MeshCoreContactSlot slot{}; + slot.flags = flags; + slot.node_id_hint = projection.node_id_hint; + std::memcpy(slot.public_key, + projection.identity.public_key, + sizeof(slot.public_key)); + std::memcpy(slot.alias, projection.alias, sizeof(slot.alias)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + if (protocol == MeshProtocol::Reticulum && + out_len == sizeof(ReticulumContactSlot)) + { + ReticulumContactSlot slot{}; + slot.flags = flags; + std::memcpy(slot.destination_hash, + projection.identity.reticulum.destination_hash, + sizeof(slot.destination_hash)); + std::memcpy(slot.alias, projection.alias, sizeof(slot.alias)); + slot.crc = crc32(&slot, sizeof(slot) - sizeof(slot.crc)); + std::memcpy(out, &slot, sizeof(slot)); + return true; + } + return false; +} + +bool decodeContactSlot(MeshProtocol protocol, + const void* data, + std::size_t len, + ContactProjection& out_projection) +{ + protocol = canonicalProtocol(protocol); + out_projection = ContactProjection{}; + if (!data) + { + return false; + } + uint16_t flags = 0; + if (protocol == MeshProtocol::Meshtastic && + len == sizeof(NodeContactSlot)) + { + NodeContactSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validContactSlot(slot)) + { + return false; + } + flags = slot.flags; + out_projection.identity = makeMeshPeerNodeIdentity(protocol, + slot.node_id); + std::memcpy(out_projection.alias, slot.alias, sizeof(slot.alias)); + } + else if (protocol == MeshProtocol::MeshCore && + len == sizeof(MeshCoreContactSlot)) + { + MeshCoreContactSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validContactSlot(slot) || + !makeMeshPeerPublicKeyIdentity(protocol, + slot.public_key, + sizeof(slot.public_key), + out_projection.identity)) + { + return false; + } + flags = slot.flags; + out_projection.node_id_hint = slot.node_id_hint; + std::memcpy(out_projection.alias, slot.alias, sizeof(slot.alias)); + } + else if (protocol == MeshProtocol::Reticulum && + len == sizeof(ReticulumContactSlot)) + { + ReticulumContactSlot slot{}; + std::memcpy(&slot, data, sizeof(slot)); + if (!validContactSlot(slot)) + { + return false; + } + flags = slot.flags; + out_projection.identity = makeMeshPeerReticulumIdentity( + makeReticulumDestinationIdentity(slot.destination_hash)); + std::memcpy(out_projection.alias, slot.alias, sizeof(slot.alias)); + } + else + { + return false; + } + out_projection.alias[sizeof(out_projection.alias) - 1U] = '\0'; + out_projection.flags = decodeUserFlags(flags); + out_projection.deleted = (flags & kDeleted) != 0U; + return meshPeerIdentityIsValid(out_projection.identity); +} + +} // namespace chat::storage::v2 diff --git a/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp b/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp new file mode 100644 index 00000000..078fc0cf --- /dev/null +++ b/platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp @@ -0,0 +1,2195 @@ +#include "platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h" + +#include "platform/esp/arduino_common/storage/persistence_bus_gate.h" +#include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/common/shared_spi_bus_arbiter.h" + +#include + +#include +#include +#include +#include + +namespace chat +{ +namespace +{ +namespace storage_runtime = ::platform::esp::arduino_common::storage; +namespace storage_v2 = ::chat::storage::v2; + +constexpr const char* kRoot = "/data/v2"; +constexpr MeshProtocol kProtocols[] = { + MeshProtocol::Meshtastic, + MeshProtocol::MeshCore, + MeshProtocol::Reticulum, +}; +constexpr std::size_t kEphemeralPeerCapacity = 2048U; +constexpr std::size_t kProtectedContactCapacity = 4096U; +constexpr std::size_t kPeerHotCacheCapacity[] = {16U, 128U, 64U}; +constexpr uint32_t kBootCompactionDeltaThreshold = 1024U; +constexpr std::size_t kPendingFlushBudget = 4U; +constexpr uint32_t kPeerStoreBusResource = 5U; +constexpr uint32_t kPeerStoreBusOwnerId = 0x50454552U; // PEER +constexpr const char* kPeerStoreBusOwner = "protocol_peer_v2"; + +::platform::esp::common::SharedSpiBusAdapter s_peer_store_bus_adapter( + kPeerStoreBusOwner, + kPeerStoreBusOwnerId); +::platform::esp::common::FixedSharedSpiBusPolicyStrategy s_peer_store_bus_policy( + 0, + 0, + 0, + 0); +sys::runtime::StorageBusArbiter s_peer_store_bus_arbiter( + s_peer_store_bus_adapter, + s_peer_store_bus_policy); + +class ScopedRepositoryLock final +{ + public: + explicit ScopedRepositoryLock(SemaphoreHandle_t mutex) + : mutex_(mutex), locked_(mutex_ && xSemaphoreTake(mutex_, portMAX_DELAY) == pdTRUE) + { + } + + ~ScopedRepositoryLock() + { + if (locked_) + { + xSemaphoreGive(mutex_); + } + } + + bool locked() const + { + return locked_; + } + + private: + SemaphoreHandle_t mutex_ = nullptr; + bool locked_ = false; +}; + +bool hasText(const char* text) +{ + return text && text[0] != '\0'; +} + +bool textContains(const char* text, const char* query) +{ + return text && query && std::strstr(text, query) != nullptr; +} + +void copyNodeText(char* out, std::size_t out_len, const char* text) +{ + copyMeshPeerText(out, out_len, text ? text : ""); +} + +MeshPeerNodeFacts& mutableNodeFacts(MeshPeerRecord& record) +{ + return record.identity.protocol == MeshProtocol::MeshCore + ? record.meshcore.node + : record.meshtastic.node; +} + +const MeshPeerNodeFacts* nodeFacts(const MeshPeerRecord& record) +{ + if (record.identity.protocol == MeshProtocol::Meshtastic) + { + return &record.meshtastic.node; + } + if (record.identity.protocol == MeshProtocol::MeshCore) + { + return &record.meshcore.node; + } + return nullptr; +} + +} // namespace + +SdProtocolPeerRepository::SdProtocolPeerRepository(IChatStore& chat_store) + : chat_store_(chat_store), + mutex_(xSemaphoreCreateMutex()), + node_store_view_(*this), + contact_store_view_(*this) +{ + peers_.reserve(256U); + contacts_.reserve(64U); + pending_peer_deltas_.reserve(16U); + slot_scratch_.resize(512U, 0U); +} + +SdProtocolPeerRepository::~SdProtocolPeerRepository() +{ + if (mutex_) + { + vSemaphoreDelete(mutex_); + mutex_ = nullptr; + } +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::begin() +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + if (begun_) + { + return MeshPeerDirectoryStatus::success(); + } + + storage_runtime::PersistenceBusGate bus_gate( + s_peer_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 3000U, + kPeerStoreBusResource, + kPeerStoreBusOwnerId + 1U, + kPeerStoreBusOwnerId); + if (!bus_gate.locked() || !ensureLayout()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + + peers_.clear(); + contacts_.clear(); + pending_peer_deltas_.clear(); + pending_peer_head_ = 0U; + std::memset(partitions_, 0, sizeof(partitions_)); + bool ok = true; + for (MeshProtocol protocol : kProtocols) + { + ok = loadProtocol(protocol) && ok; + } + if (!ok) + { + peers_.clear(); + contacts_.clear(); + return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); + } + + for (MeshProtocol protocol : kProtocols) + { + reconcileStableIdentities(protocol); + } + overlayContactFacts(); + for (MeshProtocol protocol : kProtocols) + { + if (!compactProtocolAtBoot(protocol)) + { + Serial.printf("[PeerStoreV2] compaction deferred protocol=%s\n", + protocolSlug(protocol)); + } + } + begun_ = true; + active_node_projection_dirty_ = true; + Serial.printf("[PeerStoreV2] ready=1 peers=%u contacts=%u root=%s\n", + static_cast(peers_.size()), + static_cast(contacts_.size()), + kRoot); + return MeshPeerDirectoryStatus::success(); +} + +bool SdProtocolPeerRepository::ensureLayout() +{ + if (!storage_runtime::sd_card_ready() || !ensureDirectory("/data") || + !ensureDirectory(kRoot)) + { + return false; + } + bool ok = true; + for (MeshProtocol protocol : kProtocols) + { + ok = ensureProtocolLayout(protocol) && ok; + } + return ok; +} + +bool SdProtocolPeerRepository::ensureProtocolLayout(MeshProtocol protocol) +{ + char path[64] = {}; + buildProtocolPath(protocol, nullptr, path, sizeof(path)); + return ensureDirectory(path); +} + +bool SdProtocolPeerRepository::loadProtocol(MeshProtocol protocol) +{ + for (const char* base : {"peers", "contacts"}) + { + char final_path[96] = {}; + char temp_path[96] = {}; + char backup_path[96] = {}; + char name[32] = {}; + std::snprintf(name, sizeof(name), "%s.snapshot", base); + buildProtocolPath(protocol, + name, + final_path, + sizeof(final_path)); + std::snprintf(name, sizeof(name), "%s.snapshot.tmp", base); + buildProtocolPath(protocol, + name, + temp_path, + sizeof(temp_path)); + std::snprintf(name, sizeof(name), "%s.snapshot.bak", base); + buildProtocolPath(protocol, + name, + backup_path, + sizeof(backup_path)); + if (!storage_v2::recoverAtomicFile(final_path, + temp_path, + backup_path)) + { + return false; + } + } + return loadPeerJournal(protocol, "peers.snapshot") && + loadPeerJournal(protocol, "peers.delta") && + loadContactJournal(protocol, "contacts.snapshot") && + loadContactJournal(protocol, "contacts.delta"); +} + +bool SdProtocolPeerRepository::loadPeerJournal(MeshProtocol protocol, + const char* name) +{ + char path[96] = {}; + buildProtocolPath(protocol, name, path, sizeof(path)); + const std::size_t slot_size = storage_v2::peerSlotSize(protocol); + const storage_v2::JournalKind kind = + std::strstr(name, ".snapshot") + ? storage_v2::JournalKind::PeerSnapshot + : storage_v2::JournalKind::PeerDelta; + const auto inspection = journal_.inspect(path, protocol, kind, slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) + { + return true; + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + Serial.printf("[PeerStoreV2] incompatible path=%s state=%u\n", + path, + static_cast(inspection.state)); + return false; + } + if (slot_size > slot_scratch_.size()) + { + return false; + } + for (uint32_t index = 0; index < inspection.slot_count; ++index) + { + storage_v2::PeerProjection projection{}; + if (!journal_.read(path, + protocol, + kind, + slot_size, + index, + slot_scratch_.data()) || + !storage_v2::decodePeerSlot(protocol, + slot_scratch_.data(), + slot_size, + projection)) + { + Serial.printf("[PeerStoreV2] corrupt peer slot path=%s index=%lu\n", + path, + static_cast(index)); + continue; + } + (void)applyPeerProjection(projection); + } + if (kind == storage_v2::JournalKind::PeerDelta) + { + partitions_[protocolIndex(protocol)].peer_delta_count = + inspection.slot_count; + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + Serial.printf("[PeerStoreV2] partial peer tail path=%s valid=%lu\n", + path, + static_cast(inspection.slot_count)); + } + return true; +} + +bool SdProtocolPeerRepository::loadContactJournal(MeshProtocol protocol, + const char* name) +{ + char path[96] = {}; + buildProtocolPath(protocol, name, path, sizeof(path)); + const std::size_t slot_size = storage_v2::contactSlotSize(protocol); + const storage_v2::JournalKind kind = + std::strstr(name, ".snapshot") + ? storage_v2::JournalKind::ContactSnapshot + : storage_v2::JournalKind::ContactDelta; + const auto inspection = journal_.inspect(path, protocol, kind, slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) + { + return true; + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + Serial.printf("[PeerStoreV2] incompatible path=%s state=%u\n", + path, + static_cast(inspection.state)); + return false; + } + if (slot_size > slot_scratch_.size()) + { + return false; + } + for (uint32_t index = 0; index < inspection.slot_count; ++index) + { + storage_v2::ContactProjection projection{}; + if (!journal_.read(path, + protocol, + kind, + slot_size, + index, + slot_scratch_.data()) || + !storage_v2::decodeContactSlot(protocol, + slot_scratch_.data(), + slot_size, + projection)) + { + Serial.printf("[PeerStoreV2] corrupt contact slot path=%s index=%lu\n", + path, + static_cast(index)); + continue; + } + (void)applyContactProjection(projection); + } + if (kind == storage_v2::JournalKind::ContactDelta) + { + partitions_[protocolIndex(protocol)].contact_delta_count = + inspection.slot_count; + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + Serial.printf("[PeerStoreV2] partial contact tail path=%s valid=%lu\n", + path, + static_cast(inspection.slot_count)); + } + return true; +} + +bool SdProtocolPeerRepository::compactProtocolAtBoot(MeshProtocol protocol) +{ + PartitionState& state = partitions_[protocolIndex(protocol)]; + bool ok = true; + if (state.peer_delta_count >= kBootCompactionDeltaThreshold) + { + ok = rewritePeerSnapshot(protocol) && ok; + } + if (state.contact_delta_count >= kBootCompactionDeltaThreshold) + { + ok = rewriteContactSnapshot(protocol) && ok; + } + return ok; +} + +bool SdProtocolPeerRepository::rewritePeerSnapshot(MeshProtocol protocol) +{ + char target[96] = {}; + char temp[96] = {}; + char backup[96] = {}; + char delta[96] = {}; + buildProtocolPath(protocol, "peers.snapshot", target, sizeof(target)); + buildProtocolPath(protocol, "peers.snapshot.tmp", temp, sizeof(temp)); + buildProtocolPath(protocol, + "peers.snapshot.bak", + backup, + sizeof(backup)); + buildProtocolPath(protocol, "peers.delta", delta, sizeof(delta)); + if (storage_runtime::sd_exists(temp)) + { + storage_runtime::sd_remove(temp); + } + const std::size_t slot_size = storage_v2::peerSlotSize(protocol); + if (!journal_.create(temp, + protocol, + storage_v2::JournalKind::PeerSnapshot, + slot_size)) + { + return false; + } + for (const MeshPeerRecord& peer : peers_) + { + if (!meshPeerSameProtocol(peer.identity.protocol, protocol)) + { + continue; + } + const storage_v2::PeerProjection projection{peer, false}; + if (!storage_v2::encodePeerSlot(protocol, + projection, + slot_scratch_.data(), + slot_size) || + !journal_.append(temp, + protocol, + storage_v2::JournalKind::PeerSnapshot, + slot_size, + slot_scratch_.data())) + { + storage_runtime::sd_remove(temp); + return false; + } + } + if (!storage_v2::replaceFileAtomically(temp, target, backup) || + !journal_.create(delta, + protocol, + storage_v2::JournalKind::PeerDelta, + slot_size)) + { + return false; + } + partitions_[protocolIndex(protocol)].peer_delta_count = 0U; + return true; +} + +bool SdProtocolPeerRepository::rewriteContactSnapshot(MeshProtocol protocol) +{ + char target[96] = {}; + char temp[96] = {}; + char backup[96] = {}; + char delta[96] = {}; + buildProtocolPath(protocol, "contacts.snapshot", target, sizeof(target)); + buildProtocolPath(protocol, "contacts.snapshot.tmp", temp, sizeof(temp)); + buildProtocolPath(protocol, + "contacts.snapshot.bak", + backup, + sizeof(backup)); + buildProtocolPath(protocol, "contacts.delta", delta, sizeof(delta)); + if (storage_runtime::sd_exists(temp)) + { + storage_runtime::sd_remove(temp); + } + const std::size_t slot_size = storage_v2::contactSlotSize(protocol); + if (!journal_.create(temp, + protocol, + storage_v2::JournalKind::ContactSnapshot, + slot_size)) + { + return false; + } + for (const storage_v2::ContactProjection& contact : contacts_) + { + if (!meshPeerSameProtocol(contact.identity.protocol, protocol)) + { + continue; + } + if (!storage_v2::encodeContactSlot(protocol, + contact, + slot_scratch_.data(), + slot_size) || + !journal_.append(temp, + protocol, + storage_v2::JournalKind::ContactSnapshot, + slot_size, + slot_scratch_.data())) + { + storage_runtime::sd_remove(temp); + return false; + } + } + if (!storage_v2::replaceFileAtomically(temp, target, backup) || + !journal_.create(delta, + protocol, + storage_v2::JournalKind::ContactDelta, + slot_size)) + { + return false; + } + partitions_[protocolIndex(protocol)].contact_delta_count = 0U; + return true; +} + +bool SdProtocolPeerRepository::appendPeerDelta( + const storage_v2::PeerProjection& projection) +{ + const MeshProtocol protocol = normalizeProtocol( + projection.record.identity.protocol); + const std::size_t slot_size = storage_v2::peerSlotSize(protocol); + if (slot_size == 0U || slot_size > slot_scratch_.size() || + !storage_v2::encodePeerSlot(protocol, + projection, + slot_scratch_.data(), + slot_size)) + { + return false; + } + storage_runtime::PersistenceBusGate bus_gate( + s_peer_store_bus_arbiter, + sys::runtime::BusAccessPolicy::BackgroundWorkerBounded, + 0U, + kPeerStoreBusResource, + kPeerStoreBusOwnerId + 2U, + kPeerStoreBusOwnerId); + if (!bus_gate.locked()) + { + return false; + } + char path[96] = {}; + buildProtocolPath(protocol, "peers.delta", path, sizeof(path)); + if (!journal_.append(path, + protocol, + storage_v2::JournalKind::PeerDelta, + slot_size, + slot_scratch_.data())) + { + return false; + } + ++partitions_[protocolIndex(protocol)].peer_delta_count; + return true; +} + +bool SdProtocolPeerRepository::appendContactDelta( + const storage_v2::ContactProjection& projection) +{ + const MeshProtocol protocol = normalizeProtocol(projection.identity.protocol); + const std::size_t slot_size = storage_v2::contactSlotSize(protocol); + if (slot_size == 0U || slot_size > slot_scratch_.size() || + !storage_v2::encodeContactSlot(protocol, + projection, + slot_scratch_.data(), + slot_size)) + { + return false; + } + storage_runtime::PersistenceBusGate bus_gate( + s_peer_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 250U, + kPeerStoreBusResource, + kPeerStoreBusOwnerId + 3U, + kPeerStoreBusOwnerId); + if (!bus_gate.locked()) + { + return false; + } + char path[96] = {}; + buildProtocolPath(protocol, "contacts.delta", path, sizeof(path)); + if (!journal_.append(path, + protocol, + storage_v2::JournalKind::ContactDelta, + slot_size, + slot_scratch_.data())) + { + return false; + } + ++partitions_[protocolIndex(protocol)].contact_delta_count; + return true; +} + +bool SdProtocolPeerRepository::queueOrAppendPeerDelta( + const storage_v2::PeerProjection& projection) +{ + if (pending_peer_head_ < pending_peer_deltas_.size()) + { + storage_v2::PeerProjection& newest = pending_peer_deltas_.back(); + if (newest.deleted == projection.deleted && + sameMeshPeerIdentity(newest.record.identity, + projection.record.identity)) + { + newest = projection; + return false; + } + pending_peer_deltas_.push_back(projection); + return false; + } + if (appendPeerDelta(projection)) + { + return true; + } + pending_peer_deltas_.push_back(projection); + return false; +} + +bool SdProtocolPeerRepository::drainPendingPeerDeltas(std::size_t budget) +{ + std::size_t drained = 0U; + while (pending_peer_head_ < pending_peer_deltas_.size() && + drained < budget) + { + if (!appendPeerDelta(pending_peer_deltas_[pending_peer_head_])) + { + return false; + } + ++pending_peer_head_; + ++drained; + } + if (pending_peer_head_ == pending_peer_deltas_.size()) + { + pending_peer_deltas_.clear(); + pending_peer_head_ = 0U; + return true; + } + return false; +} + +bool SdProtocolPeerRepository::applyPeerProjection( + const storage_v2::PeerProjection& projection) +{ + if (!meshPeerRecordIsValid(projection.record)) + { + return false; + } + const std::size_t index = findPeerIndex(projection.record.identity); + if (projection.deleted) + { + if (index < peers_.size()) + { + peers_.erase(peers_.begin() + static_cast(index)); + } + return true; + } + if (index < peers_.size()) + { + peers_[index] = projection.record; + } + else + { + peers_.push_back(projection.record); + } + return true; +} + +bool SdProtocolPeerRepository::applyContactProjection( + const storage_v2::ContactProjection& projection) +{ + if (!meshPeerIdentityIsValid(projection.identity)) + { + return false; + } + const std::size_t index = findContactIndex(projection.identity); + if (projection.deleted) + { + if (index < contacts_.size()) + { + contacts_.erase(contacts_.begin() + + static_cast(index)); + } + return true; + } + if (index < contacts_.size()) + { + contacts_[index] = projection; + } + else + { + contacts_.push_back(projection); + } + return true; +} + +void SdProtocolPeerRepository::overlayContactFacts() +{ + for (MeshPeerRecord& peer : peers_) + { + peer.flags = MeshPeerUserFlags{}; + peer.user_alias[0] = '\0'; + } + for (const storage_v2::ContactProjection& contact : contacts_) + { + std::size_t peer_index = findPeerIndex(contact.identity); + if (peer_index >= peers_.size()) + { + MeshPeerRecord placeholder{}; + placeholder.valid = true; + placeholder.identity = contact.identity; + placeholder.source = MeshPeerSource::Manual; + if (placeholder.identity.protocol == MeshProtocol::MeshCore) + { + placeholder.meshcore.has_public_key = true; + placeholder.meshcore.node_id_hint = contact.node_id_hint; + std::memcpy(placeholder.meshcore.public_key, + placeholder.identity.public_key, + kMeshPeerMeshCorePublicKeyLen); + } + else if (placeholder.identity.protocol == MeshProtocol::Reticulum) + { + placeholder.reticulum.identity = + placeholder.identity.reticulum; + } + peers_.push_back(placeholder); + peer_index = peers_.size() - 1U; + } + MeshPeerRecord& peer = peers_[peer_index]; + peer.flags = contact.flags; + copyMeshPeerText(peer.user_alias, + sizeof(peer.user_alias), + contact.alias); + if (peer.identity.protocol == MeshProtocol::MeshCore && + peer.meshcore.node_id_hint == 0U) + { + peer.meshcore.node_id_hint = contact.node_id_hint; + } + } +} + +void SdProtocolPeerRepository::overlayContactFactsForPeer( + MeshPeerRecord& peer) const +{ + peer.flags = MeshPeerUserFlags{}; + peer.user_alias[0] = '\0'; + MeshPeerIdentity stable{}; + if (!stableContactIdentity(peer, stable)) + { + return; + } + const std::size_t contact_index = findContactIndex(stable); + if (contact_index >= contacts_.size()) + { + return; + } + const storage_v2::ContactProjection& contact = contacts_[contact_index]; + peer.flags = contact.flags; + copyMeshPeerText(peer.user_alias, + sizeof(peer.user_alias), + contact.alias); + if (peer.identity.protocol == MeshProtocol::MeshCore && + peer.meshcore.node_id_hint == 0U) + { + peer.meshcore.node_id_hint = contact.node_id_hint; + } +} + +void SdProtocolPeerRepository::reconcileStableIdentities(MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + for (std::size_t stable_index = 0U; stable_index < peers_.size(); + ++stable_index) + { + MeshPeerRecord& stable = peers_[stable_index]; + if (!meshPeerSameProtocol(stable.identity.protocol, protocol) || + stable.identity.kind == MeshPeerIdentityKind::NodeId) + { + continue; + } + const NodeId node_id = projectedNodeId(stable); + if (node_id == 0U) + { + continue; + } + for (std::size_t unresolved_index = 0U; + unresolved_index < peers_.size(); ++unresolved_index) + { + if (unresolved_index == stable_index) + { + continue; + } + const MeshPeerRecord& unresolved = peers_[unresolved_index]; + if (!meshPeerSameProtocol(unresolved.identity.protocol, protocol) || + unresolved.identity.kind != MeshPeerIdentityKind::NodeId || + unresolved.identity.node_id != node_id) + { + continue; + } + MeshPeerRecord merged = mergeMeshPeerRecordFacts(unresolved, stable); + merged.identity = stable.identity; + peers_[stable_index] = merged; + peers_.erase(peers_.begin() + + static_cast(unresolved_index)); + if (unresolved_index < stable_index) + { + --stable_index; + } + break; + } + } +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::record( + const MeshPeerRecord& input) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !begun_ || !meshPeerRecordIsValid(input)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::InvalidArgument); + } + + MeshPeerRecord incoming = input; + incoming.identity.protocol = normalizeProtocol(incoming.identity.protocol); + if (incoming.identity.protocol == MeshProtocol::Reticulum && + incoming.identity.kind == MeshPeerIdentityKind::ReticulumDestination) + { + incoming.reticulum.identity = incoming.identity.reticulum; + } + const std::size_t exact_index = findPeerIndex(incoming.identity); + std::size_t merge_index = exact_index; + if (merge_index >= peers_.size() && + incoming.identity.kind != MeshPeerIdentityKind::NodeId) + { + const NodeId node_id = projectedNodeId(incoming); + if (node_id != 0U) + { + merge_index = findPeerIndexByNodeId(incoming.identity.protocol, + node_id); + } + } + + if (merge_index >= peers_.size() && + ephemeralCount(incoming.identity.protocol) >= kEphemeralPeerCapacity && + !evictOldestEphemeral(incoming.identity.protocol)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::CapacityExceeded); + } + + MeshPeerRecord next = incoming; + MeshPeerIdentity replaced_identity{}; + bool identity_upgrade = false; + if (merge_index < peers_.size()) + { + replaced_identity = peers_[merge_index].identity; + next = mergeMeshPeerRecordFacts(peers_[merge_index], incoming); + if (!sameMeshPeerIdentity(replaced_identity, incoming.identity) && + incoming.identity.kind != MeshPeerIdentityKind::NodeId) + { + next.identity = incoming.identity; + identity_upgrade = true; + } + } + next.valid = true; + next.identity.protocol = normalizeProtocol(next.identity.protocol); + if (next.last_seen_s < next.first_seen_s) + { + next.last_seen_s = next.first_seen_s; + } + + const storage_v2::PeerProjection next_projection{next, false}; + const bool immediately_durable = queueOrAppendPeerDelta(next_projection); + if (identity_upgrade) + { + storage_v2::PeerProjection tombstone{}; + tombstone.record = peers_[merge_index]; + tombstone.deleted = true; + (void)queueOrAppendPeerDelta(tombstone); + } + + if (merge_index < peers_.size()) + { + peers_[merge_index] = next; + } + else + { + peers_.push_back(next); + } + if (merge_index < peers_.size()) + { + overlayContactFactsForPeer(peers_[merge_index]); + } + active_node_projection_dirty_ = true; + if (!immediately_durable) + { + Serial.printf("[PeerStoreV2] peer queued protocol=%s pending=%u\n", + protocolSlug(next.identity.protocol), + static_cast(pending_peer_deltas_.size() - + pending_peer_head_)); + } + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::find( + const MeshPeerIdentity& identity, + MeshPeerRecord& out_record) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + const std::size_t index = findPeerIndex(identity); + if (index >= peers_.size()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::NotFound); + } + out_record = peers_[index]; + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::findByNodeId( + MeshProtocol protocol, + NodeId node_id, + MeshPeerRecord& out_record) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || node_id == 0U) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::InvalidArgument); + } + const std::size_t index = findPeerIndexByNodeId(protocol, node_id); + if (index >= peers_.size()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::NotFound); + } + out_record = peers_[index]; + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::loadRecent( + MeshProtocol protocol, + MeshPeerRecord* out_records, + std::size_t max_records, + std::size_t* out_count) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !out_count || (!out_records && max_records > 0U)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::InvalidArgument); + } + protocol = normalizeProtocol(protocol); + using PeerPtrVector = std::vector< + const MeshPeerRecord*, + ::platform::esp::arduino_common::memory::PsramAllocator< + const MeshPeerRecord*>>; + PeerPtrVector matches; + matches.reserve(peers_.size()); + for (const MeshPeerRecord& peer : peers_) + { + if (meshPeerSameProtocol(peer.identity.protocol, protocol)) + { + matches.push_back(&peer); + } + } + std::sort(matches.begin(), + matches.end(), + [](const MeshPeerRecord* lhs, const MeshPeerRecord* rhs) + { + return lhs->last_seen_s > rhs->last_seen_s; + }); + *out_count = std::min(max_records, matches.size()); + for (std::size_t index = 0U; index < *out_count; ++index) + { + out_records[index] = *matches[index]; + } + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::search( + MeshProtocol protocol, + const char* query, + MeshPeerRecord* out_records, + std::size_t max_records, + std::size_t* out_count) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !query || !out_count || + (!out_records && max_records > 0U)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::InvalidArgument); + } + protocol = normalizeProtocol(protocol); + *out_count = 0U; + for (const MeshPeerRecord& peer : peers_) + { + const MeshPeerNodeFacts* facts = nodeFacts(peer); + if (!meshPeerSameProtocol(peer.identity.protocol, protocol) || + (!textContains(peer.user_alias, query) && + !textContains(peer.display_name, query) && + (!facts || + (!textContains(facts->short_name, query) && + !textContains(facts->long_name, query))))) + { + continue; + } + if (*out_count < max_records) + { + out_records[*out_count] = peer; + } + ++(*out_count); + if (*out_count >= max_records) + { + break; + } + } + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::setUserFlags( + const MeshPeerIdentity& identity, + const MeshPeerUserFlags& flags) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + const std::size_t peer_index = findPeerIndex(identity); + if (peer_index >= peers_.size()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::NotFound); + } + MeshPeerIdentity stable{}; + if (!stableContactIdentity(peers_[peer_index], stable)) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Unsupported); + } + const bool remove_projection = !flags.favorite && !flags.ignored && + !flags.trusted && + peers_[peer_index].user_alias[0] == '\0'; + if (!persistContactFacts(stable, + flags, + peers_[peer_index].user_alias, + remove_projection)) + { + return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); + } + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::remove( + const MeshPeerIdentity& identity) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + const std::size_t index = findPeerIndex(identity); + if (index >= peers_.size()) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::NotFound); + } + if (peerIsProtected(peers_[index])) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::Unsupported); + } + storage_v2::PeerProjection tombstone{}; + tombstone.record = peers_[index]; + tombstone.deleted = true; + (void)queueOrAppendPeerDelta(tombstone); + peers_.erase(peers_.begin() + static_cast(index)); + active_node_projection_dirty_ = true; + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::clearProtocol( + MeshProtocol protocol) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !begun_) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + protocol = normalizeProtocol(protocol); + peers_.erase(std::remove_if(peers_.begin(), + peers_.end(), + [protocol](const MeshPeerRecord& peer) + { + return meshPeerSameProtocol( + peer.identity.protocol, + protocol); + }), + peers_.end()); + if (pending_peer_head_ > 0U) + { + pending_peer_deltas_.erase( + pending_peer_deltas_.begin(), + pending_peer_deltas_.begin() + + static_cast(pending_peer_head_)); + pending_peer_head_ = 0U; + } + pending_peer_deltas_.erase( + std::remove_if(pending_peer_deltas_.begin(), + pending_peer_deltas_.end(), + [protocol](const storage_v2::PeerProjection& projection) + { + return meshPeerSameProtocol( + projection.record.identity.protocol, + protocol); + }), + pending_peer_deltas_.end()); + overlayContactFacts(); + + storage_runtime::PersistenceBusGate bus_gate( + s_peer_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 1000U, + kPeerStoreBusResource, + kPeerStoreBusOwnerId + 4U, + kPeerStoreBusOwnerId); + if (!bus_gate.locked() || !rewritePeerSnapshot(protocol)) + { + return MeshPeerDirectoryStatus::fail(MeshPeerDirectoryStatusCode::IoError); + } + active_node_projection_dirty_ = true; + return MeshPeerDirectoryStatus::success(); +} + +MeshPeerDirectoryCapacity SdProtocolPeerRepository::capacityFor( + MeshProtocol protocol) const +{ + const std::size_t index = protocolIndex(protocol); + return MeshPeerDirectoryCapacity{kEphemeralPeerCapacity, + kPeerHotCacheCapacity[index]}; +} + +MeshPeerDirectoryStatus SdProtocolPeerRepository::flush() +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !begun_) + { + return MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); + } + return drainPendingPeerDeltas(kPendingFlushBudget) + ? MeshPeerDirectoryStatus::success() + : MeshPeerDirectoryStatus::fail( + MeshPeerDirectoryStatusCode::StorageUnavailable); +} + +std::size_t SdProtocolPeerRepository::findPeerIndex( + const MeshPeerIdentity& identity) const +{ + for (std::size_t index = 0U; index < peers_.size(); ++index) + { + if (sameMeshPeerIdentity(peers_[index].identity, identity)) + { + return index; + } + } + return peers_.size(); +} + +std::size_t SdProtocolPeerRepository::findPeerIndexByNodeId( + MeshProtocol protocol, + NodeId node_id) const +{ + protocol = normalizeProtocol(protocol); + for (std::size_t index = 0U; index < peers_.size(); ++index) + { + const MeshPeerRecord& peer = peers_[index]; + if (meshPeerSameProtocol(peer.identity.protocol, protocol) && + projectedNodeId(peer) == node_id) + { + return index; + } + } + return peers_.size(); +} + +std::size_t SdProtocolPeerRepository::findContactIndex( + const MeshPeerIdentity& identity) const +{ + for (std::size_t index = 0U; index < contacts_.size(); ++index) + { + if (sameMeshPeerIdentity(contacts_[index].identity, identity)) + { + return index; + } + } + return contacts_.size(); +} + +bool SdProtocolPeerRepository::stableContactIdentity( + const MeshPeerRecord& peer, + MeshPeerIdentity& out_identity) const +{ + const MeshProtocol protocol = normalizeProtocol(peer.identity.protocol); + if ((protocol == MeshProtocol::Meshtastic && + peer.identity.kind == MeshPeerIdentityKind::NodeId) || + (protocol == MeshProtocol::MeshCore && + peer.identity.kind == MeshPeerIdentityKind::PublicKey) || + (protocol == MeshProtocol::Reticulum && + peer.identity.kind == MeshPeerIdentityKind::ReticulumDestination)) + { + out_identity = peer.identity; + out_identity.protocol = protocol; + return true; + } + out_identity = MeshPeerIdentity{}; + return false; +} + +NodeId SdProtocolPeerRepository::projectedNodeId( + const MeshPeerRecord& peer) const +{ + const MeshProtocol protocol = normalizeProtocol(peer.identity.protocol); + if (peer.identity.kind == MeshPeerIdentityKind::NodeId) + { + return peer.identity.node_id; + } + if (protocol == MeshProtocol::MeshCore) + { + return peer.meshcore.node_id_hint; + } + if (protocol == MeshProtocol::Reticulum && + peer.identity.kind == MeshPeerIdentityKind::ReticulumDestination) + { + return reticulumNodeId(peer.identity.reticulum); + } + return 0U; +} + +bool SdProtocolPeerRepository::peerIsProtected( + const MeshPeerRecord& peer) const +{ + MeshPeerIdentity stable{}; + if (stableContactIdentity(peer, stable) && + findContactIndex(stable) < contacts_.size()) + { + return true; + } + return peerReferencedByConversation(peer); +} + +bool SdProtocolPeerRepository::peerReferencedByConversation( + const MeshPeerRecord& peer) const +{ + const MeshProtocol protocol = normalizeProtocol(peer.identity.protocol); + const std::vector conversations = + chat_store_.loadConversationPageForProtocol(protocol, 0U, 0U, nullptr); + return peerReferencedByConversations(peer, conversations); +} + +bool SdProtocolPeerRepository::peerReferencedByConversations( + const MeshPeerRecord& peer, + const std::vector& conversations) const +{ + const MeshProtocol protocol = normalizeProtocol(peer.identity.protocol); + const NodeId node_id = projectedNodeId(peer); + for (const ConversationMeta& conversation : conversations) + { + if (normalizeProtocol(conversation.id.protocol) != protocol) + { + continue; + } + if (protocol == MeshProtocol::Reticulum && + peer.identity.kind == MeshPeerIdentityKind::ReticulumDestination && + hasReticulumDestinationIdentity( + conversation.id.reticulum_identity) && + sameReticulumDestinationHash( + conversation.id.reticulum_identity, + peer.identity.reticulum)) + { + return true; + } + if (node_id != 0U && conversation.id.peer == node_id) + { + return true; + } + } + return false; +} + +std::size_t SdProtocolPeerRepository::ephemeralCount( + MeshProtocol protocol) const +{ + protocol = normalizeProtocol(protocol); + const std::vector conversations = + chat_store_.loadConversationPageForProtocol(protocol, 0U, 0U, nullptr); + std::size_t count = 0U; + for (const MeshPeerRecord& peer : peers_) + { + MeshPeerIdentity stable{}; + const bool is_contact = + stableContactIdentity(peer, stable) && + findContactIndex(stable) < contacts_.size(); + if (meshPeerSameProtocol(peer.identity.protocol, protocol) && + !is_contact && + !peerReferencedByConversations(peer, conversations)) + { + ++count; + } + } + return count; +} + +bool SdProtocolPeerRepository::evictOldestEphemeral(MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + const std::vector conversations = + chat_store_.loadConversationPageForProtocol(protocol, 0U, 0U, nullptr); + std::size_t candidate = peers_.size(); + uint32_t oldest_seen = UINT32_MAX; + for (std::size_t index = 0U; index < peers_.size(); ++index) + { + const MeshPeerRecord& peer = peers_[index]; + MeshPeerIdentity stable{}; + const bool is_contact = + stableContactIdentity(peer, stable) && + findContactIndex(stable) < contacts_.size(); + if (!meshPeerSameProtocol(peer.identity.protocol, protocol) || + is_contact || + peerReferencedByConversations(peer, conversations)) + { + continue; + } + if (candidate >= peers_.size() || peer.last_seen_s < oldest_seen) + { + candidate = index; + oldest_seen = peer.last_seen_s; + } + } + if (candidate >= peers_.size()) + { + return false; + } + storage_v2::PeerProjection tombstone{}; + tombstone.record = peers_[candidate]; + tombstone.deleted = true; + (void)queueOrAppendPeerDelta(tombstone); + peers_.erase(peers_.begin() + static_cast(candidate)); + active_node_projection_dirty_ = true; + return true; +} + +contacts::INodeStore& SdProtocolPeerRepository::nodeStoreView() +{ + return node_store_view_; +} + +contacts::IContactStore& SdProtocolPeerRepository::contactStoreView() +{ + return contact_store_view_; +} + +SdProtocolPeerRepository::NodeStoreView::NodeStoreView( + SdProtocolPeerRepository& owner) + : owner_(owner) +{ +} + +void SdProtocolPeerRepository::NodeStoreView::setActiveProtocol( + MeshProtocol protocol) +{ + ScopedRepositoryLock lock(owner_.mutex_); + if (!lock.locked()) + { + return; + } + owner_.active_protocol_ = normalizeProtocol(protocol); + owner_.active_node_projection_dirty_ = true; +} + +void SdProtocolPeerRepository::NodeStoreView::begin() +{ + (void)owner_.begin(); +} + +void SdProtocolPeerRepository::NodeStoreView::applyUpdate( + uint32_t node_id, + const contacts::NodeUpdate& update) +{ + owner_.applyNodeUpdate(node_id, update); +} + +void SdProtocolPeerRepository::NodeStoreView::upsert( + uint32_t node_id, + const char* short_name, + const char* long_name, + uint32_t now_secs, + float snr, + float rssi, + uint8_t protocol, + uint8_t role, + uint8_t hops_away, + uint8_t hw_model, + uint8_t channel) +{ + contacts::NodeUpdate update{}; + update.short_name = short_name; + update.long_name = long_name; + update.has_last_seen = true; + update.last_seen = now_secs; + update.has_snr = !std::isnan(snr); + update.snr = snr; + update.has_rssi = !std::isnan(rssi); + update.rssi = rssi; + update.has_protocol = protocol != 0U; + update.protocol = protocol; + update.has_role = role != contacts::kNodeRoleUnknown; + update.role = role; + update.has_hops_away = hops_away != 0xFFU; + update.hops_away = hops_away; + update.has_hw_model = hw_model != 0U; + update.hw_model = hw_model; + update.has_channel = channel != 0xFFU; + update.channel = channel; + owner_.applyNodeUpdate(node_id, update); +} + +void SdProtocolPeerRepository::NodeStoreView::updateProtocol( + uint32_t node_id, + uint8_t protocol, + uint32_t now_secs) +{ + contacts::NodeUpdate update{}; + update.has_protocol = protocol != 0U; + update.protocol = protocol; + update.has_last_seen = true; + update.last_seen = now_secs; + owner_.applyNodeUpdate(node_id, update); +} + +void SdProtocolPeerRepository::NodeStoreView::updatePosition( + uint32_t node_id, + const contacts::NodePosition& position) +{ + contacts::NodeUpdate update{}; + update.has_position = true; + update.position = position; + owner_.applyNodeUpdate(node_id, update); +} + +bool SdProtocolPeerRepository::NodeStoreView::remove(uint32_t node_id) +{ + return owner_.removeNode(node_id); +} + +const std::vector& +SdProtocolPeerRepository::NodeStoreView::getEntries() const +{ + return owner_.activeNodeEntries(); +} + +void SdProtocolPeerRepository::NodeStoreView::clear() +{ + owner_.clearActiveNodes(); +} + +bool SdProtocolPeerRepository::NodeStoreView::flush() +{ + return owner_.flush().succeeded(); +} + +void SdProtocolPeerRepository::applyNodeUpdate( + uint32_t node_id, + const contacts::NodeUpdate& update) +{ + if (node_id == 0U) + { + return; + } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || !begun_) + { + return; + } + MeshProtocol protocol = active_protocol_; + if (update.has_protocol) + { + const MeshProtocol requested = + static_cast(update.protocol); + if (requested == MeshProtocol::Meshtastic || + requested == MeshProtocol::MeshCore || + requested == MeshProtocol::Reticulum || + requested == MeshProtocol::RNode) + { + protocol = normalizeProtocol(requested); + } + } + + std::size_t existing_index = findPeerIndexByNodeId(protocol, node_id); + MeshPeerRecord incoming{}; + incoming.valid = true; + incoming.source = MeshPeerSource::RuntimeRx; + if (protocol == MeshProtocol::Reticulum && + hasReticulumDestinationIdentity(update.reticulum_identity)) + { + incoming.identity = makeMeshPeerReticulumIdentity( + update.reticulum_identity); + } + else if (existing_index < peers_.size()) + { + incoming.identity = peers_[existing_index].identity; + } + else + { + incoming.identity = makeMeshPeerNodeIdentity(protocol, node_id); + } + incoming.identity.protocol = protocol; + incoming.first_seen_s = update.has_last_seen ? update.last_seen : 0U; + incoming.last_seen_s = update.has_last_seen ? update.last_seen : 0U; + if (hasText(update.long_name)) + { + copyMeshPeerText(incoming.display_name, + sizeof(incoming.display_name), + update.long_name); + } + else if (hasText(update.short_name)) + { + copyMeshPeerText(incoming.display_name, + sizeof(incoming.display_name), + update.short_name); + } + if (update.has_snr) + { + incoming.observations.has_snr = true; + incoming.observations.snr = update.snr; + } + if (update.has_rssi) + { + incoming.observations.has_rssi = true; + incoming.observations.rssi = update.rssi; + } + if (update.has_device_metrics) + { + incoming.observations.has_device_metrics = true; + incoming.observations.device_metrics = update.device_metrics; + } + if (update.has_position) + { + incoming.observations.has_position = true; + incoming.observations.position = update.position; + } + + if (protocol == MeshProtocol::Reticulum) + { + incoming.reticulum.identity = incoming.identity.reticulum; + } + else + { + MeshPeerNodeFacts& facts = mutableNodeFacts(incoming); + copyNodeText(facts.short_name, + sizeof(facts.short_name), + update.short_name); + copyNodeText(facts.long_name, + sizeof(facts.long_name), + update.long_name); + if (update.has_role) + { + facts.role = update.role; + } + if (update.has_hw_model) + { + facts.hw_model = update.hw_model; + } + if (update.has_channel) + { + facts.channel = update.channel; + } + if (update.has_hops_away) + { + facts.hops_away = update.hops_away; + } + if (update.has_macaddr) + { + facts.has_macaddr = true; + std::memcpy(facts.macaddr, + update.macaddr, + sizeof(facts.macaddr)); + } + if (update.has_via_mqtt) + { + facts.via_mqtt = update.via_mqtt; + } + if (update.has_next_hop) + { + if (protocol == MeshProtocol::Meshtastic) + { + incoming.meshtastic.has_next_hop = true; + incoming.meshtastic.next_hop = update.next_hop; + } + else + { + incoming.meshcore.has_next_hop = true; + incoming.meshcore.next_hop = update.next_hop; + } + } + if (protocol == MeshProtocol::MeshCore) + { + incoming.meshcore.node_id_hint = node_id; + } + } + + MeshPeerRecord next = incoming; + bool identity_upgrade = false; + if (existing_index < peers_.size()) + { + next = mergeMeshPeerRecordFacts(peers_[existing_index], incoming); + if (!sameMeshPeerIdentity(peers_[existing_index].identity, + incoming.identity) && + incoming.identity.kind != MeshPeerIdentityKind::NodeId) + { + next.identity = incoming.identity; + identity_upgrade = true; + } + if (update.has_key_manually_verified) + { + if (protocol == MeshProtocol::Meshtastic && + next.meshtastic.has_public_key) + { + next.meshtastic.key_manually_verified = + update.key_manually_verified; + } + else if (protocol == MeshProtocol::MeshCore && + next.meshcore.has_public_key) + { + next.meshcore.public_key_verified = + update.key_manually_verified; + } + } + } + const storage_v2::PeerProjection projection{next, false}; + (void)queueOrAppendPeerDelta(projection); + if (identity_upgrade) + { + storage_v2::PeerProjection tombstone{}; + tombstone.record = peers_[existing_index]; + tombstone.deleted = true; + (void)queueOrAppendPeerDelta(tombstone); + } + if (existing_index < peers_.size()) + { + peers_[existing_index] = next; + } + else + { + peers_.push_back(next); + existing_index = peers_.size() - 1U; + } + + if (update.has_is_ignored && + (update.is_ignored || + findContactIndex(peers_[existing_index].identity) < contacts_.size())) + { + MeshPeerIdentity stable{}; + if (stableContactIdentity(peers_[existing_index], stable)) + { + MeshPeerUserFlags flags = peers_[existing_index].flags; + flags.ignored = update.is_ignored; + const bool remove_projection = !flags.favorite && !flags.ignored && + !flags.trusted && + peers_[existing_index] + .user_alias[0] == '\0'; + (void)persistContactFacts( + stable, + flags, + peers_[existing_index].user_alias, + remove_projection); + } + } + overlayContactFactsForPeer(peers_[existing_index]); + active_node_projection_dirty_ = true; +} + +bool SdProtocolPeerRepository::removeNode(uint32_t node_id) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked() || node_id == 0U) + { + return false; + } + const std::size_t index = findPeerIndexByNodeId(active_protocol_, node_id); + if (index >= peers_.size() || peerIsProtected(peers_[index])) + { + return false; + } + storage_v2::PeerProjection tombstone{}; + tombstone.record = peers_[index]; + tombstone.deleted = true; + (void)queueOrAppendPeerDelta(tombstone); + peers_.erase(peers_.begin() + static_cast(index)); + active_node_projection_dirty_ = true; + return true; +} + +void SdProtocolPeerRepository::clearActiveNodes() +{ + (void)clearProtocol(active_protocol_); +} + +const std::vector& +SdProtocolPeerRepository::activeNodeEntries() const +{ + ScopedRepositoryLock lock(mutex_); + if (lock.locked() && active_node_projection_dirty_) + { + rebuildActiveNodeProjection(); + } + return active_node_projection_; +} + +void SdProtocolPeerRepository::rebuildActiveNodeProjection() const +{ + active_node_projection_.clear(); + active_node_projection_.reserve( + std::count_if(peers_.begin(), + peers_.end(), + [this](const MeshPeerRecord& peer) + { + return meshPeerSameProtocol( + peer.identity.protocol, + active_protocol_); + })); + for (const MeshPeerRecord& peer : peers_) + { + contacts::NodeEntry entry{}; + if (meshPeerSameProtocol(peer.identity.protocol, active_protocol_) && + toNodeEntry(peer, entry)) + { + active_node_projection_.push_back(entry); + } + } + active_node_projection_dirty_ = false; +} + +bool SdProtocolPeerRepository::toNodeEntry( + const MeshPeerRecord& peer, + contacts::NodeEntry& out) const +{ + const NodeId node_id = projectedNodeId(peer); + if (node_id == 0U) + { + return false; + } + out = contacts::NodeEntry{}; + out.node_id = node_id; + out.last_seen = peer.last_seen_s; + out.protocol = static_cast(normalizeProtocol( + peer.identity.protocol)); + const MeshPeerNodeFacts* facts = nodeFacts(peer); + if (facts) + { + copyNodeText(out.short_name, + sizeof(out.short_name), + facts->short_name); + copyNodeText(out.long_name, + sizeof(out.long_name), + facts->long_name); + out.role = facts->role; + out.hw_model = facts->hw_model; + out.channel = facts->channel; + out.hops_away = facts->hops_away; + out.has_macaddr = facts->has_macaddr; + std::memcpy(out.macaddr, facts->macaddr, sizeof(out.macaddr)); + out.via_mqtt = facts->via_mqtt; + } + else + { + copyNodeText(out.long_name, + sizeof(out.long_name), + peer.display_name); + out.role = static_cast(contacts::NodeRoleType::Client); + out.hops_away = 0U; + } + if (out.long_name[0] == '\0') + { + copyNodeText(out.long_name, + sizeof(out.long_name), + peer.display_name); + } + out.snr = peer.observations.has_snr ? peer.observations.snr : 0.0F; + out.rssi = peer.observations.has_rssi ? peer.observations.rssi : 0.0F; + out.is_ignored = peer.flags.ignored; + if (peer.identity.protocol == MeshProtocol::Meshtastic) + { + out.next_hop = peer.meshtastic.has_next_hop + ? peer.meshtastic.next_hop + : 0U; + out.has_public_key = peer.meshtastic.has_public_key; + out.key_manually_verified = + peer.meshtastic.key_manually_verified; + } + else if (peer.identity.protocol == MeshProtocol::MeshCore) + { + out.next_hop = peer.meshcore.has_next_hop + ? peer.meshcore.next_hop + : 0U; + out.has_public_key = peer.meshcore.has_public_key || + peer.identity.kind == + MeshPeerIdentityKind::PublicKey; + out.key_manually_verified = peer.meshcore.public_key_verified; + } + else if (peer.identity.protocol == MeshProtocol::Reticulum) + { + out.has_public_key = peer.reticulum.has_public_keys; + out.reticulum_identity = peer.identity.reticulum; + } + if (peer.observations.has_device_metrics) + { + out.has_device_metrics = true; + out.device_metrics = peer.observations.device_metrics; + } + if (peer.observations.has_position) + { + const contacts::NodePosition& position = + peer.observations.position; + out.position_valid = position.valid; + out.position_latitude_i = position.latitude_i; + out.position_longitude_i = position.longitude_i; + out.position_has_altitude = position.has_altitude; + out.position_altitude = position.altitude; + out.position_timestamp = position.timestamp; + out.position_precision_bits = position.precision_bits; + out.position_pdop = position.pdop; + out.position_hdop = position.hdop; + out.position_vdop = position.vdop; + out.position_gps_accuracy_mm = position.gps_accuracy_mm; + } + return true; +} + +SdProtocolPeerRepository::ContactStoreView::ContactStoreView( + SdProtocolPeerRepository& owner) + : owner_(owner) +{ +} + +void SdProtocolPeerRepository::ContactStoreView::setActiveProtocol( + MeshProtocol protocol) +{ + ScopedRepositoryLock lock(owner_.mutex_); + if (!lock.locked()) + { + return; + } + owner_.active_protocol_ = normalizeProtocol(protocol); + owner_.active_node_projection_dirty_ = true; +} + +void SdProtocolPeerRepository::ContactStoreView::begin() +{ + (void)owner_.begin(); +} + +std::string SdProtocolPeerRepository::ContactStoreView::getNickname( + uint32_t node_id) const +{ + return owner_.nicknameForNode(node_id); +} + +bool SdProtocolPeerRepository::ContactStoreView::setNickname( + uint32_t node_id, + const char* nickname) +{ + return owner_.setNicknameForNode(node_id, nickname); +} + +bool SdProtocolPeerRepository::ContactStoreView::removeNickname( + uint32_t node_id) +{ + return owner_.removeNicknameForNode(node_id); +} + +bool SdProtocolPeerRepository::ContactStoreView::hasNickname( + const char* nickname) const +{ + return owner_.hasNicknameForActiveProtocol(nickname); +} + +std::vector +SdProtocolPeerRepository::ContactStoreView::getAllContactIds() const +{ + return owner_.activeContactIds(); +} + +size_t SdProtocolPeerRepository::ContactStoreView::getCount() const +{ + return owner_.activeContactCount(); +} + +std::string SdProtocolPeerRepository::nicknameForNode(uint32_t node_id) const +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return {}; + } + const std::size_t peer_index = findPeerIndexByNodeId(active_protocol_, + node_id); + if (peer_index >= peers_.size()) + { + return {}; + } + MeshPeerIdentity stable{}; + if (!stableContactIdentity(peers_[peer_index], stable)) + { + return {}; + } + const std::size_t contact_index = findContactIndex(stable); + return contact_index < contacts_.size() + ? std::string(contacts_[contact_index].alias) + : std::string(); +} + +bool SdProtocolPeerRepository::setNicknameForNode(uint32_t node_id, + const char* nickname) +{ + if (!nickname || nickname[0] == '\0' || + std::strlen(nickname) > kMeshPeerUserAliasMaxLen) + { + return false; + } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return false; + } + const std::size_t peer_index = findPeerIndexByNodeId(active_protocol_, + node_id); + if (peer_index >= peers_.size()) + { + return false; + } + MeshPeerIdentity stable{}; + if (!stableContactIdentity(peers_[peer_index], stable)) + { + return false; + } + MeshPeerUserFlags flags = peers_[peer_index].flags; + flags.favorite = true; + return persistContactFacts(stable, flags, nickname, false); +} + +bool SdProtocolPeerRepository::removeNicknameForNode(uint32_t node_id) +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return false; + } + const std::size_t peer_index = findPeerIndexByNodeId(active_protocol_, + node_id); + if (peer_index >= peers_.size()) + { + return false; + } + MeshPeerIdentity stable{}; + if (!stableContactIdentity(peers_[peer_index], stable)) + { + return false; + } + const std::size_t contact_index = findContactIndex(stable); + if (contact_index >= contacts_.size()) + { + return false; + } + MeshPeerUserFlags flags = contacts_[contact_index].flags; + flags.favorite = false; + const bool deleted = !flags.ignored && !flags.trusted; + return persistContactFacts(stable, flags, "", deleted); +} + +bool SdProtocolPeerRepository::hasNicknameForActiveProtocol( + const char* nickname) const +{ + if (!nickname || nickname[0] == '\0') + { + return false; + } + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return false; + } + for (const storage_v2::ContactProjection& contact : contacts_) + { + if (meshPeerSameProtocol(contact.identity.protocol, + active_protocol_) && + std::strcmp(contact.alias, nickname) == 0) + { + return true; + } + } + return false; +} + +std::vector SdProtocolPeerRepository::activeContactIds() const +{ + ScopedRepositoryLock lock(mutex_); + std::vector ids; + if (!lock.locked()) + { + return ids; + } + ids.reserve(contacts_.size()); + for (const storage_v2::ContactProjection& contact : contacts_) + { + if (!meshPeerSameProtocol(contact.identity.protocol, + active_protocol_) || + (!contact.flags.favorite && contact.alias[0] == '\0')) + { + continue; + } + const std::size_t peer_index = findPeerIndex(contact.identity); + if (peer_index < peers_.size()) + { + const NodeId node_id = projectedNodeId(peers_[peer_index]); + if (node_id != 0U) + { + ids.push_back(node_id); + } + } + } + return ids; +} + +size_t SdProtocolPeerRepository::activeContactCount() const +{ + ScopedRepositoryLock lock(mutex_); + if (!lock.locked()) + { + return 0U; + } + return static_cast(std::count_if( + contacts_.begin(), + contacts_.end(), + [this](const storage_v2::ContactProjection& contact) + { + return meshPeerSameProtocol(contact.identity.protocol, + active_protocol_) && + (contact.flags.favorite || contact.alias[0] != '\0'); + })); +} + +bool SdProtocolPeerRepository::persistContactFacts( + const MeshPeerIdentity& identity, + const MeshPeerUserFlags& flags, + const char* alias, + bool deleted) +{ + const std::size_t existing_contact_index = findContactIndex(identity); + if (!deleted && existing_contact_index >= contacts_.size() && + contacts_.size() >= kProtectedContactCapacity) + { + Serial.printf("[PeerStoreV2] contact capacity reached count=%u\n", + static_cast(contacts_.size())); + return false; + } + storage_v2::ContactProjection projection{}; + projection.identity = identity; + projection.flags = flags; + projection.deleted = deleted; + copyMeshPeerText(projection.alias, + sizeof(projection.alias), + alias ? alias : ""); + const std::size_t peer_index = findPeerIndex(identity); + if (peer_index < peers_.size()) + { + projection.node_id_hint = projectedNodeId(peers_[peer_index]); + } + if (!appendContactDelta(projection)) + { + return false; + } + (void)applyContactProjection(projection); + if (peer_index < peers_.size()) + { + overlayContactFactsForPeer(peers_[peer_index]); + } + active_node_projection_dirty_ = true; + return true; +} + +MeshProtocol SdProtocolPeerRepository::normalizeProtocol( + MeshProtocol protocol) +{ + return protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum + : protocol; +} + +const char* SdProtocolPeerRepository::protocolSlug(MeshProtocol protocol) +{ + switch (normalizeProtocol(protocol)) + { + case MeshProtocol::Meshtastic: + return "mt"; + case MeshProtocol::MeshCore: + return "mc"; + case MeshProtocol::Reticulum: + return "rt"; + default: + return "unknown"; + } +} + +std::size_t SdProtocolPeerRepository::protocolIndex(MeshProtocol protocol) +{ + switch (normalizeProtocol(protocol)) + { + case MeshProtocol::MeshCore: + return 1U; + case MeshProtocol::Reticulum: + return 2U; + case MeshProtocol::Meshtastic: + default: + return 0U; + } +} + +NodeId SdProtocolPeerRepository::reticulumNodeId( + const ReticulumPeerIdentity& identity) +{ + if (!hasReticulumDestinationIdentity(identity)) + { + return 0U; + } + const uint8_t* hash = identity.destination_hash; + return (static_cast(hash[12]) << 24U) | + (static_cast(hash[13]) << 16U) | + (static_cast(hash[14]) << 8U) | + static_cast(hash[15]); +} + +bool SdProtocolPeerRepository::ensureDirectory(const char* path) +{ + return path && path[0] != '\0' && + (storage_runtime::sd_exists(path) || + storage_runtime::sd_mkdir(path)); +} + +void SdProtocolPeerRepository::buildProtocolPath(MeshProtocol protocol, + const char* name, + char* out, + std::size_t out_len) +{ + if (!out || out_len == 0U) + { + return; + } + if (name && name[0] != '\0') + { + std::snprintf(out, + out_len, + "%s/%s/%s", + kRoot, + protocolSlug(protocol), + name); + } + else + { + std::snprintf(out, + out_len, + "%s/%s", + kRoot, + protocolSlug(protocol)); + } +} + +bool SdProtocolPeerRepository::lock() const +{ + return mutex_ && xSemaphoreTake(mutex_, portMAX_DELAY) == pdTRUE; +} + +void SdProtocolPeerRepository::unlock() const +{ + if (mutex_) + { + xSemaphoreGive(mutex_); + } +} + +} // namespace chat diff --git a/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp b/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp index 515e8897..02cf1398 100644 --- a/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp +++ b/platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp @@ -1,12 +1,13 @@ /** * @file sd_store.cpp - * @brief SD-backed ESP Arduino chat storage using per-conversation log files. + * @brief Protocol-partitioned append-only ESP chat storage. */ #include "platform/esp/arduino_common/chat/infra/store/sd_store.h" -#include "chat/infra/mesh_protocol_utils.h" +#include "platform/esp/arduino_common/storage/persistence_bus_gate.h" #include "platform/esp/arduino_common/storage/sd_card_runtime.h" +#include "platform/esp/common/shared_spi_bus_arbiter.h" #if defined(ARDUINO) #include @@ -15,13 +16,13 @@ #include #include #include -#include namespace chat { namespace { -namespace storage = ::platform::esp::arduino_common::storage; +namespace storage_runtime = ::platform::esp::arduino_common::storage; +namespace storage_v2 = ::chat::storage::v2; #if defined(ARDUINO) #define CHAT_STORE_LOG(...) Serial.printf(__VA_ARGS__) @@ -29,104 +30,125 @@ namespace storage = ::platform::esp::arduino_common::storage; #define CHAT_STORE_LOG(...) std::printf(__VA_ARGS__) #endif -constexpr uint8_t kRecordHasReticulumIdentityFlag = 0x01U; -constexpr uint8_t kRecordSourceUnverifiedFlag = 0x02U; -constexpr uint8_t kRecordHasReticulumLxmfHashFlag = 0x04U; -constexpr uint8_t kIndexHasReticulumIdentityFlag = 0x01U; -constexpr uint8_t kReadStateHasReticulumIdentityFlag = 0x01U; -// FileHeader::reserved is a durable unread count once the valid bit is set. -constexpr uint16_t kUnreadStateValidMask = 0x8000U; -constexpr uint16_t kUnreadStateCountMask = 0x7FFFU; -constexpr const char* kTempIndexFile = "/chat/index.tmp"; -constexpr const char* kBackupIndexFile = "/chat/index.bak"; -constexpr const char* kTempReadStateFile = "/chat/read_state.tmp"; -constexpr const char* kBackupReadStateFile = "/chat/read_state.bak"; +constexpr uint32_t kChatStoreBusResource = 5; +constexpr uint32_t kChatStoreBusOwnerId = 0x43484154U; // CHAT +constexpr const char* kChatStoreBusOwner = "chat_store_v2"; -static_assert(SdStore::kMaxMessagesPerConv <= kUnreadStateCountMask, - "Unread state must cover a full conversation ring"); +::platform::esp::common::SharedSpiBusAdapter s_chat_store_bus_adapter( + kChatStoreBusOwner, + kChatStoreBusOwnerId); +::platform::esp::common::FixedSharedSpiBusPolicyStrategy s_chat_store_bus_policy( + 0, + 0, + 0, + 0); +sys::runtime::StorageBusArbiter s_chat_store_bus_arbiter(s_chat_store_bus_adapter, + s_chat_store_bus_policy); -uint16_t encodeUnreadState(uint16_t unread) +constexpr MeshProtocol kProtocols[] = { + MeshProtocol::Meshtastic, + MeshProtocol::MeshCore, + MeshProtocol::Reticulum, +}; + +std::size_t protocolIndex(MeshProtocol protocol) { - return static_cast(kUnreadStateValidMask | - std::min(unread, kUnreadStateCountMask)); + if (protocol == MeshProtocol::MeshCore) + { + return 1; + } + if (protocol == MeshProtocol::Reticulum || + protocol == MeshProtocol::RNode) + { + return 2; + } + return 0; } -bool decodeUnreadState(uint16_t state, uint16_t* unread) +bool sameConversationKey(const ConversationId& lhs, + const ConversationId& rhs) { - if ((state & kUnreadStateValidMask) == 0) + const MeshProtocol lhs_protocol = + lhs.protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum + : lhs.protocol; + const MeshProtocol rhs_protocol = + rhs.protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum + : rhs.protocol; + if (lhs_protocol != rhs_protocol || lhs.channel != rhs.channel) { return false; } - if (unread) + if (lhs_protocol == MeshProtocol::Reticulum) { - *unread = static_cast(state & kUnreadStateCountMask); + const bool lhs_has_destination = + hasReticulumDestinationIdentity(lhs.reticulum_identity); + const bool rhs_has_destination = + hasReticulumDestinationIdentity(rhs.reticulum_identity); + if (lhs_has_destination || rhs_has_destination) + { + return lhs_has_destination && rhs_has_destination && + sameReticulumDestinationHash(lhs.reticulum_identity, + rhs.reticulum_identity); + } } + return lhs.peer == rhs.peer; +} + +bool copyTextPreview(char* out, + std::size_t out_len, + const std::string& text) +{ + if (!out || out_len == 0U) + { + return false; + } + const std::size_t length = std::min(out_len - 1U, text.size()); + if (length != 0U) + { + std::memcpy(out, text.data(), length); + } + out[length] = '\0'; return true; } -bool readExact(storage::SdRuntimeFile& file, void* out, size_t len) +bool hasSuffix(const char* value, const char* suffix) { - return file.read(out, len) == static_cast(len); -} - -bool writeExact(storage::SdRuntimeFile& file, const void* data, size_t len) -{ - return file.write(data, len) == len; -} - -const char* protocolTag(MeshProtocol protocol) -{ - return chat::infra::meshProtocolSlug(protocol); -} - -bool hasReticulumConversationKey(const ConversationId& conv) -{ - return conv.protocol == MeshProtocol::Reticulum && - hasReticulumDestinationIdentity(conv.reticulum_identity); -} - -bool sameReticulumDestination(const ReticulumPeerIdentity& identity, - const uint8_t* destination_hash) -{ - return sameReticulumDestinationHash(identity, destination_hash); -} - -bool validLxmfMessageHash(const uint8_t* lxmf_hash) -{ - return lxmf_hash && - !isAllZeroKeyBytes(lxmf_hash, kReticulumLxmfHashSize); -} - -void copyReticulumIdentityToStorage(uint8_t* destination_hash, - uint8_t* identity_hash, - const ReticulumPeerIdentity& identity) -{ - if (!destination_hash || !identity_hash) + if (!value || !suffix) { - return; + return false; } - (void)copyReticulumIdentityHashes(destination_hash, identity_hash, identity); + const std::size_t value_len = std::strlen(value); + const std::size_t suffix_len = std::strlen(suffix); + return value_len >= suffix_len && + std::memcmp(value + value_len - suffix_len, + suffix, + suffix_len) == 0; } -void readReticulumIdentityFromStorage(ReticulumPeerIdentity& identity, - const uint8_t* destination_hash, - const uint8_t* identity_hash) +bool replaceSnapshot(const char* temp_path, const char* final_path) { - if (!destination_hash || !identity_hash) + if (!temp_path || !final_path) { - return; + return false; } - identity = makeReticulumPeerIdentity(destination_hash, identity_hash); + char backup_path[160] = {}; + std::snprintf(backup_path, + sizeof(backup_path), + "%s.bak", + final_path); + return storage_v2::replaceFileAtomically(temp_path, + final_path, + backup_path); } -void hashToHex(const uint8_t* hash, char* out, size_t out_len) +void hashToHex(const uint8_t* hash, char* out, std::size_t out_len) { static constexpr char kHex[] = "0123456789abcdef"; - if (!hash || !out || out_len < (kReticulumPeerHashSize * 2U + 1U)) + if (!hash || !out || out_len < kReticulumPeerHashSize * 2U + 1U) { return; } - for (size_t index = 0; index < kReticulumPeerHashSize; ++index) + for (std::size_t index = 0; index < kReticulumPeerHashSize; ++index) { out[index * 2U] = kHex[(hash[index] >> 4U) & 0x0FU]; out[index * 2U + 1U] = kHex[hash[index] & 0x0FU]; @@ -134,637 +156,504 @@ void hashToHex(const uint8_t* hash, char* out, size_t out_len) out[kReticulumPeerHashSize * 2U] = '\0'; } -std::string pathInChatDir(const char* name) -{ - if (!name || name[0] == '\0') - { - return {}; - } - if (name[0] == '/') - { - return std::string(name); - } - return std::string(SdStore::kDir) + "/" + name; -} } // namespace SdStore::SdStore() { - ready_ = ensureFs() && ensureDir(); - if (!ready_) + scratch_.resize(kScratchCapacity); + catalog_.reserve(64); + read_state_.reserve(64); + statuses_.reserve(256); + seen_hot_.reserve(256); + storage_runtime::PersistenceBusGate startup_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 3000U, + kChatStoreBusResource, + kChatStoreBusOwnerId + 9U, + kChatStoreBusOwnerId); + ready_ = startup_gate.locked() && ensureLayout() && loadRuntimeState(); + if (ready_) { - CHAT_STORE_LOG("[AppContext] chat store=SdStore layout=logs ready=0 root=%s\n", kDir); - return; + for (MeshProtocol protocol : kProtocols) + { + if (!compactProtocolProjections(protocol)) + { + projection_dirty_[protocolIndex(protocol)] = true; + } + } } - - std::vector entries; - if (!readIndex(entries) && - (!recoverIndex() || !readIndex(entries))) - { - rebuildIndex(); - (void)readIndex(entries); - } - if (reconcileIndexUnread(entries)) - { - unread_reconcile_pending_ = !writeIndex(entries); - } - CHAT_STORE_LOG("[AppContext] chat store=SdStore layout=logs root=%s index=%u\n", - kDir, - static_cast(entries.size())); + CHAT_STORE_LOG("[ChatStoreV2] ready=%u root=%s conversations=%u statuses=%u seen_hot=%u\n", + ready_ ? 1U : 0U, + kRoot, + static_cast(catalog_.size()), + static_cast(statuses_.size()), + static_cast(seen_hot_.size())); } void SdStore::append(const ChatMessage& msg) { - (void)appendInternal(msg); + if (!appendDurably(msg)) + { + CHAT_STORE_LOG("[ChatStoreV2] append deferred protocol=%s msg=%08lX\n", + protocolSlug(msg.protocol), + static_cast(msg.msg_id)); + } +} + +bool SdStore::appendDurably(const ChatMessage& msg) +{ + return appendInternal(msg, false); } bool SdStore::appendIncomingDurably(const ChatMessage& msg) { - return appendInternal(msg); + return appendInternal(msg, true); } -bool SdStore::appendInternal(const ChatMessage& msg) +bool SdStore::appendInternal(const ChatMessage& input, bool incoming_commit) { - if (!ready_ || !ensureDir()) + storage_runtime::PersistenceBusGate transaction_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 0, + kChatStoreBusResource, + kChatStoreBusOwnerId + 1U, + kChatStoreBusOwnerId); + if (!transaction_gate.locked() || !ready_) { return false; } - const ConversationId conv = conversationIdForMessage(msg); - storage::SdRuntimeFile file; - FileHeader header{}; - if (!openConversationForUpdate(conv, file, header)) + ChatMessage message = input; + message.protocol = normalizeProtocol(message.protocol); + if (!storage_v2::supportedProtocol(message.protocol) || + message.text.size() > + (message.protocol == MeshProtocol::Meshtastic + ? storage_v2::kMeshtasticTextMax + : message.protocol == MeshProtocol::MeshCore + ? storage_v2::kMeshCoreTextMax + : storage_v2::kReticulumTextMax) || + !ensureProtocolLayout(message.protocol)) { return false; } - if (chat::hasReticulumLxmfMessageHash(msg)) + const ConversationId conversation = conversationIdForMessage(message); + bool already_stored = false; + uint32_t stored_count = 0; + if (!storedMessageMatches(message, &already_stored, &stored_count)) { - bool already_committed = false; - if (!conversationContainsReticulumLxmfHash(file, - header, - msg.reticulum_lxmf_hash, - &already_committed)) - { - file.close(); - return false; - } - if (already_committed) - { - uint16_t committed_unread = 0; - if (!readStateUnreadOrLegacy(conv, &committed_unread)) - { - committed_unread = 0; - } - file.close(); - updateIndexForMessage(msg, committed_unread); - return rememberReticulumLxmfMessageHash(msg.reticulum_lxmf_hash); - } - } - - uint16_t unread = 0; - if (!readStateUnreadOrLegacy(conv, &unread)) - { - unread = 0; - } - if (msg.status == MessageStatus::Incoming && unread < kUnreadStateCountMask) - { - unread = static_cast(unread + 1U); - } - - const Record rec = recordFromMessage(msg); - if (!writeRecord(file, header.head, rec)) - { - file.close(); return false; } - header.head = static_cast((header.head + 1U) % kMaxMessagesPerConv); - if (header.count < kMaxMessagesPerConv) + if (!already_stored) { - header.count = static_cast(header.count + 1U); - } - - header.reserved = encodeUnreadState(unread); - if (!file.seek(0) || !writeExact(file, &header, sizeof(header))) - { - file.close(); - return false; - } - if (!file.flush()) - { - file.close(); - return false; - } - file.close(); - - if (!writeReadStateUnread(conv, unread)) - { - CHAT_STORE_LOG("[AppContext] chat unread persist failed stage=read_state unread=%u\n", - static_cast(unread)); - unread_reconcile_pending_ = true; - if (!removeReadStateEntry(conv)) + const uint32_t sequence = stored_count + 1U; + if (!appendMessageRecord(message, sequence)) { return false; } + stored_count = sequence; } - updateIndexForMessage(msg, unread); - if (chat::hasReticulumLxmfMessageHash(msg)) + + if (incoming_commit && chat::hasReticulumLxmfMessageHash(message) && + !rememberReticulumHash(message.reticulum_lxmf_hash)) { - return rememberReticulumLxmfMessageHash(msg.reticulum_lxmf_hash); + // The message record is authoritative and retry detection above is + // idempotent. Do not publish until its Reticulum dedup identity is also + // durable. + return false; } + + storage_v2::ChatCatalogProjection* projection = findCatalog(conversation); + const bool new_projection = projection == nullptr; + if (!projection) + { + storage_v2::ChatCatalogProjection created{}; + created.conversation = conversation; + catalog_.push_back(created); + projection = &catalog_.back(); + } + const bool projection_was_current = + projection->message_count == stored_count && + projection->last_message_id == message.msg_id; + if (!projection_was_current || new_projection) + { + projection->conversation = conversation; + projection->message_count = stored_count; + projection->last_sequence = stored_count; + projection->last_message_id = message.msg_id; + projection->last_timestamp = message.timestamp; + projection->last_status = message.status; + projection->deleted = false; + const storage_v2::ChatReadProjection* read_state = + findReadState(conversation); + projection->unread = countUnreadAfter( + conversation, + read_state ? read_state->last_read_sequence : 0U); + copyTextPreview(projection->preview, + sizeof(projection->preview), + message.text); + if (!appendCatalogProjection(*projection)) + { + projection_dirty_[protocolIndex(message.protocol)] = true; + CHAT_STORE_LOG("[ChatStoreV2] projection deferred protocol=%s msg=%08lX authoritative=1\n", + protocolSlug(message.protocol), + static_cast(message.msg_id)); + } + } + + CHAT_STORE_LOG("[ChatStoreV2] commit protocol=%s msg=%08lX seq=%lu duplicate=%u publish=1\n", + protocolSlug(message.protocol), + static_cast(message.msg_id), + static_cast(stored_count), + already_stored ? 1U : 0U); return true; } -std::vector SdStore::loadRecent(const ConversationId& conv, size_t n) +std::vector SdStore::loadRecent(const ConversationId& conv, + std::size_t n) { return loadPageFromLatest(conv, 0, n, nullptr); } -std::vector SdStore::loadPageFromLatest(const ConversationId& conv, - size_t offset_from_latest, - size_t limit, - size_t* total) +std::vector SdStore::loadPageFromLatest( + const ConversationId& input, + std::size_t offset_from_latest, + std::size_t limit, + std::size_t* total) { - std::vector out; + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + const uint32_t count = messageCountOnDisk(conversation); if (total) { - *total = 0; - } - if (!ready_ || limit == 0) - { - return out; + *total = count; } - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (!storage::sd_exists(path)) + std::vector result; + if (limit == 0U || offset_from_latest >= count) { - return out; + return result; } - - storage::SdRuntimeFile file; - if (!file.open(path, "r")) + const uint32_t end = count - static_cast(offset_from_latest); + const uint32_t start = + end > limit ? end - static_cast(limit) : 0U; + result.reserve(end - start); + for (uint32_t ordinal = start; ordinal < end; ++ordinal) { - return out; - } - - FileHeader header{}; - if (!loadFileHeader(file, header) || header.count == 0) - { - file.close(); - return out; - } - - if (total) - { - *total = header.count; - } - if (offset_from_latest >= header.count) - { - file.close(); - return out; - } - - const size_t available = header.count - offset_from_latest; - const size_t to_read = std::min(limit, available); - const size_t logical_start = header.count - offset_from_latest - to_read; - const uint16_t start = static_cast( - (header.head + kMaxMessagesPerConv - header.count + logical_start) % - kMaxMessagesPerConv); - - out.reserve(to_read); - for (size_t index = 0; index < to_read; ++index) - { - const uint16_t slot = static_cast((start + index) % kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec) || rec.text_len == 0) + ChatMessage message{}; + if (readMessageByOrdinal(conversation, ordinal, message)) { - continue; + result.push_back(std::move(message)); } - out.push_back(messageFromRecord(rec)); } - file.close(); - return out; + return result; } -std::vector SdStore::loadConversationPage(size_t offset, - size_t limit, - size_t* total) +std::vector SdStore::loadConversationPage( + std::size_t offset, + std::size_t limit, + std::size_t* total) { - std::vector entries; - if (!ready_ || !ensureIndex(entries)) + std::vector result; + result.reserve(catalog_.size()); + for (const storage_v2::ChatCatalogProjection& projection : catalog_) { - if (total) + if (!projection.deleted && projection.message_count != 0U) { - *total = 0; + result.push_back(makeMeta(projection)); } - return {}; } - - std::sort(entries.begin(), - entries.end(), - [](const IndexEntry& a, const IndexEntry& b) - { - return a.last_timestamp > b.last_timestamp; - }); - + std::stable_sort(result.begin(), + result.end(), + [](const ConversationMeta& lhs, + const ConversationMeta& rhs) + { + return lhs.last_timestamp > rhs.last_timestamp; + }); if (total) { - *total = entries.size(); + *total = result.size(); } - if (offset >= entries.size()) + if (offset >= result.size()) { return {}; } - - size_t end = entries.size(); - if (limit != 0 && offset + limit < end) - { - end = offset + limit; - } - - std::vector list; - list.reserve(end - offset); - for (size_t index = offset; index < end; ++index) - { - list.push_back(metaFromIndexEntry(entries[index])); - } - return list; + const std::size_t end = + limit == 0U ? result.size() + : std::min(result.size(), offset + limit); + return std::vector( + result.begin() + static_cast(offset), + result.begin() + static_cast(end)); } -bool SdStore::setUnread(const ConversationId& conv, int unread) +std::vector SdStore::loadConversationPageForProtocol( + MeshProtocol protocol, + std::size_t offset, + std::size_t limit, + std::size_t* total) { - if (!ready_) + protocol = normalizeProtocol(protocol); + std::vector result; + for (const storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (!projection.deleted && projection.message_count != 0U && + sameProtocol(projection.conversation.protocol, protocol)) + { + result.push_back(makeMeta(projection)); + } + } + std::stable_sort(result.begin(), + result.end(), + [](const ConversationMeta& lhs, + const ConversationMeta& rhs) + { + return lhs.last_timestamp > rhs.last_timestamp; + }); + if (total) + { + *total = result.size(); + } + if (offset >= result.size()) + { + return {}; + } + const std::size_t end = + limit == 0U ? result.size() + : std::min(result.size(), offset + limit); + return std::vector( + result.begin() + static_cast(offset), + result.begin() + static_cast(end)); +} + +bool SdStore::setUnread(const ConversationId& input, int unread) +{ + storage_runtime::PersistenceBusGate transaction_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 0, + kChatStoreBusResource, + kChatStoreBusOwnerId + 2U, + kChatStoreBusOwnerId); + if (!transaction_gate.locked() || !ready_) { return false; } - - const uint16_t unread_count = static_cast( - std::min(std::max(0, unread), kUnreadStateCountMask)); - if (!writeReadStateUnread(conv, unread_count)) + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + storage_v2::ChatCatalogProjection* catalog = findCatalog(conversation); + if (!catalog) + { + return unread == 0; + } + const uint32_t bounded_unread = + unread <= 0 ? 0U : static_cast(unread); + storage_v2::ChatReadProjection projection{}; + projection.conversation = conversation; + projection.last_read_sequence = + sequenceForUnread(conversation, bounded_unread); + if (!appendReadProjection(projection)) { - CHAT_STORE_LOG("[AppContext] chat unread persist failed stage=read_state unread=%u\n", - static_cast(unread_count)); return false; } - - const bool header_written = writeConversationUnread(conv, unread_count); - if (!header_written) + storage_v2::ChatReadProjection* current = findReadState(conversation); + if (current) { - CHAT_STORE_LOG("[AppContext] chat unread persist failed stage=conversation_header unread=%u\n", - static_cast(unread_count)); + *current = projection; } - - std::vector entries; - if (!ensureIndex(entries)) + else { - unread_reconcile_pending_ = true; - return true; + read_state_.push_back(projection); } - - size_t index = 0; - if (!findIndexEntry(conv, entries, &index)) + catalog->unread = countUnreadAfter(conversation, + projection.last_read_sequence); + if (!appendCatalogProjection(*catalog)) { - return true; - } - entries[index].unread = unread_count; - const bool index_written = writeIndex(entries); - if (!index_written) - { - CHAT_STORE_LOG("[AppContext] chat unread persist failed stage=index unread=%u\n", - static_cast(unread_count)); - unread_reconcile_pending_ = true; - rebuildIndex(); + projection_dirty_[protocolIndex(conversation.protocol)] = true; } return true; } -int SdStore::getUnread(const ConversationId& conv) const +int SdStore::getUnread(const ConversationId& input) const { - if (!ready_) - { - return 0; - } - - uint16_t unread = 0; - if (readStateUnreadOrLegacy(conv, &unread)) - { - return unread; - } - return 0; + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + const storage_v2::ChatCatalogProjection* projection = + findCatalog(conversation); + return projection ? static_cast(projection->unread) : 0; } -void SdStore::clearConversation(const ConversationId& conv) +void SdStore::clearConversation(const ConversationId& input) { - if (!ready_) + storage_runtime::PersistenceBusGate transaction_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 0, + kChatStoreBusResource, + kChatStoreBusOwnerId + 3U, + kChatStoreBusOwnerId); + if (!transaction_gate.locked() || !ready_) + { + return; + } + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + char path[128]{}; + buildConversationDirectory(conversation, path, sizeof(path)); + if (!removeTree(path)) { return; } - (void)removeReadStateEntry(conv); - - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (storage::sd_exists(path)) + storage_v2::ChatCatalogProjection tombstone{}; + if (const storage_v2::ChatCatalogProjection* existing = + findCatalog(conversation)) { - storage::sd_remove(path); + tombstone = *existing; } + tombstone.conversation = conversation; + tombstone.deleted = true; + (void)appendCatalogProjection(tombstone); - std::vector entries; - if (!readIndex(entries)) - { - return; - } - entries.erase(std::remove_if(entries.begin(), - entries.end(), - [&](const IndexEntry& entry) - { - return indexEntryMatchesConversation(entry, conv); - }), - entries.end()); - (void)writeIndex(entries); + storage_v2::ChatReadProjection read_tombstone{}; + read_tombstone.conversation = conversation; + read_tombstone.deleted = true; + (void)appendReadProjection(read_tombstone); + + catalog_.erase(std::remove_if(catalog_.begin(), + catalog_.end(), + [&](const auto& value) + { + return sameConversationKey( + value.conversation, + conversation); + }), + catalog_.end()); + read_state_.erase(std::remove_if(read_state_.begin(), + read_state_.end(), + [&](const auto& value) + { + return sameConversationKey( + value.conversation, + conversation); + }), + read_state_.end()); } void SdStore::clearAll() { - if (!ready_) + storage_runtime::PersistenceBusGate transaction_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::RecoveryExclusive, + 0, + kChatStoreBusResource, + kChatStoreBusOwnerId + 4U, + kChatStoreBusOwnerId); + if (!transaction_gate.locked()) { return; } - - if (storage::sd_exists(kIndexFile)) + for (MeshProtocol protocol : kProtocols) { - storage::sd_remove(kIndexFile); + (void)removeTree(protocolRoot(protocol)); } - if (storage::sd_exists(kTempIndexFile)) - { - storage::sd_remove(kTempIndexFile); - } - if (storage::sd_exists(kBackupIndexFile)) - { - storage::sd_remove(kBackupIndexFile); - } - if (storage::sd_exists(kReadStateFile)) - { - storage::sd_remove(kReadStateFile); - } - if (storage::sd_exists(kTempReadStateFile)) - { - storage::sd_remove(kTempReadStateFile); - } - if (storage::sd_exists(kBackupReadStateFile)) - { - storage::sd_remove(kBackupReadStateFile); - } - - std::vector log_paths; - storage::SdRuntimeDir dir; - if (dir.open(kDir)) - { - char name[96]{}; - bool is_dir = false; - while (dir.read_next(name, sizeof(name), &is_dir)) - { - if (!is_dir && hasLogSuffix(name)) - { - log_paths.push_back(pathInChatDir(name)); - } - } - dir.close(); - } - - for (const auto& path : log_paths) - { - if (!path.empty() && storage::sd_exists(path.c_str())) - { - storage::sd_remove(path.c_str()); - } - } - unread_reconcile_pending_ = false; + catalog_.clear(); + read_state_.clear(); + statuses_.clear(); + seen_hot_.clear(); + std::memset(projection_dirty_, 0, sizeof(projection_dirty_)); + ready_ = ensureLayout(); } bool SdStore::updateMessageStatus(MessageId msg_id, MessageStatus status) { - if (!ready_ || msg_id == 0) + ChatMessage message{}; + if (!getMessage(msg_id, &message)) { return false; } - - std::vector entries; - if (!readIndex(entries)) - { - return false; - } - - bool updated = false; - for (auto& entry : entries) - { - const ConversationId conv = conversationFromIndexEntry(entry); - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (!storage::sd_exists(path)) - { - continue; - } - - storage::SdRuntimeFile file; - if (!file.open(path, "r+")) - { - continue; - } - - FileHeader header{}; - if (!loadFileHeader(file, header) || - !upgradeConversationFile(file, path, header)) - { - file.close(); - continue; - } - - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec)) - { - continue; - } - if (rec.msg_id != msg_id || rec.from != 0) - { - continue; - } - - rec.status = static_cast(status); - updated = writeRecord(file, slot, rec); - if (updated) - { - file.flush(); - if (entry.last_msg_id == msg_id) - { - entry.status = static_cast(status); - } - } - break; - } - - file.close(); - if (updated) - { - break; - } - } - - if (updated) - { - (void)writeIndex(entries); - } - return updated; + return updateMessageStatusForProtocol(msg_id, message.protocol, status); } bool SdStore::updateMessageStatusForProtocol(MessageId msg_id, MeshProtocol protocol, MessageStatus status) { - if (!ready_ || msg_id == 0) + protocol = normalizeProtocol(protocol); + if (msg_id == 0U || !storage_v2::supportedProtocol(protocol)) + { + return false; + } + ChatMessage message{}; + if (!getMessageForProtocol(msg_id, protocol, &message) || + message.from != 0U) { return false; } - std::vector entries; - if (!readIndex(entries)) + storage_runtime::PersistenceBusGate transaction_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::DurableCommit, + 0, + kChatStoreBusResource, + kChatStoreBusOwnerId + 5U, + kChatStoreBusOwnerId); + if (!transaction_gate.locked()) { return false; } - - bool updated = false; - for (auto& entry : entries) + storage_v2::ChatStatusProjection projection{}; + projection.message_id = msg_id; + projection.status = status; + if (const storage_v2::ChatStatusProjection* current = + findStatus(msg_id, protocol)) { - if (static_cast(entry.protocol) != protocol) - { - continue; - } - const ConversationId conv = conversationFromIndexEntry(entry); - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (!storage::sd_exists(path)) - { - continue; - } - - storage::SdRuntimeFile file; - if (!file.open(path, "r+")) - { - continue; - } - - FileHeader header{}; - if (!loadFileHeader(file, header) || - !upgradeConversationFile(file, path, header)) - { - file.close(); - continue; - } - - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec)) - { - continue; - } - if (rec.msg_id != msg_id || rec.from != 0 || - static_cast(rec.protocol) != protocol) - { - continue; - } - - rec.status = static_cast(status); - updated = writeRecord(file, slot, rec); - if (updated) - { - file.flush(); - if (entry.last_msg_id == msg_id) - { - entry.status = static_cast(status); - } - } - break; - } - - file.close(); - if (updated) - { - break; - } + projection.sequence = current->sequence + 1U; + } + else + { + projection.sequence = 1U; + } + if (!appendStatusProjection(protocol, projection)) + { + return false; + } + storage_v2::ChatStatusProjection* current = findStatus(msg_id, protocol); + if (current) + { + *current = projection; + } + else + { + ProtocolStatusProjection state{}; + state.protocol = protocol; + state.value = projection; + statuses_.push_back(state); } - if (updated) + const ConversationId conversation = conversationIdForMessage(message); + storage_v2::ChatCatalogProjection* catalog = findCatalog(conversation); + if (catalog && catalog->last_message_id == msg_id) { - (void)writeIndex(entries); + catalog->last_status = status; + if (!appendCatalogProjection(*catalog)) + { + projection_dirty_[protocolIndex(protocol)] = true; + } } - return updated; + return true; } bool SdStore::getMessage(MessageId msg_id, ChatMessage* out) const { - if (!ready_ || msg_id == 0) + for (MeshProtocol protocol : kProtocols) { - return false; - } - - std::vector entries; - if (!readIndex(entries)) - { - return false; - } - - for (const auto& entry : entries) - { - const ConversationId conv = conversationFromIndexEntry(entry); - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (!storage::sd_exists(path)) + if (getMessageForProtocol(msg_id, protocol, out)) { - continue; - } - - storage::SdRuntimeFile file; - if (!file.open(path, "r")) - { - continue; - } - - FileHeader header{}; - if (!loadFileHeader(file, header)) - { - file.close(); - continue; - } - - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec) || rec.text_len == 0 || rec.msg_id != msg_id) - { - continue; - } - if (out) - { - *out = messageFromRecord(rec); - } - file.close(); return true; } - file.close(); } return false; } @@ -773,1484 +662,1739 @@ bool SdStore::getMessageForProtocol(MessageId msg_id, MeshProtocol protocol, ChatMessage* out) const { - if (!ready_ || msg_id == 0) + protocol = normalizeProtocol(protocol); + if (msg_id == 0U) { return false; } - - std::vector entries; - if (!readIndex(entries)) + for (const storage_v2::ChatCatalogProjection& projection : catalog_) { - return false; - } - - for (const auto& entry : entries) - { - if (static_cast(entry.protocol) != protocol) + if (projection.deleted || + !sameProtocol(projection.conversation.protocol, protocol)) { continue; } - const ConversationId conv = conversationFromIndexEntry(entry); - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - if (!storage::sd_exists(path)) + for (uint32_t ordinal = projection.message_count; ordinal > 0U; + --ordinal) { - continue; - } - - storage::SdRuntimeFile file; - if (!file.open(path, "r")) - { - continue; - } - - FileHeader header{}; - if (!loadFileHeader(file, header)) - { - file.close(); - continue; - } - - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec) || rec.text_len == 0 || - rec.msg_id != msg_id || - static_cast(rec.protocol) != protocol) + ChatMessage message{}; + if (!readMessageByOrdinal(projection.conversation, + ordinal - 1U, + message) || + message.msg_id != msg_id) { continue; } if (out) { - *out = messageFromRecord(rec); + *out = std::move(message); } - file.close(); return true; } - file.close(); } return false; } -bool SdStore::hasReticulumLxmfMessageHash(const uint8_t* lxmf_hash) const +bool SdStore::hasReticulumLxmfMessageHash(const uint8_t* hash) const { - if (!ready_ || !validLxmfMessageHash(lxmf_hash)) + if (!hash || isAllZeroKeyBytes(hash, kReticulumLxmfHashSize)) { return false; } - - storage::SdRuntimeFile file; - if (!file.open(kReticulumLxmfSeenFile, "r")) + for (const storage_v2::ReticulumSeenProjection& seen : seen_hot_) { - return false; - } - - LxmfSeenHeader header{}; - if (!loadLxmfSeenHeader(file, header)) - { - file.close(); - return false; - } - - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxReticulumLxmfSeen - header.count + index) % - kMaxReticulumLxmfSeen); - LxmfSeenRecord rec{}; - if (!readLxmfSeenRecord(file, slot, rec)) + if (std::memcmp(seen.hash, hash, sizeof(seen.hash)) == 0) { - continue; - } - if (std::memcmp(rec.hash, lxmf_hash, kReticulumLxmfHashSize) == 0) - { - file.close(); return true; } } - file.close(); + + for (const char* name : {"seen.delta", "seen.snapshot"}) + { + char path[128]{}; + buildProjectionPath(MeshProtocol::Reticulum, + name, + path, + sizeof(path)); + const auto inspection = journal_.inspect( + path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + storage_v2::reticulumSeenSlotSize()); + for (uint32_t index = inspection.slot_count; index > 0U; --index) + { + if (!journal_.read(path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + storage_v2::reticulumSeenSlotSize(), + index - 1U, + scratch_.data())) + { + continue; + } + storage_v2::ReticulumSeenProjection projection{}; + if (storage_v2::decodeReticulumSeenSlot( + scratch_.data(), + storage_v2::reticulumSeenSlotSize(), + projection) && + std::memcmp(projection.hash, hash, sizeof(projection.hash)) == 0) + { + return true; + } + } + } return false; } void SdStore::flush() { + if (!ready_) + { + return; + } + const uint32_t now_ms = millis(); + if (last_projection_retry_ms_ != 0U && + now_ms - last_projection_retry_ms_ < kProjectionRetryIntervalMs) + { + return; + } + MeshProtocol protocol = MeshProtocol::Meshtastic; + bool found_dirty = false; + for (std::size_t offset = 0U; offset < 3U; ++offset) + { + const std::size_t index = + (static_cast(flush_protocol_cursor_) + offset) % 3U; + if (projection_dirty_[index]) + { + protocol = kProtocols[index]; + flush_protocol_cursor_ = static_cast((index + 1U) % 3U); + found_dirty = true; + break; + } + } + if (!found_dirty) + { + return; + } + last_projection_retry_ms_ = now_ms; + storage_runtime::PersistenceBusGate maintenance_gate( + s_chat_store_bus_arbiter, + sys::runtime::BusAccessPolicy::BackgroundWorkerBounded, + 0U, + kChatStoreBusResource, + kChatStoreBusOwnerId + 10U, + kChatStoreBusOwnerId); + if (maintenance_gate.locked()) + { + (void)compactProtocolProjections(protocol); + } } -bool SdStore::ensureFs() const +bool SdStore::ensureLayout() const { - return storage::sd_card_ready(); -} - -bool SdStore::ensureDir() const -{ - if (!ensureFs()) + if (!storage_runtime::sd_card_ready() || !ensureDirectory("/data") || + !ensureDirectory(kRoot)) { return false; } - if (storage::sd_is_directory(kDir)) + for (MeshProtocol protocol : kProtocols) { - return true; - } - return storage::sd_mkdir(kDir); -} - -bool SdStore::recoverIndex() const -{ - if (storage::sd_exists(kIndexFile)) - { - return false; - } - const auto restore_candidate = [this](const char* path, const char* source) - { - if (!storage::sd_exists(path) || !storage::sd_rename(path, kIndexFile)) + if (!ensureProtocolLayout(protocol)) { return false; } - std::vector recovered; - if (readIndex(recovered)) - { - CHAT_STORE_LOG("[AppContext] chat store=SdStore recovered index source=%s\n", - source); - return true; - } - (void)storage::sd_remove(kIndexFile); - return false; - }; + } + return true; +} - if (restore_candidate(kTempIndexFile, "temp")) +bool SdStore::ensureProtocolLayout(MeshProtocol protocol) const +{ + protocol = normalizeProtocol(protocol); + const char* slug = protocolSlug(protocol); + char protocol_dir[64]{}; + char conversations[96]{}; + std::snprintf(protocol_dir, + sizeof(protocol_dir), + "%s/%s", + kRoot, + slug); + std::snprintf(conversations, + sizeof(conversations), + "%s/conversations", + protocolRoot(protocol)); + return ensureDirectory(protocol_dir) && + ensureDirectory(protocolRoot(protocol)) && + ensureDirectory(conversations); +} + +bool SdStore::loadRuntimeState() +{ + catalog_.clear(); + read_state_.clear(); + statuses_.clear(); + seen_hot_.clear(); + bool ok = true; + for (MeshProtocol protocol : kProtocols) + { + ok = loadProtocolState(protocol) && ok; + } + ok = loadSeenJournal() && ok; + return ok; +} + +bool SdStore::loadProtocolState(MeshProtocol protocol) +{ + if (!recoverProjectionSnapshot(protocol, "catalog") || + !recoverProjectionSnapshot(protocol, "read") || + !recoverProjectionSnapshot(protocol, "status")) + { + return false; + } + bool ok = true; + ok = loadCatalogJournal(protocol, "catalog.snapshot") && ok; + ok = loadCatalogJournal(protocol, "catalog.delta") && ok; + ok = loadReadJournal(protocol, "read.snapshot") && ok; + ok = loadReadJournal(protocol, "read.delta") && ok; + ok = loadStatusJournal(protocol, "status.snapshot") && ok; + ok = loadStatusJournal(protocol, "status.delta") && ok; + return reconcileProtocolCatalog(protocol) && ok; +} + +bool SdStore::loadCatalogJournal(MeshProtocol protocol, const char* name) +{ + char path[128]{}; + buildProjectionPath(protocol, name, path, sizeof(path)); + const auto inspection = journal_.inspect( + path, + protocol, + hasSuffix(name, ".snapshot") + ? storage_v2::JournalKind::CatalogSnapshot + : storage_v2::JournalKind::CatalogDelta, + storage_v2::catalogSlotSize(protocol)); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) { return true; } - return restore_candidate(kBackupIndexFile, "backup"); -} - -bool SdStore::reconcileIndexUnread(std::vector& entries) const -{ - bool changed = false; - for (auto& entry : entries) + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) { - const ConversationId conv = conversationFromIndexEntry(entry); - uint16_t durable_unread = 0; - if (!readStateUnreadOrLegacy(conv, &durable_unread)) + projection_dirty_[protocolIndex(protocol)] = true; + return true; + } + const storage_v2::JournalKind kind = + hasSuffix(name, ".snapshot") + ? storage_v2::JournalKind::CatalogSnapshot + : storage_v2::JournalKind::CatalogDelta; + for (uint32_t index = 0; index < inspection.slot_count; ++index) + { + storage_v2::ChatCatalogProjection projection{}; + if (!journal_.read(path, + protocol, + kind, + storage_v2::catalogSlotSize(protocol), + index, + scratch_.data()) || + !storage_v2::decodeCatalogSlot(protocol, + scratch_.data(), + storage_v2::catalogSlotSize(protocol), + projection)) { - durable_unread = entry.unread; - (void)writeReadStateUnread(conv, durable_unread); + projection_dirty_[protocolIndex(protocol)] = true; + continue; } - if (entry.unread != durable_unread) + storage_v2::ChatCatalogProjection* existing = + findCatalog(projection.conversation); + if (projection.deleted) { - entry.unread = durable_unread; - changed = true; - } - } - return changed; -} - -bool SdStore::readIndex(std::vector& entries) const -{ - entries.clear(); - if (!ensureFs() || !storage::sd_exists(kIndexFile)) - { - return false; - } - - storage::SdRuntimeFile file; - if (!file.open(kIndexFile, "r")) - { - return false; - } - - IndexHeader header{}; - if (!readExact(file, &header, sizeof(header)) || - header.magic != kIndexMagic || - (header.version != kIndexVersion && header.version != kLegacyVersion)) - { - file.close(); - return false; - } - - entries.reserve(header.count); - bool ok = true; - for (uint16_t index = 0; ok && index < header.count; ++index) - { - IndexEntry entry{}; - if (header.version == kLegacyVersion) - { - IndexEntryV2 legacy_entry{}; - ok = readExact(file, &legacy_entry, sizeof(legacy_entry)); - if (ok) + if (existing) { - entry.protocol = legacy_entry.protocol; - entry.channel = legacy_entry.channel; - entry.status = legacy_entry.status; - entry.unread = legacy_entry.unread; - entry.peer = legacy_entry.peer; - entry.last_msg_id = legacy_entry.last_msg_id; - entry.last_timestamp = legacy_entry.last_timestamp; - entry.last_from = legacy_entry.last_from; - entry.preview_len = legacy_entry.preview_len; - std::memcpy(entry.preview, - legacy_entry.preview, - sizeof(entry.preview)); + catalog_.erase(catalog_.begin() + + static_cast(existing - + catalog_.data())); } } + else if (existing) + { + *existing = projection; + } else { - ok = readExact(file, &entry, sizeof(entry)); + catalog_.push_back(projection); } - if (ok) - { - entries.push_back(entry); - } - } - file.close(); - if (!ok) - { - entries.clear(); - return false; } return true; } -bool SdStore::writeIndex(const std::vector& entries) const +bool SdStore::loadReadJournal(MeshProtocol protocol, const char* name) { - if (!ensureDir()) + char path[128]{}; + buildProjectionPath(protocol, name, path, sizeof(path)); + const storage_v2::JournalKind kind = + hasSuffix(name, ".snapshot") + ? storage_v2::JournalKind::ReadStateSnapshot + : storage_v2::JournalKind::ReadStateDelta; + const auto inspection = journal_.inspect( + path, + protocol, + kind, + storage_v2::readStateSlotSize(protocol)); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) { - return false; + return true; } - - if (storage::sd_exists(kTempIndexFile)) + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) { - storage::sd_remove(kTempIndexFile); + projection_dirty_[protocolIndex(protocol)] = true; + return true; } - - storage::SdRuntimeFile file; - if (!file.open(kTempIndexFile, "w")) + for (uint32_t index = 0; index < inspection.slot_count; ++index) { - return false; - } - - IndexHeader header{}; - header.magic = kIndexMagic; - header.version = kIndexVersion; - header.count = static_cast(std::min(entries.size(), 0xFFFFU)); - bool ok = writeExact(file, &header, sizeof(header)); - for (size_t index = 0; ok && index < header.count; ++index) - { - ok = writeExact(file, &entries[index], sizeof(IndexEntry)); - } - file.flush(); - file.close(); - - if (!ok) - { - storage::sd_remove(kTempIndexFile); - return false; - } - - const bool had_index = storage::sd_exists(kIndexFile); - if (had_index) - { - if (storage::sd_exists(kBackupIndexFile) && - !storage::sd_remove(kBackupIndexFile)) + storage_v2::ChatReadProjection projection{}; + if (!journal_.read(path, + protocol, + kind, + storage_v2::readStateSlotSize(protocol), + index, + scratch_.data()) || + !storage_v2::decodeReadStateSlot( + protocol, + scratch_.data(), + storage_v2::readStateSlotSize(protocol), + projection)) { - storage::sd_remove(kTempIndexFile); - return false; + projection_dirty_[protocolIndex(protocol)] = true; + continue; } - if (!storage::sd_rename(kIndexFile, kBackupIndexFile)) + storage_v2::ChatReadProjection* existing = + findReadState(projection.conversation); + if (projection.deleted) { - storage::sd_remove(kTempIndexFile); - return false; - } - } - if (!storage::sd_rename(kTempIndexFile, kIndexFile)) - { - if (had_index && !storage::sd_exists(kIndexFile)) - { - (void)storage::sd_rename(kBackupIndexFile, kIndexFile); - } - storage::sd_remove(kTempIndexFile); - return false; - } - if (storage::sd_exists(kBackupIndexFile)) - { - (void)storage::sd_remove(kBackupIndexFile); - } - return true; -} - -bool SdStore::readReadState(std::vector& entries) const -{ - entries.clear(); - if (!ensureFs() || !storage::sd_exists(kReadStateFile)) - { - return false; - } - - storage::SdRuntimeFile file; - if (!file.open(kReadStateFile, "r")) - { - return false; - } - - ReadStateHeader header{}; - if (!readExact(file, &header, sizeof(header)) || - header.magic != kReadStateMagic || - header.version != kReadStateVersion) - { - file.close(); - return false; - } - - entries.reserve(header.count); - bool ok = true; - for (uint16_t index = 0; ok && index < header.count; ++index) - { - ReadStateEntry entry{}; - ok = readExact(file, &entry, sizeof(entry)); - if (ok) - { - entries.push_back(entry); - } - } - file.close(); - if (!ok) - { - entries.clear(); - return false; - } - return true; -} - -bool SdStore::writeReadState(const std::vector& entries) const -{ - if (!ensureDir()) - { - return false; - } - - if (storage::sd_exists(kTempReadStateFile)) - { - storage::sd_remove(kTempReadStateFile); - } - - storage::SdRuntimeFile file; - if (!file.open(kTempReadStateFile, "w")) - { - return false; - } - - ReadStateHeader header{}; - header.magic = kReadStateMagic; - header.version = kReadStateVersion; - header.count = static_cast(std::min(entries.size(), 0xFFFFU)); - bool ok = writeExact(file, &header, sizeof(header)); - for (size_t index = 0; ok && index < header.count; ++index) - { - ok = writeExact(file, &entries[index], sizeof(ReadStateEntry)); - } - file.flush(); - file.close(); - - if (!ok) - { - storage::sd_remove(kTempReadStateFile); - return false; - } - - const bool had_state = storage::sd_exists(kReadStateFile); - if (had_state) - { - if (storage::sd_exists(kBackupReadStateFile) && - !storage::sd_remove(kBackupReadStateFile)) - { - storage::sd_remove(kTempReadStateFile); - return false; - } - if (!storage::sd_rename(kReadStateFile, kBackupReadStateFile)) - { - storage::sd_remove(kTempReadStateFile); - return false; - } - } - if (!storage::sd_rename(kTempReadStateFile, kReadStateFile)) - { - if (had_state && !storage::sd_exists(kReadStateFile)) - { - (void)storage::sd_rename(kBackupReadStateFile, kReadStateFile); - } - storage::sd_remove(kTempReadStateFile); - return false; - } - if (storage::sd_exists(kBackupReadStateFile)) - { - (void)storage::sd_remove(kBackupReadStateFile); - } - return true; -} - -bool SdStore::findReadStateEntry(const ConversationId& conv, - std::vector& entries, - size_t* out_idx) const -{ - return findReadStateEntry( - conv, - static_cast&>(entries), - out_idx); -} - -bool SdStore::findReadStateEntry(const ConversationId& conv, - const std::vector& entries, - size_t* out_idx) const -{ - for (size_t index = 0; index < entries.size(); ++index) - { - if (readStateEntryMatchesConversation(entries[index], conv)) - { - if (out_idx) + if (existing) { - *out_idx = index; + read_state_.erase( + read_state_.begin() + + static_cast(existing - read_state_.data())); } - return true; } - } - return false; -} - -bool SdStore::readStateUnreadOnly(const ConversationId& conv, uint16_t* unread) const -{ - if (!unread) - { - return false; - } - std::vector entries; - if (!readReadState(entries)) - { - return false; - } - size_t index = 0; - if (!findReadStateEntry(conv, entries, &index)) - { - return false; - } - *unread = entries[index].unread; - return true; -} - -bool SdStore::readStateUnreadOrLegacy(const ConversationId& conv, uint16_t* unread) const -{ - if (!unread) - { - return false; - } - if (readStateUnreadOnly(conv, unread)) - { - return true; - } - if (readConversationUnread(conv, unread)) - { - (void)writeReadStateUnread(conv, *unread); - return true; - } - std::vector entries; - size_t index = 0; - if (readIndex(entries) && findIndexEntry(conv, entries, &index)) - { - *unread = entries[index].unread; - (void)writeReadStateUnread(conv, *unread); - return true; - } - return false; -} - -bool SdStore::writeReadStateUnread(const ConversationId& conv, uint16_t unread) const -{ - std::vector entries; - if (!readReadState(entries)) - { - entries.clear(); - } - - size_t index = 0; - if (findReadStateEntry(conv, entries, &index)) - { - entries[index] = readStateEntryFromConversation(conv, unread); - } - else - { - entries.push_back(readStateEntryFromConversation(conv, unread)); - } - return writeReadState(entries); -} - -bool SdStore::removeReadStateEntry(const ConversationId& conv) const -{ - std::vector entries; - if (!readReadState(entries)) - { - return true; - } - const size_t before = entries.size(); - entries.erase(std::remove_if(entries.begin(), - entries.end(), - [&](const ReadStateEntry& entry) - { - return readStateEntryMatchesConversation(entry, conv); - }), - entries.end()); - if (entries.size() == before) - { - return true; - } - return writeReadState(entries); -} - -bool SdStore::ensureIndex(std::vector& entries) -{ - bool loaded = readIndex(entries); - if (!loaded && recoverIndex()) - { - loaded = readIndex(entries); - } - if (!loaded) - { - rebuildIndex(); - loaded = readIndex(entries); - } - if (!loaded) - { - return false; - } - if (unread_reconcile_pending_) - { - const bool changed = reconcileIndexUnread(entries); - if (!changed || writeIndex(entries)) + else if (existing) { - unread_reconcile_pending_ = false; + *existing = projection; + } + else + { + read_state_.push_back(projection); } } return true; } -bool SdStore::findIndexEntry(const ConversationId& conv, - std::vector& entries, - size_t* out_idx) const +bool SdStore::loadStatusJournal(MeshProtocol protocol, const char* name) { - return findIndexEntry(conv, - static_cast&>(entries), - out_idx); + char path[128]{}; + buildProjectionPath(protocol, name, path, sizeof(path)); + const storage_v2::JournalKind kind = + hasSuffix(name, ".snapshot") + ? storage_v2::JournalKind::StatusSnapshot + : storage_v2::JournalKind::StatusDelta; + const auto inspection = journal_.inspect(path, + protocol, + kind, + storage_v2::statusSlotSize()); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) + { + return true; + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + projection_dirty_[protocolIndex(protocol)] = true; + return true; + } + for (uint32_t index = 0; index < inspection.slot_count; ++index) + { + storage_v2::ChatStatusProjection projection{}; + if (!journal_.read(path, + protocol, + kind, + storage_v2::statusSlotSize(), + index, + scratch_.data()) || + !storage_v2::decodeStatusSlot(scratch_.data(), + storage_v2::statusSlotSize(), + projection)) + { + projection_dirty_[protocolIndex(protocol)] = true; + continue; + } + storage_v2::ChatStatusProjection* existing = + findStatus(projection.message_id, protocol); + if (existing) + { + *existing = projection; + } + else + { + ProtocolStatusProjection state{}; + state.protocol = normalizeProtocol(protocol); + state.value = projection; + statuses_.push_back(state); + } + } + return true; } -bool SdStore::findIndexEntry(const ConversationId& conv, - const std::vector& entries, - size_t* out_idx) const +bool SdStore::loadSeenJournal() { - for (size_t index = 0; index < entries.size(); ++index) + if (!recoverProjectionSnapshot(MeshProtocol::Reticulum, "seen")) { - const IndexEntry& entry = entries[index]; - if (indexEntryMatchesConversation(entry, conv)) + return false; + } + bool journal_found = false; + bool rebuild_required = false; + for (const char* name : {"seen.snapshot", "seen.delta"}) + { + char path[128]{}; + buildProjectionPath(MeshProtocol::Reticulum, + name, + path, + sizeof(path)); + const auto inspection = journal_.inspect( + path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + storage_v2::reticulumSeenSlotSize()); + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::Missing) { - if (out_idx) + continue; + } + journal_found = true; + if (inspection.state != + storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + rebuild_required = true; + continue; + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + rebuild_required = true; + } + const uint32_t start = + inspection.slot_count > kSeenHotCapacity + ? inspection.slot_count - static_cast(kSeenHotCapacity) + : 0U; + for (uint32_t index = start; index < inspection.slot_count; ++index) + { + storage_v2::ReticulumSeenProjection projection{}; + if (!journal_.read(path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + storage_v2::reticulumSeenSlotSize(), + index, + scratch_.data()) || + !storage_v2::decodeReticulumSeenSlot( + scratch_.data(), + storage_v2::reticulumSeenSlotSize(), + projection)) { - *out_idx = index; + rebuild_required = true; + continue; } - return true; + if (seen_hot_.size() == kSeenHotCapacity) + { + seen_hot_.erase(seen_hot_.begin()); + } + seen_hot_.push_back(projection); } } - return false; + if (!journal_found) + { + for (const storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (!projection.deleted && + sameProtocol(projection.conversation.protocol, + MeshProtocol::Reticulum) && + messageCountOnDisk(projection.conversation) > 0U) + { + rebuild_required = true; + break; + } + } + } + return !rebuild_required || rebuildSeenJournalFromMessages(); } -void SdStore::updateIndexForMessage(const ChatMessage& msg, uint16_t unread) +bool SdStore::rebuildSeenJournalFromMessages() { - std::vector entries; - if (!ensureIndex(entries)) + char final_path[128] = {}; + char temp_path[128] = {}; + char backup_path[128] = {}; + char delta_path[128] = {}; + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot", + final_path, + sizeof(final_path)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot.tmp", + temp_path, + sizeof(temp_path)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.snapshot.bak", + backup_path, + sizeof(backup_path)); + buildProjectionPath(MeshProtocol::Reticulum, + "seen.delta", + delta_path, + sizeof(delta_path)); + (void)storage_runtime::sd_remove(temp_path); + const std::size_t slot_size = storage_v2::reticulumSeenSlotSize(); + if (!journal_.create(temp_path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size)) { - entries.clear(); + return false; } - const ConversationId conv = conversationIdForMessage(msg); - size_t index = 0; - if (!findIndexEntry(conv, entries, &index)) + seen_hot_.clear(); + uint32_t rebuilt = 0U; + for (const storage_v2::ChatCatalogProjection& catalog : catalog_) { - IndexEntry entry{}; - entry.protocol = static_cast(msg.protocol); - entry.channel = static_cast(msg.channel); - entry.peer = msg.peer; - if (hasReticulumConversationKey(conv)) - { - entry.flags |= kIndexHasReticulumIdentityFlag; - copyReticulumIdentityToStorage(entry.reticulum_destination_hash, - entry.reticulum_identity_hash, - conv.reticulum_identity); - } - entries.push_back(entry); - index = entries.size() - 1; - } - - IndexEntry& entry = entries[index]; - entry.unread = std::min(unread, kUnreadStateCountMask); - entry.protocol = static_cast(msg.protocol); - entry.channel = static_cast(msg.channel); - entry.status = static_cast(msg.status); - entry.peer = msg.peer; - entry.flags &= static_cast(~kIndexHasReticulumIdentityFlag); - std::memset(entry.reticulum_destination_hash, 0, sizeof(entry.reticulum_destination_hash)); - std::memset(entry.reticulum_identity_hash, 0, sizeof(entry.reticulum_identity_hash)); - if (hasReticulumConversationKey(conv)) - { - entry.flags |= kIndexHasReticulumIdentityFlag; - copyReticulumIdentityToStorage(entry.reticulum_destination_hash, - entry.reticulum_identity_hash, - conv.reticulum_identity); - } - entry.last_msg_id = msg.msg_id; - entry.last_timestamp = msg.timestamp; - entry.last_from = msg.from; - entry.preview_len = static_cast(std::min(msg.text.size(), kPreviewLen)); - std::memset(entry.preview, 0, sizeof(entry.preview)); - if (entry.preview_len > 0) - { - std::memcpy(entry.preview, msg.text.data(), entry.preview_len); - } - const bool index_written = writeIndex(entries); - if (!index_written) - { - unread_reconcile_pending_ = true; - rebuildIndex(); - } -} - -void SdStore::rebuildIndex() -{ - if (!ensureDir()) - { - return; - } - - std::vector entries; - storage::SdRuntimeDir dir; - if (!dir.open(kDir)) - { - return; - } - - char name[96]{}; - bool is_dir = false; - while (dir.read_next(name, sizeof(name), &is_dir)) - { - if (is_dir || !hasLogSuffix(name)) + if (catalog.deleted || + !sameProtocol(catalog.conversation.protocol, + MeshProtocol::Reticulum)) { continue; } - - const std::string path = pathInChatDir(name); - storage::SdRuntimeFile file; - if (path.empty() || !file.open(path.c_str(), "r")) + const uint32_t message_count = + messageCountOnDisk(catalog.conversation); + for (uint32_t ordinal = 0U; ordinal < message_count; ++ordinal) { - continue; - } - - FileHeader header{}; - if (!loadFileHeader(file, header)) - { - file.close(); - continue; - } - - ChatMessage last_msg; - bool have_last = false; - uint16_t unread = 0; - const bool has_header_unread = decodeUnreadState(header.reserved, &unread); - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec) || rec.text_len == 0) + ChatMessage message{}; + if (!readMessageByOrdinal(catalog.conversation, + ordinal, + message) || + !chat::hasReticulumLxmfMessageHash(message)) { continue; } - ChatMessage msg = messageFromRecord(rec); - if (!has_header_unread && - msg.status == MessageStatus::Incoming && - unread < kUnreadStateCountMask) + storage_v2::ReticulumSeenProjection projection{}; + std::memcpy(projection.hash, + message.reticulum_lxmf_hash, + sizeof(projection.hash)); + if (!storage_v2::encodeReticulumSeenSlot(projection, + scratch_.data(), + slot_size) || + !journal_.append(temp_path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size, + scratch_.data())) { - ++unread; + (void)storage_runtime::sd_remove(temp_path); + return false; } - if (!have_last || msg.timestamp >= last_msg.timestamp) + if (seen_hot_.size() == kSeenHotCapacity) { - last_msg = msg; - have_last = true; + seen_hot_.erase(seen_hot_.begin()); } + seen_hot_.push_back(projection); + ++rebuilt; } - file.close(); + } + if (!storage_v2::replaceFileAtomically(temp_path, + final_path, + backup_path)) + { + return false; + } + (void)storage_runtime::sd_remove(delta_path); + CHAT_STORE_LOG("[ChatStoreV2] seen rebuilt hashes=%lu authoritative=messages\n", + static_cast(rebuilt)); + return true; +} - if (!have_last) +bool SdStore::recoverProjectionSnapshot(MeshProtocol protocol, + const char* base_name) +{ + if (!base_name || base_name[0] == '\0') + { + return false; + } + char final_path[128] = {}; + char temp_path[128] = {}; + char backup_path[128] = {}; + char name[40] = {}; + std::snprintf(name, sizeof(name), "%s.snapshot", base_name); + buildProjectionPath(protocol, name, final_path, sizeof(final_path)); + std::snprintf(name, sizeof(name), "%s.snapshot.tmp", base_name); + buildProjectionPath(protocol, name, temp_path, sizeof(temp_path)); + std::snprintf(name, sizeof(name), "%s.snapshot.bak", base_name); + buildProjectionPath(protocol, name, backup_path, sizeof(backup_path)); + return storage_v2::recoverAtomicFile(final_path, + temp_path, + backup_path); +} + +bool SdStore::reconcileProtocolCatalog(MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + for (storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (sameProtocol(projection.conversation.protocol, protocol)) + { + projection.deleted = true; + } + } + + char conversations_path[96]{}; + std::snprintf(conversations_path, + sizeof(conversations_path), + "%s/conversations", + protocolRoot(protocol)); + storage_runtime::SdRuntimeDir directory; + if (!directory.open(conversations_path)) + { + return false; + } + char name[80]{}; + bool is_directory = false; + while (directory.read_next(name, sizeof(name), &is_directory)) + { + if (is_directory && name[0] != '\0' && + !reconcileConversationDirectory(protocol, name)) + { + projection_dirty_[protocolIndex(protocol)] = true; + } + } + catalog_.erase(std::remove_if(catalog_.begin(), + catalog_.end(), + [&](const auto& value) + { + return sameProtocol( + value.conversation.protocol, + protocol) && + value.deleted; + }), + catalog_.end()); + return true; +} + +bool SdStore::reconcileConversationDirectory(MeshProtocol protocol, + const char* directory_name) +{ + char directory_path[128]{}; + std::snprintf(directory_path, + sizeof(directory_path), + "%s/conversations/%s", + protocolRoot(protocol), + directory_name); + const std::size_t slot_size = storage_v2::messageSlotSize(protocol); + uint32_t total_count = 0; + uint32_t last_segment = 0; + uint32_t last_segment_count = 0; + bool found_segment = false; + for (uint32_t segment = 0; segment < 10000U; ++segment) + { + char path[128]{}; + std::snprintf(path, + sizeof(path), + "%s/%04lu.msg", + directory_path, + static_cast(segment)); + auto inspection = journal_.inspect( + path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) + { + break; + } + if (inspection.state == + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + if (!repairPartialJournal(path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size)) + { + return false; + } + inspection = journal_.inspect( + path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready) + { + return false; + } + found_segment = true; + total_count += inspection.slot_count; + last_segment = segment; + last_segment_count = inspection.slot_count; + if (inspection.slot_count < slotsPerMessageSegment(protocol)) + { + break; + } + } + if (!found_segment || total_count == 0U || last_segment_count == 0U) + { + return true; + } + + char last_path[128]{}; + std::snprintf(last_path, + sizeof(last_path), + "%s/%04lu.msg", + directory_path, + static_cast(last_segment)); + if (!journal_.read(last_path, + protocol, + storage_v2::JournalKind::MessageSegment, + slot_size, + last_segment_count - 1U, + scratch_.data())) + { + return false; + } + ChatMessage latest{}; + uint32_t sequence = 0; + if (!storage_v2::decodeMessageSlot(protocol, + scratch_.data(), + slot_size, + latest, + &sequence)) + { + return false; + } + applyStoredStatus(latest); + const ConversationId conversation = conversationIdForMessage(latest); + storage_v2::ChatCatalogProjection* projection = findCatalog(conversation); + const bool catalog_current = + projection && projection->message_count == total_count && + projection->last_message_id == latest.msg_id && + projection->last_sequence == sequence; + if (!projection) + { + storage_v2::ChatCatalogProjection created{}; + created.conversation = conversation; + catalog_.push_back(created); + projection = &catalog_.back(); + } + const uint32_t last_read = + findReadState(conversation) + ? findReadState(conversation)->last_read_sequence + : 0U; + projection->conversation = conversation; + projection->message_count = total_count; + projection->last_sequence = sequence; + projection->last_message_id = latest.msg_id; + projection->last_timestamp = latest.timestamp; + projection->last_status = latest.status; + projection->deleted = false; + copyTextPreview(projection->preview, + sizeof(projection->preview), + latest.text); + if (!catalog_current) + { + projection->unread = countUnreadAfter(conversation, last_read); + projection_dirty_[protocolIndex(protocol)] = true; + } + return true; +} + +std::size_t SdStore::slotsPerMessageSegment(MeshProtocol protocol) const +{ + const std::size_t slot_size = storage_v2::messageSlotSize(protocol); + if (slot_size == 0U || kMessageSegmentBytes <= + storage_v2::FixedSlotJournalEngine::headerSize()) + { + return 0U; + } + return (kMessageSegmentBytes - + storage_v2::FixedSlotJournalEngine::headerSize()) / + slot_size; +} + +uint32_t SdStore::messageCountOnDisk(const ConversationId& conversation) const +{ + const std::size_t slot_size = + storage_v2::messageSlotSize(conversation.protocol); + uint32_t total_count = 0; + for (uint32_t segment = 0; segment < 10000U; ++segment) + { + char path[128]{}; + buildMessageSegmentPath(conversation, segment, path, sizeof(path)); + const auto inspection = journal_.inspect( + path, + conversation.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing) + { + break; + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + break; + } + total_count += inspection.slot_count; + if (inspection.slot_count < slotsPerMessageSegment(conversation.protocol)) + { + break; + } + } + return total_count; +} + +bool SdStore::readMessageByOrdinal(const ConversationId& input, + uint32_t ordinal, + ChatMessage& out_message, + uint32_t* out_sequence) const +{ + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + const std::size_t capacity = slotsPerMessageSegment(conversation.protocol); + const std::size_t slot_size = + storage_v2::messageSlotSize(conversation.protocol); + if (capacity == 0U || slot_size > scratch_.size()) + { + return false; + } + const uint32_t segment = static_cast(ordinal / capacity); + const uint32_t slot = static_cast(ordinal % capacity); + char path[128]{}; + buildMessageSegmentPath(conversation, segment, path, sizeof(path)); + if (!journal_.read(path, + conversation.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size, + slot, + scratch_.data()) || + !storage_v2::decodeMessageSlot(conversation.protocol, + scratch_.data(), + slot_size, + out_message, + out_sequence)) + { + return false; + } + applyStoredStatus(out_message); + return true; +} + +bool SdStore::latestStoredMessage(const ConversationId& conversation, + ChatMessage& out_message, + uint32_t* out_count) const +{ + const uint32_t count = messageCountOnDisk(conversation); + if (out_count) + { + *out_count = count; + } + return count != 0U && + readMessageByOrdinal(conversation, count - 1U, out_message); +} + +bool SdStore::storedMessageMatches(const ChatMessage& message, + bool* out_matches, + uint32_t* out_count) const +{ + if (!out_matches || !out_count) + { + return false; + } + *out_matches = false; + *out_count = 0U; + const ConversationId conversation = conversationIdForMessage(message); + ChatMessage latest{}; + if (!latestStoredMessage(conversation, latest, out_count)) + { + return *out_count == 0U; + } + *out_matches = sameStoredMessage(latest, message); + return true; +} + +bool SdStore::appendMessageRecord(const ChatMessage& message, + uint32_t sequence) +{ + const ConversationId conversation = conversationIdForMessage(message); + char directory[128]{}; + buildConversationDirectory(conversation, directory, sizeof(directory)); + if (!ensureDirectory(directory)) + { + return false; + } + const std::size_t capacity = slotsPerMessageSegment(message.protocol); + const std::size_t slot_size = storage_v2::messageSlotSize(message.protocol); + if (capacity == 0U || slot_size > scratch_.size() || sequence == 0U || + !storage_v2::encodeMessageSlot(message, + sequence, + scratch_.data(), + slot_size)) + { + return false; + } + const uint32_t ordinal = sequence - 1U; + const uint32_t segment = static_cast(ordinal / capacity); + const uint32_t expected_slot = static_cast(ordinal % capacity); + char path[128]{}; + buildMessageSegmentPath(conversation, segment, path, sizeof(path)); + auto inspection = journal_.inspect(path, + message.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + if (!repairPartialJournal(path, + message.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size)) + { + return false; + } + inspection = journal_.inspect(path, + message.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size); + } + if (inspection.state != storage_v2::FixedSlotJournalEngine::State::Missing && + inspection.state != storage_v2::FixedSlotJournalEngine::State::Ready) + { + return false; + } + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Ready && + inspection.slot_count != expected_slot) + { + return false; + } + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::Missing && + expected_slot != 0U) + { + return false; + } + // Tail recovery reuses scratch_, so restore the command record before the + // authoritative append. + if (!storage_v2::encodeMessageSlot(message, + sequence, + scratch_.data(), + slot_size)) + { + return false; + } + return journal_.append(path, + message.protocol, + storage_v2::JournalKind::MessageSegment, + slot_size, + scratch_.data()); +} + +bool SdStore::repairPartialJournal(const char* path, + MeshProtocol protocol, + storage_v2::JournalKind kind, + std::size_t slot_size) const +{ + const auto inspection = journal_.inspect(path, protocol, kind, slot_size); + if (inspection.state != + storage_v2::FixedSlotJournalEngine::State::PartialTail) + { + return inspection.state == + storage_v2::FixedSlotJournalEngine::State::Ready; + } + char temp[144]{}; + char backup[144]{}; + std::snprintf(temp, sizeof(temp), "%s.repair", path); + std::snprintf(backup, sizeof(backup), "%s.partial", path); + (void)storage_runtime::sd_remove(temp); + (void)storage_runtime::sd_remove(backup); + if (!journal_.create(temp, protocol, kind, slot_size)) + { + return false; + } + for (uint32_t index = 0; index < inspection.slot_count; ++index) + { + if (!journal_.read(path, + protocol, + kind, + slot_size, + index, + scratch_.data()) || + !journal_.append(temp, + protocol, + kind, + slot_size, + scratch_.data())) + { + (void)storage_runtime::sd_remove(temp); + return false; + } + } + if (!storage_runtime::sd_rename(path, backup) || + !storage_runtime::sd_rename(temp, path)) + { + if (!storage_runtime::sd_exists(path)) + { + (void)storage_runtime::sd_rename(backup, path); + } + (void)storage_runtime::sd_remove(temp); + return false; + } + (void)storage_runtime::sd_remove(backup); + return true; +} + +bool SdStore::appendCatalogProjection( + const storage_v2::ChatCatalogProjection& projection) +{ + const MeshProtocol protocol = + normalizeProtocol(projection.conversation.protocol); + const std::size_t slot_size = storage_v2::catalogSlotSize(protocol); + if (slot_size > scratch_.size() || + !storage_v2::encodeCatalogSlot(protocol, + projection, + scratch_.data(), + slot_size)) + { + return false; + } + char path[128]{}; + buildProjectionPath(protocol, "catalog.delta", path, sizeof(path)); + return journal_.append(path, + protocol, + storage_v2::JournalKind::CatalogDelta, + slot_size, + scratch_.data()); +} + +bool SdStore::appendReadProjection( + const storage_v2::ChatReadProjection& projection) +{ + const MeshProtocol protocol = + normalizeProtocol(projection.conversation.protocol); + const std::size_t slot_size = storage_v2::readStateSlotSize(protocol); + if (slot_size > scratch_.size() || + !storage_v2::encodeReadStateSlot(protocol, + projection, + scratch_.data(), + slot_size)) + { + return false; + } + char path[128]{}; + buildProjectionPath(protocol, "read.delta", path, sizeof(path)); + auto inspection = journal_.inspect(path, + protocol, + storage_v2::JournalKind::ReadStateDelta, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::PartialTail && + !repairPartialJournal(path, + protocol, + storage_v2::JournalKind::ReadStateDelta, + slot_size)) + { + return false; + } + if (!storage_v2::encodeReadStateSlot(protocol, + projection, + scratch_.data(), + slot_size)) + { + return false; + } + return journal_.append(path, + protocol, + storage_v2::JournalKind::ReadStateDelta, + slot_size, + scratch_.data()); +} + +bool SdStore::appendStatusProjection( + MeshProtocol protocol, + const storage_v2::ChatStatusProjection& projection) +{ + protocol = normalizeProtocol(protocol); + const std::size_t slot_size = storage_v2::statusSlotSize(); + if (slot_size > scratch_.size() || + !storage_v2::encodeStatusSlot(projection, + scratch_.data(), + slot_size)) + { + return false; + } + char path[128]{}; + buildProjectionPath(protocol, "status.delta", path, sizeof(path)); + auto inspection = journal_.inspect(path, + protocol, + storage_v2::JournalKind::StatusDelta, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::PartialTail && + !repairPartialJournal(path, + protocol, + storage_v2::JournalKind::StatusDelta, + slot_size)) + { + return false; + } + if (!storage_v2::encodeStatusSlot(projection, + scratch_.data(), + slot_size)) + { + return false; + } + return journal_.append(path, + protocol, + storage_v2::JournalKind::StatusDelta, + slot_size, + scratch_.data()); +} + +bool SdStore::appendSeenProjection( + const storage_v2::ReticulumSeenProjection& projection) const +{ + const std::size_t slot_size = storage_v2::reticulumSeenSlotSize(); + if (slot_size > scratch_.size() || + !storage_v2::encodeReticulumSeenSlot(projection, + scratch_.data(), + slot_size)) + { + return false; + } + char path[128]{}; + buildProjectionPath(MeshProtocol::Reticulum, + "seen.delta", + path, + sizeof(path)); + auto inspection = journal_.inspect( + path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size); + if (inspection.state == storage_v2::FixedSlotJournalEngine::State::PartialTail && + !repairPartialJournal(path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size)) + { + return false; + } + if (!storage_v2::encodeReticulumSeenSlot(projection, + scratch_.data(), + slot_size)) + { + return false; + } + return journal_.append(path, + MeshProtocol::Reticulum, + storage_v2::JournalKind::ReticulumSeen, + slot_size, + scratch_.data()); +} + +bool SdStore::rememberReticulumHash(const uint8_t* hash) +{ + if (hasReticulumLxmfMessageHash(hash)) + { + return true; + } + storage_v2::ReticulumSeenProjection projection{}; + std::memcpy(projection.hash, hash, sizeof(projection.hash)); + if (!appendSeenProjection(projection)) + { + return false; + } + if (seen_hot_.size() == kSeenHotCapacity) + { + seen_hot_.erase(seen_hot_.begin()); + } + seen_hot_.push_back(projection); + return true; +} + +storage_v2::ChatCatalogProjection* SdStore::findCatalog( + const ConversationId& conversation) +{ + for (storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (sameConversationKey(projection.conversation, conversation)) + { + return &projection; + } + } + return nullptr; +} + +const storage_v2::ChatCatalogProjection* SdStore::findCatalog( + const ConversationId& conversation) const +{ + for (const storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (sameConversationKey(projection.conversation, conversation)) + { + return &projection; + } + } + return nullptr; +} + +storage_v2::ChatReadProjection* SdStore::findReadState( + const ConversationId& conversation) +{ + for (storage_v2::ChatReadProjection& projection : read_state_) + { + if (sameConversationKey(projection.conversation, conversation)) + { + return &projection; + } + } + return nullptr; +} + +const storage_v2::ChatReadProjection* SdStore::findReadState( + const ConversationId& conversation) const +{ + for (const storage_v2::ChatReadProjection& projection : read_state_) + { + if (sameConversationKey(projection.conversation, conversation)) + { + return &projection; + } + } + return nullptr; +} + +storage_v2::ChatStatusProjection* SdStore::findStatus(MessageId message_id, + MeshProtocol protocol) +{ + for (ProtocolStatusProjection& state : statuses_) + { + if (sameProtocol(state.protocol, protocol) && + state.value.message_id == message_id) + { + return &state.value; + } + } + return nullptr; +} + +const storage_v2::ChatStatusProjection* SdStore::findStatus( + MessageId message_id, + MeshProtocol protocol) const +{ + for (const ProtocolStatusProjection& state : statuses_) + { + if (sameProtocol(state.protocol, protocol) && + state.value.message_id == message_id) + { + return &state.value; + } + } + return nullptr; +} + +void SdStore::applyStoredStatus(ChatMessage& message) const +{ + if (message.from != 0U) + { + return; + } + if (const storage_v2::ChatStatusProjection* status = + findStatus(message.msg_id, message.protocol)) + { + message.status = status->status; + } +} + +uint32_t SdStore::countUnreadAfter(const ConversationId& conversation, + uint32_t last_read_sequence) const +{ + const uint32_t count = messageCountOnDisk(conversation); + uint32_t unread = 0; + for (uint32_t ordinal = last_read_sequence; ordinal < count; ++ordinal) + { + ChatMessage message{}; + uint32_t sequence = 0; + if (readMessageByOrdinal(conversation, + ordinal, + message, + &sequence) && + sequence > last_read_sequence && + message.status == MessageStatus::Incoming) + { + ++unread; + } + } + return unread; +} + +uint32_t SdStore::sequenceForUnread(const ConversationId& conversation, + uint32_t unread) const +{ + const uint32_t count = messageCountOnDisk(conversation); + if (unread == 0U) + { + return count; + } + uint32_t incoming = 0; + for (uint32_t ordinal = count; ordinal > 0U; --ordinal) + { + ChatMessage message{}; + uint32_t sequence = 0; + if (!readMessageByOrdinal(conversation, + ordinal - 1U, + message, + &sequence) || + message.status != MessageStatus::Incoming) { continue; } - - const ConversationId conv = conversationIdForMessage(last_msg); - uint16_t ledger_unread = 0; - if (readStateUnreadOrLegacy(conv, &ledger_unread)) + ++incoming; + if (incoming == unread) { - unread = ledger_unread; - } - - IndexEntry entry{}; - entry.protocol = static_cast(last_msg.protocol); - entry.channel = static_cast(last_msg.channel); - entry.status = static_cast(last_msg.status); - entry.unread = unread; - entry.peer = last_msg.peer; - if (last_msg.protocol == MeshProtocol::Reticulum && - hasReticulumDestinationIdentity(last_msg.reticulum_identity)) - { - entry.flags |= kIndexHasReticulumIdentityFlag; - copyReticulumIdentityToStorage(entry.reticulum_destination_hash, - entry.reticulum_identity_hash, - last_msg.reticulum_identity); - } - entry.last_msg_id = last_msg.msg_id; - entry.last_timestamp = last_msg.timestamp; - entry.last_from = last_msg.from; - entry.preview_len = static_cast(std::min(last_msg.text.size(), kPreviewLen)); - if (entry.preview_len > 0) - { - std::memcpy(entry.preview, last_msg.text.data(), entry.preview_len); - } - entries.push_back(entry); - if (!readStateUnreadOnly(conv, &ledger_unread)) - { - (void)writeReadStateUnread(conv, unread); - } - if (!has_header_unread) - { - (void)writeConversationUnread(conv, unread); + return sequence > 0U ? sequence - 1U : 0U; } } - dir.close(); - - unread_reconcile_pending_ = !writeIndex(entries); - CHAT_STORE_LOG("[AppContext] chat store=SdStore rebuild index entries=%u\n", - static_cast(entries.size())); + return 0U; } -bool SdStore::loadFileHeader(storage::SdRuntimeFile& file, FileHeader& header) const +bool SdStore::rewriteCatalogSnapshot(MeshProtocol protocol) { - if (!file.is_open() || file.size() < sizeof(FileHeader)) - { - return false; - } - if (!file.seek(0) || !readExact(file, &header, sizeof(header))) - { - return false; - } - return header.magic == kFileMagic && - (header.version == kFileVersion || - header.version == kRxOriginVersion || - header.version == kReticulumIdentityVersion || - header.version == kLegacyVersion) && - header.head < kMaxMessagesPerConv && header.count <= kMaxMessagesPerConv; + char final_path[128]{}; + char temp_path[128]{}; + buildProjectionPath(protocol, + "catalog.snapshot", + final_path, + sizeof(final_path)); + buildProjectionPath(protocol, + "catalog.snapshot.tmp", + temp_path, + sizeof(temp_path)); + return rewriteJournalFromCatalog(protocol, final_path, temp_path); } -bool SdStore::initFileHeader(storage::SdRuntimeFile& file) const +bool SdStore::rewriteReadSnapshot(MeshProtocol protocol) { - FileHeader header{}; - header.magic = kFileMagic; - header.version = kFileVersion; - if (!file.seek(0) || !writeExact(file, &header, sizeof(header))) - { - return false; - } - return file.flush(); + char final_path[128]{}; + char temp_path[128]{}; + buildProjectionPath(protocol, + "read.snapshot", + final_path, + sizeof(final_path)); + buildProjectionPath(protocol, + "read.snapshot.tmp", + temp_path, + sizeof(temp_path)); + return rewriteJournalFromReadState(protocol, final_path, temp_path); } -bool SdStore::upgradeConversationFile(storage::SdRuntimeFile& file, - const char* path, - FileHeader& header) const +bool SdStore::rewriteStatusSnapshot(MeshProtocol protocol) { - if (header.version == kFileVersion) - { - return true; - } - if ((header.version != kLegacyVersion && - header.version != kReticulumIdentityVersion && - header.version != kRxOriginVersion) || - !path || path[0] == '\0') - { - return false; - } - - std::vector records; - records.reserve(header.count); - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = - static_cast((header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec)) - { - return false; - } - records.push_back(rec); - } - - file.close(); - const std::string temp_path = std::string(path) + ".upgrade"; - const std::string backup_path = std::string(path) + ".bak"; - if (storage::sd_exists(temp_path.c_str())) - { - (void)storage::sd_remove(temp_path.c_str()); - } - if (storage::sd_exists(backup_path.c_str()) && - !storage::sd_remove(backup_path.c_str())) - { - return false; - } - if (!file.open(temp_path.c_str(), "w+")) - { - return false; - } - - FileHeader upgraded{}; - upgraded.magic = kFileMagic; - upgraded.version = kFileVersion; - upgraded.reserved = header.reserved; - upgraded.count = static_cast(std::min(records.size(), kMaxMessagesPerConv)); - upgraded.head = static_cast(upgraded.count % kMaxMessagesPerConv); - if (!file.seek(0) || !writeExact(file, &upgraded, sizeof(upgraded))) - { - file.close(); - (void)storage::sd_remove(temp_path.c_str()); - return false; - } - for (uint16_t index = 0; index < upgraded.count; ++index) - { - if (!writeRecord(file, index, records[index])) - { - file.close(); - (void)storage::sd_remove(temp_path.c_str()); - return false; - } - } - if (!file.flush()) - { - file.close(); - (void)storage::sd_remove(temp_path.c_str()); - return false; - } - file.close(); - - if (!storage::sd_rename(path, backup_path.c_str())) - { - (void)storage::sd_remove(temp_path.c_str()); - return false; - } - if (!storage::sd_rename(temp_path.c_str(), path)) - { - (void)storage::sd_rename(backup_path.c_str(), path); - (void)storage::sd_remove(temp_path.c_str()); - return false; - } - if (!file.open(path, "r+") || !loadFileHeader(file, header) || - header.version != kFileVersion) - { - file.close(); - (void)storage::sd_remove(path); - (void)storage::sd_rename(backup_path.c_str(), path); - return false; - } - (void)storage::sd_remove(backup_path.c_str()); - return true; + char final_path[128]{}; + char temp_path[128]{}; + buildProjectionPath(protocol, + "status.snapshot", + final_path, + sizeof(final_path)); + buildProjectionPath(protocol, + "status.snapshot.tmp", + temp_path, + sizeof(temp_path)); + return rewriteJournalFromStatus(protocol, final_path, temp_path); } -bool SdStore::readRecord(storage::SdRuntimeFile& file, - const FileHeader& header, - uint16_t slot, - Record& rec) const +bool SdStore::compactProtocolProjections(MeshProtocol protocol) { - if (slot >= kMaxMessagesPerConv) + protocol = normalizeProtocol(protocol); + char catalog_delta[128]{}; + char read_delta[128]{}; + char status_delta[128]{}; + buildProjectionPath(protocol, + "catalog.delta", + catalog_delta, + sizeof(catalog_delta)); + buildProjectionPath(protocol, + "read.delta", + read_delta, + sizeof(read_delta)); + buildProjectionPath(protocol, + "status.delta", + status_delta, + sizeof(status_delta)); + const auto catalog_inspection = journal_.inspect( + catalog_delta, + protocol, + storage_v2::JournalKind::CatalogDelta, + storage_v2::catalogSlotSize(protocol)); + const auto read_inspection = journal_.inspect( + read_delta, + protocol, + storage_v2::JournalKind::ReadStateDelta, + storage_v2::readStateSlotSize(protocol)); + const auto status_inspection = journal_.inspect( + status_delta, + protocol, + storage_v2::JournalKind::StatusDelta, + storage_v2::statusSlotSize()); + const bool compact_catalog = + projection_dirty_[protocolIndex(protocol)] || + catalog_inspection.slot_count >= kCatalogCompactThreshold; + const bool compact_read = + read_inspection.slot_count >= kReadCompactThreshold; + const bool compact_status = + status_inspection.slot_count >= kStatusCompactThreshold; + if (compact_catalog && !rewriteCatalogSnapshot(protocol)) { return false; } - size_t record_size = sizeof(Record); - if (header.version == kLegacyVersion) - { - record_size = sizeof(RecordV2); - } - else if (header.version == kReticulumIdentityVersion) - { - record_size = sizeof(RecordV3); - } - else if (header.version == kRxOriginVersion) - { - record_size = sizeof(RecordV4); - } - const uint64_t offset = sizeof(FileHeader) + static_cast(slot) * record_size; - if (file.size() < offset + record_size) + if (compact_read && !rewriteReadSnapshot(protocol)) { return false; } - if (!file.seek(offset)) + if (compact_status && !rewriteStatusSnapshot(protocol)) { return false; } - if (header.version == kFileVersion) + if (compact_catalog) { - return readExact(file, &rec, sizeof(rec)); + (void)storage_runtime::sd_remove(catalog_delta); + projection_dirty_[protocolIndex(protocol)] = false; } - - if (header.version == kRxOriginVersion) + if (compact_read) { - RecordV4 rec_v4{}; - if (!readExact(file, &rec_v4, sizeof(rec_v4))) - { - return false; - } - rec.protocol = rec_v4.protocol; - rec.channel = rec_v4.channel; - rec.status = rec_v4.status; - rec.flags = rec_v4.flags; - rec.rx_origin = rec_v4.rx_origin; - rec.text_len = rec_v4.text_len; - rec.from = rec_v4.from; - rec.peer = rec_v4.peer; - rec.msg_id = rec_v4.msg_id; - rec.timestamp = rec_v4.timestamp; - std::memcpy(rec.reticulum_destination_hash, - rec_v4.reticulum_destination_hash, - sizeof(rec.reticulum_destination_hash)); - std::memcpy(rec.reticulum_identity_hash, - rec_v4.reticulum_identity_hash, - sizeof(rec.reticulum_identity_hash)); - std::memcpy(rec.text, rec_v4.text, sizeof(rec.text)); - return true; + (void)storage_runtime::sd_remove(read_delta); } - - if (header.version == kReticulumIdentityVersion) + if (compact_status) { - RecordV3 rec_v3{}; - if (!readExact(file, &rec_v3, sizeof(rec_v3))) - { - return false; - } - rec.protocol = rec_v3.protocol; - rec.channel = rec_v3.channel; - rec.status = rec_v3.status; - rec.flags = rec_v3.flags; - rec.text_len = rec_v3.text_len; - rec.from = rec_v3.from; - rec.peer = rec_v3.peer; - rec.msg_id = rec_v3.msg_id; - rec.timestamp = rec_v3.timestamp; - std::memcpy(rec.reticulum_destination_hash, - rec_v3.reticulum_destination_hash, - sizeof(rec.reticulum_destination_hash)); - std::memcpy(rec.reticulum_identity_hash, - rec_v3.reticulum_identity_hash, - sizeof(rec.reticulum_identity_hash)); - std::memcpy(rec.text, rec_v3.text, sizeof(rec.text)); - return true; - } - - RecordV2 legacy_rec{}; - if (header.version != kLegacyVersion || - !readExact(file, &legacy_rec, sizeof(legacy_rec))) - { - return false; - } - rec.protocol = legacy_rec.protocol; - rec.channel = legacy_rec.channel; - rec.status = legacy_rec.status; - rec.text_len = legacy_rec.text_len; - rec.from = legacy_rec.from; - rec.peer = legacy_rec.peer; - rec.msg_id = legacy_rec.msg_id; - rec.timestamp = legacy_rec.timestamp; - std::memcpy(rec.text, legacy_rec.text, sizeof(rec.text)); - return true; -} - -bool SdStore::writeRecord(storage::SdRuntimeFile& file, uint16_t slot, const Record& rec) const -{ - if (slot >= kMaxMessagesPerConv) - { - return false; - } - const uint64_t offset = sizeof(FileHeader) + static_cast(slot) * sizeof(Record); - return file.seek(offset) && writeExact(file, &rec, sizeof(rec)); -} - -bool SdStore::initLxmfSeenFile(storage::SdRuntimeFile& file) const -{ - LxmfSeenHeader header{}; - header.magic = kLxmfSeenMagic; - header.version = kLxmfSeenVersion; - if (!file.seek(0) || !writeExact(file, &header, sizeof(header))) - { - return false; - } - return file.flush(); -} - -bool SdStore::loadLxmfSeenHeader(storage::SdRuntimeFile& file, - LxmfSeenHeader& header) const -{ - if (!file.is_open() || file.size() < sizeof(LxmfSeenHeader)) - { - return false; - } - if (!file.seek(0) || !readExact(file, &header, sizeof(header))) - { - return false; - } - return header.magic == kLxmfSeenMagic && - header.version == kLxmfSeenVersion && - header.head < kMaxReticulumLxmfSeen && - header.count <= kMaxReticulumLxmfSeen; -} - -bool SdStore::readLxmfSeenRecord(storage::SdRuntimeFile& file, - uint16_t slot, - LxmfSeenRecord& rec) const -{ - if (slot >= kMaxReticulumLxmfSeen) - { - return false; - } - const uint64_t offset = - sizeof(LxmfSeenHeader) + static_cast(slot) * sizeof(LxmfSeenRecord); - if (file.size() < offset + sizeof(LxmfSeenRecord)) - { - return false; - } - return file.seek(offset) && readExact(file, &rec, sizeof(rec)); -} - -bool SdStore::writeLxmfSeenRecord(storage::SdRuntimeFile& file, - uint16_t slot, - const LxmfSeenRecord& rec) const -{ - if (slot >= kMaxReticulumLxmfSeen) - { - return false; - } - const uint64_t offset = - sizeof(LxmfSeenHeader) + static_cast(slot) * sizeof(LxmfSeenRecord); - return file.seek(offset) && writeExact(file, &rec, sizeof(rec)); -} - -bool SdStore::rememberReticulumLxmfMessageHash(const uint8_t* lxmf_hash) const -{ - if (!ready_ || !validLxmfMessageHash(lxmf_hash) || !ensureDir()) - { - return false; - } - if (hasReticulumLxmfMessageHash(lxmf_hash)) - { - return true; - } - - storage::SdRuntimeFile file; - if (storage::sd_exists(kReticulumLxmfSeenFile)) - { - if (!file.open(kReticulumLxmfSeenFile, "r+")) - { - return false; - } - } - else if (!file.open(kReticulumLxmfSeenFile, "w+")) - { - return false; - } - - LxmfSeenHeader header{}; - if (!loadLxmfSeenHeader(file, header)) - { - if (!initLxmfSeenFile(file) || !loadLxmfSeenHeader(file, header)) - { - file.close(); - return false; - } - } - - LxmfSeenRecord rec{}; - std::memcpy(rec.hash, lxmf_hash, sizeof(rec.hash)); - if (!writeLxmfSeenRecord(file, header.head, rec)) - { - file.close(); - return false; - } - - header.head = static_cast((header.head + 1U) % kMaxReticulumLxmfSeen); - if (header.count < kMaxReticulumLxmfSeen) - { - header.count = static_cast(header.count + 1U); - } - if (!file.seek(0) || !writeExact(file, &header, sizeof(header)) || !file.flush()) - { - file.close(); - return false; - } - file.close(); - return true; -} - -bool SdStore::conversationContainsReticulumLxmfHash( - storage::SdRuntimeFile& file, - const FileHeader& header, - const uint8_t* lxmf_hash, - bool* found) const -{ - if (!file.is_open() || !validLxmfMessageHash(lxmf_hash) || !found) - { - return false; - } - *found = false; - for (uint16_t index = 0; index < header.count; ++index) - { - const uint16_t slot = static_cast( - (header.head + kMaxMessagesPerConv - header.count + index) % - kMaxMessagesPerConv); - Record rec{}; - if (!readRecord(file, header, slot, rec)) - { - return false; - } - if ((rec.flags & kRecordHasReticulumLxmfHashFlag) != 0 && - std::memcmp(rec.reticulum_lxmf_hash, - lxmf_hash, - kReticulumLxmfHashSize) == 0) - { - *found = true; - return true; - } + (void)storage_runtime::sd_remove(status_delta); } return true; } -bool SdStore::openConversationForUpdate(const ConversationId& conv, - storage::SdRuntimeFile& file, - FileHeader& header) const +bool SdStore::rewriteJournalFromCatalog(MeshProtocol protocol, + const char* final_path, + const char* temp_path) { - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - - if (storage::sd_exists(path)) + const std::size_t slot_size = storage_v2::catalogSlotSize(protocol); + (void)storage_runtime::sd_remove(temp_path); + if (!journal_.create(temp_path, + protocol, + storage_v2::JournalKind::CatalogSnapshot, + slot_size)) { - if (!file.open(path, "r+")) + return false; + } + for (const storage_v2::ChatCatalogProjection& projection : catalog_) + { + if (!sameProtocol(projection.conversation.protocol, protocol) || + projection.deleted) { + continue; + } + if (!storage_v2::encodeCatalogSlot(protocol, + projection, + scratch_.data(), + slot_size) || + !journal_.append(temp_path, + protocol, + storage_v2::JournalKind::CatalogSnapshot, + slot_size, + scratch_.data())) + { + (void)storage_runtime::sd_remove(temp_path); return false; } - if (loadFileHeader(file, header) && upgradeConversationFile(file, path, header)) + } + return replaceSnapshot(temp_path, final_path); +} + +bool SdStore::rewriteJournalFromReadState(MeshProtocol protocol, + const char* final_path, + const char* temp_path) +{ + const std::size_t slot_size = storage_v2::readStateSlotSize(protocol); + (void)storage_runtime::sd_remove(temp_path); + if (!journal_.create(temp_path, + protocol, + storage_v2::JournalKind::ReadStateSnapshot, + slot_size)) + { + return false; + } + for (const storage_v2::ChatReadProjection& projection : read_state_) + { + if (!sameProtocol(projection.conversation.protocol, protocol) || + projection.deleted) { - return true; + continue; + } + if (!storage_v2::encodeReadStateSlot(protocol, + projection, + scratch_.data(), + slot_size) || + !journal_.append(temp_path, + protocol, + storage_v2::JournalKind::ReadStateSnapshot, + slot_size, + scratch_.data())) + { + (void)storage_runtime::sd_remove(temp_path); + return false; } - file.close(); - return false; } + return replaceSnapshot(temp_path, final_path); +} - if (!file.open(path, "w+")) +bool SdStore::rewriteJournalFromStatus(MeshProtocol protocol, + const char* final_path, + const char* temp_path) +{ + const std::size_t slot_size = storage_v2::statusSlotSize(); + (void)storage_runtime::sd_remove(temp_path); + if (!journal_.create(temp_path, + protocol, + storage_v2::JournalKind::StatusSnapshot, + slot_size)) { return false; } - if (!initFileHeader(file)) + for (const ProtocolStatusProjection& state : statuses_) { - file.close(); - return false; + if (!sameProtocol(state.protocol, protocol)) + { + continue; + } + if (!storage_v2::encodeStatusSlot(state.value, + scratch_.data(), + slot_size) || + !journal_.append(temp_path, + protocol, + storage_v2::JournalKind::StatusSnapshot, + slot_size, + scratch_.data())) + { + (void)storage_runtime::sd_remove(temp_path); + return false; + } } - return loadFileHeader(file, header); + return replaceSnapshot(temp_path, final_path); } -bool SdStore::readConversationUnread(const ConversationId& conv, uint16_t* unread) const +MeshProtocol SdStore::normalizeProtocol(MeshProtocol protocol) { - if (!unread) + return protocol == MeshProtocol::RNode ? MeshProtocol::Reticulum : protocol; +} + +const char* SdStore::protocolRoot(MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + if (protocol == MeshProtocol::MeshCore) + { + return kMeshCoreRoot; + } + if (protocol == MeshProtocol::Reticulum) + { + return kReticulumRoot; + } + return kMeshtasticRoot; +} + +const char* SdStore::protocolSlug(MeshProtocol protocol) +{ + protocol = normalizeProtocol(protocol); + if (protocol == MeshProtocol::MeshCore) + { + return "mc"; + } + if (protocol == MeshProtocol::Reticulum) + { + return "rt"; + } + return "mt"; +} + +bool SdStore::sameProtocol(MeshProtocol lhs, MeshProtocol rhs) +{ + return normalizeProtocol(lhs) == normalizeProtocol(rhs); +} + +bool SdStore::sameStoredMessage(const ChatMessage& lhs, + const ChatMessage& rhs) +{ + if (!sameProtocol(lhs.protocol, rhs.protocol) || + !sameConversationKey(conversationIdForMessage(lhs), + conversationIdForMessage(rhs))) { return false; } - - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - storage::SdRuntimeFile file; - if (!storage::sd_exists(path) || !file.open(path, "r")) + if (chat::hasReticulumLxmfMessageHash(lhs) && + chat::hasReticulumLxmfMessageHash(rhs)) { - return false; + return std::memcmp(lhs.reticulum_lxmf_hash, + rhs.reticulum_lxmf_hash, + kReticulumLxmfHashSize) == 0; } - - FileHeader header{}; - const bool ok = loadFileHeader(file, header) && - decodeUnreadState(header.reserved, unread); - file.close(); - return ok; + if (lhs.msg_id != 0U || rhs.msg_id != 0U) + { + return lhs.msg_id == rhs.msg_id && lhs.from == rhs.from; + } + return lhs.from == rhs.from && lhs.timestamp == rhs.timestamp && + lhs.text == rhs.text; } -bool SdStore::writeConversationUnread(const ConversationId& conv, uint16_t unread) const +ConversationMeta SdStore::makeMeta( + const storage_v2::ChatCatalogProjection& projection) { - char path[96]{}; - buildConversationPath(conv, path, sizeof(path)); - storage::SdRuntimeFile file; - if (!storage::sd_exists(path) || !file.open(path, "r+")) - { - return false; - } - - FileHeader header{}; - if (!loadFileHeader(file, header) || - !upgradeConversationFile(file, path, header)) - { - file.close(); - return false; - } - header.reserved = encodeUnreadState(unread); - const bool ok = file.seek(0) && - writeExact(file, &header, sizeof(header)) && - file.flush(); - file.close(); - return ok; -} - -void SdStore::buildConversationPath(const ConversationId& conv, char* out, size_t out_len) const -{ - if (!out || out_len == 0) - { - return; - } - if (hasReticulumConversationKey(conv)) - { - uint8_t destination_hash[kReticulumPeerHashSize] = {}; - char destination_key[kReticulumPeerHashSize * 2U + 1U]{}; - (void)copyReticulumDestinationHash(destination_hash, - conv.reticulum_identity); - hashToHex(destination_hash, destination_key, sizeof(destination_key)); - std::snprintf(out, - out_len, - "%s/%s_r_%s.log", - kDir, - protocolTag(conv.protocol), - destination_key); - return; - } - if (conv.peer == 0) - { - std::snprintf(out, - out_len, - "%s/%s_broadcast_%s.log", - kDir, - protocolTag(conv.protocol), - channelName(conv.channel)); - return; - } - - std::snprintf(out, - out_len, - "%s/%s_n_%08lX.log", - kDir, - protocolTag(conv.protocol), - static_cast(conv.peer)); -} - -const char* SdStore::channelName(ChannelId channel) const -{ - switch (channel) - { - case ChannelId::PRIMARY: - return "LongFast"; - case ChannelId::SECONDARY: - return "Squad"; - default: - return "Unknown"; - } -} - -ChatMessage SdStore::messageFromRecord(const Record& rec) -{ - ChatMessage msg; - msg.protocol = static_cast(rec.protocol); - msg.channel = static_cast(rec.channel); - msg.from = rec.from; - msg.peer = rec.peer; - msg.msg_id = rec.msg_id; - msg.timestamp = rec.timestamp; - msg.text.assign(rec.text, std::min(rec.text_len, sizeof(rec.text))); - msg.status = static_cast(rec.status); - msg.rx_origin = static_cast(rec.rx_origin); - msg.source_unverified = (rec.flags & kRecordSourceUnverifiedFlag) != 0; - if ((rec.flags & kRecordHasReticulumIdentityFlag) != 0) - { - readReticulumIdentityFromStorage(msg.reticulum_identity, - rec.reticulum_destination_hash, - rec.reticulum_identity_hash); - } - if ((rec.flags & kRecordHasReticulumLxmfHashFlag) != 0) - { - std::memcpy(msg.reticulum_lxmf_hash, - rec.reticulum_lxmf_hash, - sizeof(msg.reticulum_lxmf_hash)); - } - return msg; -} - -SdStore::Record SdStore::recordFromMessage(const ChatMessage& msg) -{ - Record rec{}; - rec.protocol = static_cast(msg.protocol); - rec.channel = static_cast(msg.channel); - rec.status = static_cast(msg.status); - rec.text_len = static_cast(std::min(msg.text.size(), sizeof(rec.text))); - rec.from = msg.from; - rec.peer = msg.peer; - rec.msg_id = msg.msg_id; - rec.timestamp = msg.timestamp; - rec.rx_origin = static_cast(msg.rx_origin); - if (msg.source_unverified) - { - rec.flags |= kRecordSourceUnverifiedFlag; - } - if (msg.protocol == MeshProtocol::Reticulum && - hasReticulumDestinationIdentity(msg.reticulum_identity)) - { - rec.flags |= kRecordHasReticulumIdentityFlag; - copyReticulumIdentityToStorage(rec.reticulum_destination_hash, - rec.reticulum_identity_hash, - msg.reticulum_identity); - } - if (chat::hasReticulumLxmfMessageHash(msg)) - { - rec.flags |= kRecordHasReticulumLxmfHashFlag; - std::memcpy(rec.reticulum_lxmf_hash, - msg.reticulum_lxmf_hash, - sizeof(rec.reticulum_lxmf_hash)); - } - if (rec.text_len > 0) - { - std::memcpy(rec.text, msg.text.data(), rec.text_len); - } - return rec; -} - -bool SdStore::indexEntryHasReticulumIdentity(const IndexEntry& entry) -{ - return static_cast(entry.protocol) == MeshProtocol::Reticulum && - (entry.flags & kIndexHasReticulumIdentityFlag) != 0; -} - -bool SdStore::indexEntryMatchesConversation(const IndexEntry& entry, - const ConversationId& conv) -{ - if (entry.protocol != static_cast(conv.protocol) || - entry.channel != static_cast(conv.channel)) - { - return false; - } - - const bool conv_has_reticulum_key = hasReticulumConversationKey(conv); - const bool entry_has_reticulum_key = indexEntryHasReticulumIdentity(entry); - if (conv_has_reticulum_key || entry_has_reticulum_key) - { - return conv_has_reticulum_key && entry_has_reticulum_key && - sameReticulumDestination(conv.reticulum_identity, - entry.reticulum_destination_hash); - } - return entry.peer == conv.peer; -} - -bool SdStore::readStateEntryHasReticulumIdentity(const ReadStateEntry& entry) -{ - return static_cast(entry.protocol) == MeshProtocol::Reticulum && - (entry.flags & kReadStateHasReticulumIdentityFlag) != 0; -} - -bool SdStore::readStateEntryMatchesConversation(const ReadStateEntry& entry, - const ConversationId& conv) -{ - if (entry.protocol != static_cast(conv.protocol) || - entry.channel != static_cast(conv.channel)) - { - return false; - } - - const bool conv_has_reticulum_key = hasReticulumConversationKey(conv); - const bool entry_has_reticulum_key = readStateEntryHasReticulumIdentity(entry); - if (conv_has_reticulum_key || entry_has_reticulum_key) - { - return conv_has_reticulum_key && entry_has_reticulum_key && - sameReticulumDestination(conv.reticulum_identity, - entry.reticulum_destination_hash); - } - return entry.peer == conv.peer; -} - -SdStore::ReadStateEntry SdStore::readStateEntryFromConversation( - const ConversationId& conv, - uint16_t unread) -{ - ReadStateEntry entry{}; - entry.protocol = static_cast(conv.protocol); - entry.channel = static_cast(conv.channel); - entry.unread = static_cast( - std::min(unread, kUnreadStateCountMask)); - entry.peer = conv.peer; - if (hasReticulumConversationKey(conv)) - { - entry.flags |= kReadStateHasReticulumIdentityFlag; - copyReticulumIdentityToStorage(entry.reticulum_destination_hash, - entry.reticulum_identity_hash, - conv.reticulum_identity); - } - return entry; -} - -ConversationId SdStore::conversationFromIndexEntry(const IndexEntry& entry) -{ - ConversationId conv(static_cast(entry.channel), - entry.peer, - static_cast(entry.protocol)); - if (indexEntryHasReticulumIdentity(entry)) - { - readReticulumIdentityFromStorage(conv.reticulum_identity, - entry.reticulum_destination_hash, - entry.reticulum_identity_hash); - } - return conv; -} - -ConversationMeta SdStore::metaFromIndexEntry(const IndexEntry& entry) -{ - ConversationMeta meta; - meta.id = conversationFromIndexEntry(entry); - meta.preview.assign(entry.preview, std::min(entry.preview_len, sizeof(entry.preview))); - meta.last_timestamp = entry.last_timestamp; - meta.unread = entry.unread; - meta.reticulum_identity = meta.id.reticulum_identity; - if (entry.peer == 0 && !indexEntryHasReticulumIdentity(entry)) + ConversationMeta meta{}; + meta.id = projection.conversation; + meta.preview = projection.preview; + meta.last_timestamp = projection.last_timestamp; + meta.unread = static_cast(projection.unread); + meta.reticulum_identity = projection.conversation.reticulum_identity; + if (projection.conversation.peer == 0U && + projection.conversation.protocol != MeshProtocol::Reticulum) { meta.name = "Broadcast"; } + else if (projection.conversation.protocol == MeshProtocol::Reticulum && + hasReticulumDestinationIdentity( + projection.conversation.reticulum_identity)) + { + char name[12]{}; + std::snprintf(name, + sizeof(name), + "%02X%02X%02X%02X", + projection.conversation.reticulum_identity + .destination_hash[0], + projection.conversation.reticulum_identity + .destination_hash[1], + projection.conversation.reticulum_identity + .destination_hash[2], + projection.conversation.reticulum_identity + .destination_hash[3]); + meta.name = name; + } else { - char buf[16]; - std::snprintf(buf, - sizeof(buf), + char name[16]{}; + std::snprintf(name, + sizeof(name), "%04lX", - static_cast(entry.peer & 0xFFFFUL)); - meta.name = buf; + static_cast(projection.conversation.peer & + 0xFFFFU)); + meta.name = name; } return meta; } -bool SdStore::hasLogSuffix(const char* name) +void SdStore::buildConversationDirectory(const ConversationId& input, + char* out, + std::size_t out_len) { - if (!name) + if (!out || out_len == 0U) + { + return; + } + ConversationId conversation = input; + conversation.protocol = normalizeProtocol(conversation.protocol); + if (conversation.protocol == MeshProtocol::Reticulum && + hasReticulumDestinationIdentity(conversation.reticulum_identity)) + { + char hash[2U * kReticulumPeerHashSize + 1U]{}; + hashToHex(conversation.reticulum_identity.destination_hash, + hash, + sizeof(hash)); + std::snprintf(out, + out_len, + "%s/conversations/d_%s", + protocolRoot(conversation.protocol), + hash); + return; + } + std::snprintf(out, + out_len, + "%s/conversations/c%02X_p%08lX", + protocolRoot(conversation.protocol), + static_cast(conversation.channel), + static_cast(conversation.peer)); +} + +void SdStore::buildMessageSegmentPath(const ConversationId& conversation, + uint32_t segment, + char* out, + std::size_t out_len) +{ + char directory[112]{}; + buildConversationDirectory(conversation, directory, sizeof(directory)); + std::snprintf(out, + out_len, + "%s/%04lu.msg", + directory, + static_cast(segment)); +} + +void SdStore::buildProjectionPath(MeshProtocol protocol, + const char* name, + char* out, + std::size_t out_len) +{ + if (!out || out_len == 0U) + { + return; + } + std::snprintf(out, out_len, "%s/%s", protocolRoot(protocol), name); +} + +bool SdStore::ensureDirectory(const char* path) +{ + return path && path[0] != '\0' && + (storage_runtime::sd_is_directory(path) || + storage_runtime::sd_mkdir(path)); +} + +bool SdStore::removeTree(const char* path) +{ + if (!path || std::strncmp(path, kRoot, std::strlen(kRoot)) != 0) { return false; } - const char* suffix = std::strstr(name, ".log"); - return suffix != nullptr && suffix[4] == '\0'; + if (!storage_runtime::sd_exists(path)) + { + return true; + } + if (!storage_runtime::sd_is_directory(path)) + { + return storage_runtime::sd_remove(path); + } + storage_runtime::SdRuntimeDir directory; + if (!directory.open(path)) + { + return false; + } + char name[96]{}; + bool is_directory = false; + bool ok = true; + while (directory.read_next(name, sizeof(name), &is_directory)) + { + char child[160]{}; + std::snprintf(child, sizeof(child), "%s/%s", path, name); + ok = (is_directory ? removeTree(child) + : storage_runtime::sd_remove(child)) && + ok; + } + directory.close(); + return storage_runtime::sd_rmdir(path) && ok; } } // namespace chat diff --git a/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp b/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp index 97b0025d..ba670868 100644 --- a/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp +++ b/platform/esp/arduino_common/src/platform_ui_reticulum_directory_runtime.cpp @@ -40,7 +40,7 @@ using ::platform::esp::arduino_common::storage::SdRuntimeFile; constexpr const char* kConfigDir = "/trailmate/reticulum"; constexpr const char* kAnnouncesPath = "/trailmate/reticulum/announces.tsv"; constexpr const char* kAnnouncesTempPath = "/trailmate/reticulum/announces.tmp"; -constexpr const char* kLxmfAddressesPath = "/mesh/peers.bin"; +constexpr const char* kLxmfAddressesPath = "/data/v2/rt/peers.*"; constexpr std::size_t kMaxDirectoryEntries = 1024; constexpr std::size_t kMaxLineBytes = 4096; constexpr std::size_t kMaxTsvFields = 14; diff --git a/scripts/check_esp_stack_hygiene.py b/scripts/check_esp_stack_hygiene.py index bf220eba..c02b7e1a 100644 --- a/scripts/check_esp_stack_hygiene.py +++ b/scripts/check_esp_stack_hygiene.py @@ -25,6 +25,10 @@ HOT_PATH_PREFIXES = ( HOT_PATH_FILES = { "platform/esp/arduino_common/src/ble/meshtastic_ble.cpp", + "platform/esp/arduino_common/src/chat/infra/store/protocol_chat_codec.cpp", + "platform/esp/arduino_common/src/chat/infra/store/protocol_peer_codec.cpp", + "platform/esp/arduino_common/src/chat/infra/store/sd_protocol_peer_repository.cpp", + "platform/esp/arduino_common/src/chat/infra/store/sd_store.cpp", } HOT_HEADER_PREFIXES = ( @@ -36,6 +40,10 @@ HOT_HEADER_PREFIXES = ( HOT_HEADER_FILES = { "platform/esp/arduino_common/include/ble/meshtastic_ble.h", + "platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_chat_codec.h", + "platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/protocol_peer_codec.h", + "platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_protocol_peer_repository.h", + "platform/esp/arduino_common/include/platform/esp/arduino_common/chat/infra/store/sd_store.h", } APP_CONFIG_PATHS = { @@ -100,7 +108,7 @@ def is_under(relative: str, prefixes: tuple[str, ...]) -> bool: def git_tracked_files() -> list[Path]: try: result = subprocess.run( - ["git", "ls-files"], + ["git", "ls-files", "--cached", "--others", "--exclude-standard"], cwd=REPO_ROOT, check=True, text=True, @@ -109,7 +117,8 @@ def git_tracked_files() -> list[Path]: ) except (OSError, subprocess.CalledProcessError): return [] - return [REPO_ROOT / line for line in result.stdout.splitlines() if line] + paths = [REPO_ROOT / line for line in result.stdout.splitlines() if line] + return [path for path in paths if path.is_file()] def walk_source_files() -> list[Path]: