From 71849cbb62027a16a1365734d6ccf9c120ab005c Mon Sep 17 00:00:00 2001 From: DeFiDude <59237470+DeFiDude@users.noreply.github.com> Date: Thu, 19 Mar 2026 12:37:09 -0600 Subject: [PATCH] Fix LXMF wire format for Python interoperability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opportunistic (non-link) delivery must strip dest_hash from the LXMF payload — it's carried by the RNS packet header. Python prepends it back on receive; we now do the same. Previously ratdeck always included dest_hash when sending and never prepended it when receiving, breaking interop in both directions. - packFull: return [src:16][sig:64][msgpack] instead of [dest:16][src:16][sig:64][msgpack] - onPacketReceived: prepend dest_hash from packet header before unpacking - sendDirect: use messageId computed by packFull instead of re-hashing wire payload - Add docs/WIRE-FORMAT.md documenting the exact format for all delivery methods --- docs/WIRE-FORMAT.md | 91 +++++++++++++++++++++++++++++++++++ src/reticulum/LXMFManager.cpp | 11 ++++- src/reticulum/LXMFMessage.cpp | 17 ++++--- src/reticulum/LXMFMessage.h | 2 +- 4 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 docs/WIRE-FORMAT.md diff --git a/docs/WIRE-FORMAT.md b/docs/WIRE-FORMAT.md new file mode 100644 index 0000000..812716d --- /dev/null +++ b/docs/WIRE-FORMAT.md @@ -0,0 +1,91 @@ +# LXMF Wire Format Reference + +This documents the exact LXMF wire format as implemented by Python Sideband/NomadNet and ratdeck. All fields are big-endian. + +## Canonical (Internal) Format + +Used for signing, messageId computation, and storage: + +``` +[dest_hash:16][src_hash:16][signature:64][msgpack_payload] +``` + +- `dest_hash` — 16-byte truncated hash of the destination identity +- `src_hash` — 16-byte truncated hash of the source identity +- `signature` — 64-byte Ed25519 signature +- `msgpack_payload` — MsgPack array: `[timestamp, title, content, fields]` + +## Wire Formats by Delivery Method + +### Opportunistic (single-packet, non-link) + +The destination hash is **not** included in the LXMF payload — it's carried by the RNS packet header. + +``` +RNS Packet payload = [src_hash:16][signature:64][msgpack_payload] +``` + +Python reference (`LXMessage.py:628-631`): +```python +if self.method == LXMessage.OPPORTUNISTIC: + return RNS.Packet(self.__delivery_destination, self.packed[DESTINATION_LENGTH:]) +``` + +### Direct (link-based) + +The destination hash IS included in the payload: + +``` +Link packet payload = [dest_hash:16][src_hash:16][signature:64][msgpack_payload] +``` + +Python reference: +```python +elif self.method == LXMessage.DIRECT: + return RNS.Packet(self.__delivery_destination, self.packed) +``` + +## Receiving + +On receive, the router reconstructs the canonical format before unpacking: + +- **Non-link packets**: Prepend `packet.destination.hash` to the data +- **Link packets**: Data already contains dest_hash + +Python reference (`LXMRouter.py:1821-1828`): +```python +if packet.destination_type != RNS.Destination.LINK: + lxmf_data = packet.destination.hash + data # prepend dest_hash +else: + lxmf_data = data # already has dest_hash +``` + +## Signature Computation + +The signature covers the canonical data **plus** a message hash: + +``` +hashed_part = dest_hash || src_hash || msgpack_payload +msg_hash = SHA256(hashed_part) +signable = hashed_part || msg_hash +signature = Ed25519_sign(signable) +``` + +## Message ID + +``` +messageId = SHA256(dest_hash || src_hash || msgpack_payload) +``` + +This is the same as `msg_hash` above — computed from the hashed_part before appending the hash for signing. It must be identical on sender and receiver for deduplication. + +## MsgPack Payload + +Fixed 4-element array (`0x94`): + +| Index | Field | MsgPack Type | Notes | +|-------|-----------|-------------|-------| +| 0 | timestamp | float64 (0xCB) | Unix epoch seconds | +| 1 | title | bin8/bin16 (0xC4/0xC5) | Python expects bytes, not str | +| 2 | content | bin8/bin16 (0xC4/0xC5) | Python expects bytes, not str | +| 3 | fields | fixmap (0x80) | Empty map for basic messages | diff --git a/src/reticulum/LXMFManager.cpp b/src/reticulum/LXMFManager.cpp index ae7c766..6aaccfa 100644 --- a/src/reticulum/LXMFManager.cpp +++ b/src/reticulum/LXMFManager.cpp @@ -148,7 +148,7 @@ bool LXMFManager::sendDirect(LXMFMessage& msg) { RNS::PacketReceipt receipt = packet.send(); if (receipt) { msg.status = LXMFStatus::SENT; - msg.messageId = RNS::Identity::full_hash(payloadBytes); + // messageId already computed by packFull() matching Python's LXMessage.pack() Serial.printf("[LXMF] SENT OK: %d bytes, msgId=%s\n", (int)payloadBytes.size(), msg.messageId.toHex().substr(0, 8).c_str()); } else { Serial.println("[LXMF] send FAILED: no receipt"); @@ -159,7 +159,14 @@ bool LXMFManager::sendDirect(LXMFMessage& msg) { void LXMFManager::onPacketReceived(const RNS::Bytes& data, const RNS::Packet& packet) { if (!_instance) return; - _instance->processIncoming(data.data(), data.size(), packet.destination_hash()); + // Non-link delivery: dest_hash is NOT in LXMF payload (it's in the RNS packet header). + // Reconstruct full format by prepending it, matching Python LXMRouter.delivery_packet(). + const RNS::Bytes& destHash = packet.destination_hash(); + std::vector fullData; + fullData.reserve(destHash.size() + data.size()); + fullData.insert(fullData.end(), destHash.data(), destHash.data() + destHash.size()); + fullData.insert(fullData.end(), data.data(), data.data() + data.size()); + _instance->processIncoming(fullData.data(), fullData.size(), destHash); } void LXMFManager::onLinkEstablished(RNS::Link& link) { diff --git a/src/reticulum/LXMFMessage.cpp b/src/reticulum/LXMFMessage.cpp index 7863ed3..4460e37 100644 --- a/src/reticulum/LXMFMessage.cpp +++ b/src/reticulum/LXMFMessage.cpp @@ -98,7 +98,7 @@ std::vector LXMFMessage::packContent(double timestamp, const std::strin return buf; } -std::vector LXMFMessage::packFull(const RNS::Identity& signingIdentity) const { +std::vector LXMFMessage::packFull(const RNS::Identity& signingIdentity) { std::vector packed = packContent(timestamp, content, title); if (sourceHash.size() < 16 || destHash.size() < 16) return {}; @@ -110,21 +110,24 @@ std::vector LXMFMessage::packFull(const RNS::Identity& signingIdentity) hashed_part.insert(hashed_part.end(), packed.begin(), packed.end()); RNS::Bytes hashedBytes(hashed_part.data(), hashed_part.size()); - RNS::Bytes messageHash = RNS::Identity::full_hash(hashedBytes); + RNS::Bytes msgHash = RNS::Identity::full_hash(hashedBytes); + + // Store messageId = SHA256(dest + src + payload), matching Python LXMessage.pack() + messageId = msgHash; std::vector signed_part; - signed_part.reserve(hashed_part.size() + messageHash.size()); + signed_part.reserve(hashed_part.size() + msgHash.size()); signed_part.insert(signed_part.end(), hashed_part.begin(), hashed_part.end()); - signed_part.insert(signed_part.end(), messageHash.data(), messageHash.data() + messageHash.size()); + signed_part.insert(signed_part.end(), msgHash.data(), msgHash.data() + msgHash.size()); RNS::Bytes signableBytes(signed_part.data(), signed_part.size()); RNS::Bytes sig = signingIdentity.sign(signableBytes); if (sig.size() < 64) return {}; - // Wire: [dest_hash:16][src_hash:16][signature:64][packed_content] + // Wire (opportunistic): [src_hash:16][signature:64][packed_content] + // dest_hash is carried by the RNS packet header, not the LXMF payload std::vector payload; - payload.reserve(16 + 16 + 64 + packed.size()); - payload.insert(payload.end(), destHash.data(), destHash.data() + 16); + payload.reserve(16 + 64 + packed.size()); payload.insert(payload.end(), sourceHash.data(), sourceHash.data() + 16); payload.insert(payload.end(), sig.data(), sig.data() + 64); payload.insert(payload.end(), packed.begin(), packed.end()); diff --git a/src/reticulum/LXMFMessage.h b/src/reticulum/LXMFMessage.h index c808e10..596738d 100644 --- a/src/reticulum/LXMFMessage.h +++ b/src/reticulum/LXMFMessage.h @@ -27,7 +27,7 @@ struct LXMFMessage { RNS::Bytes messageId; static std::vector packContent(double timestamp, const std::string& content, const std::string& title); - std::vector packFull(const RNS::Identity& signingIdentity) const; + std::vector packFull(const RNS::Identity& signingIdentity); static bool unpackFull(const uint8_t* data, size_t len, LXMFMessage& msg); const char* statusStr() const; };