From 575d95c4d141e4538ce5d4bf112de7fe6f3c384e Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Fri, 27 Mar 2026 21:48:31 +0000 Subject: [PATCH] fetch link data via websocket --- .../2026-03-20-smp-agent-web-spike.md | 30 +++- smp-web/src/agent/protocol.ts | 157 +++++++++++++++++- tests/SMPWebTests.hs | 113 ++++++++++++- 3 files changed, 279 insertions(+), 21 deletions(-) diff --git a/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md b/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md index 136d8fe52..cf78ed307 100644 --- a/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md +++ b/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md @@ -248,19 +248,35 @@ Each step produces working, tested code. Steps 1-11 work without block encryptio **Tests**: Haskell `encodeSignLinkData` + `sbEncrypt` with known key/nonce → TypeScript decrypts → plaintext matches. -### Step 9: FixedLinkData / ConnLinkData Parse +### Step 9: ConnLinkData Parse **File**: `agent/protocol.ts` -**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnLinkData`, `UserContactData` Encoding instances +**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `ConnLinkData`, `UserContactData`, `OwnerAuth`, `ConnShortLink`, `ProtocolServer` Encoding instances -**Implementation**: -- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), `decodeLarge` for linkConnReq, optional `decodeBytes` for linkEntityId (if bytes remaining) +**Implementation** (proper decoding, not skipping): - `decodeConnLinkData(d)`: `anyByte` for connectionMode ('C'=Contact), `decodeWord16` × 2 for agentVRange, then `decodeUserContactData` -- `decodeUserContactData(d)`: `decodeBool` for direct, `decodeList(decodeOwnerAuth, d)` for owners, `decodeList(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData -- `decodeUserLinkData(d)`: peek first byte — if 0xFF, skip it and `decodeLarge(d)`; otherwise `decodeBytes(d)` +- `decodeUserContactData(d)`: `decodeBool` for direct, `smpListP(decodeOwnerAuth, d)` for owners, `smpListP(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData +- `decodeOwnerAuth(d)`: `decodeBytes` for outer wrapper, then parse inner: `(ownerId, ownerKey, authOwnerSig)` all as ByteStrings +- `decodeConnShortLink(d)`: `anyByte` for mode, then Contact: `(ctTypeChar, srv, linkKey)` or Invitation: `(srv, linkId, linkKey)` +- `decodeProtocolServer(d)`: `decodeBytes` for scheme+keyHash, `decodeBytes` for host, `decodeBytes` for port — need to verify exact encoding +- `decodeUserLinkData(d)`: first byte 0xFF → `decodeLarge`; otherwise it's the 1-byte length of a ByteString - `parseProfile(userData)`: check first byte for 'X' (0x58, zstd compressed) — if so, decompress; otherwise `JSON.parse` directly -**Tests**: Haskell encodes full `FixedLinkData` and `ContactLinkData` with known values → TypeScript decodes → all fields match. +**Tests**: Haskell encodes `ContactLinkData` with known values → TypeScript decodes → all fields match. + +**FixedLinkData**: deferred to step 15. `linkConnReq` (ConnectionRequestUri) is NOT length-prefixed in the tuple encoding — it requires full parsing. FixedLinkData is also needed to validate mutable data signature using rootKey. + +### Step 15: FixedLinkData Parse + +**File**: `agent/protocol.ts` +**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnectionRequestUri`, `ConnReqUriData` Encoding instances + +**Implementation**: +- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), then parse `ConnectionRequestUri` (mode byte + `ConnReqUriData`), optional `decodeBytes` for linkEntityId +- `decodeConnectionRequestUri(d)`: full parsing of `ConnReqUriData` including SMP queue URIs +- Needed for: connecting to the contact, and validating mutable data signature with rootKey + +**Tests**: Haskell encodes full `FixedLinkData` → TypeScript decodes → rootKey and linkConnReq fields match. ### Step 10: WebSocket Transport diff --git a/smp-web/src/agent/protocol.ts b/smp-web/src/agent/protocol.ts index a875a6090..8d0a9479f 100644 --- a/smp-web/src/agent/protocol.ts +++ b/smp-web/src/agent/protocol.ts @@ -2,6 +2,9 @@ // Mirrors: Simplex.Messaging.Agent.Protocol import {base64urlDecode} from "@simplex-chat/xftp-web/dist/protocol/description.js" +import { + Decoder, decodeBytes, decodeLarge, decodeWord16, decodeBool, +} from "@simplex-chat/xftp-web/dist/protocol/encoding.js" // -- Short link types (Agent/Protocol.hs:1462-1470) @@ -9,18 +12,158 @@ export type ShortLinkScheme = "simplex" | "https" export type ContactConnType = "contact" | "channel" | "group" | "relay" +export interface ProtocolServer { + hosts: Uint8Array[] // NonEmpty, each is the strEncoded host bytes + port: Uint8Array + keyHash: Uint8Array +} + +export type ConnShortLink = + | {mode: "invitation", scheme: ShortLinkScheme, server: ProtocolServer, linkId: Uint8Array, linkKey: Uint8Array} + | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ProtocolServer, linkKey: Uint8Array} + +// -- ProtocolServer binary encoding (Protocol.hs:1264-1269) +// smpEncode (host, port, keyHash) +// host: NonEmpty TransportHost = smpEncodeList (1-byte count + each as ByteString) +// port: String = ByteString (1-byte len + bytes) +// keyHash: KeyHash = ByteString (1-byte len + bytes) + +export function decodeProtocolServer(d: Decoder): ProtocolServer { + const hostCount = d.anyByte() + if (hostCount === 0) throw new Error("empty server host list") + const hosts: Uint8Array[] = [] + for (let i = 0; i < hostCount; i++) hosts.push(decodeBytes(d)) + const port = decodeBytes(d) + const keyHash = decodeBytes(d) + return {hosts, port, keyHash} +} + +// -- ConnShortLink binary encoding (Agent/Protocol.hs:1631-1649) +// Contact: smpEncode (CMContact, ctTypeChar, srv, linkKey) +// Invitation: smpEncode (CMInvitation, srv, linkId, linkKey) + +export interface ConnShortLinkBinary { + mode: "contact" | "invitation" + connType?: ContactConnType + server: ProtocolServer + linkId?: Uint8Array + linkKey: Uint8Array +} + +const ctTypeFromByte: Record = { + 0x41: "contact", // 'A' + 0x43: "channel", // 'C' + 0x47: "group", // 'G' + 0x52: "relay", // 'R' +} + +export function decodeConnShortLink(d: Decoder): ConnShortLinkBinary { + const mode = d.anyByte() + if (mode === 0x49) { + // Invitation: (srv, linkId, linkKey) + const server = decodeProtocolServer(d) + const linkId = decodeBytes(d) + const linkKey = decodeBytes(d) + return {mode: "invitation", server, linkId, linkKey} + } else if (mode === 0x43) { + // Contact: (ctTypeChar, srv, linkKey) + const ctByte = d.anyByte() + const connType = ctTypeFromByte[ctByte] + if (!connType) throw new Error("unknown contact type: 0x" + ctByte.toString(16)) + const server = decodeProtocolServer(d) + const linkKey = decodeBytes(d) + return {mode: "contact", connType, server, linkKey} + } + throw new Error("unknown ConnShortLink mode: 0x" + mode.toString(16)) +} + +// -- OwnerAuth (Agent/Protocol.hs:1793-1800) +// Outer ByteString wrapping inner: (ownerId, ownerKey, authOwnerSig) + +export interface OwnerAuth { + ownerId: Uint8Array + ownerKey: Uint8Array + authOwnerSig: Uint8Array +} + +export function decodeOwnerAuth(d: Decoder): OwnerAuth { + const inner = decodeBytes(d) + const id = new Decoder(inner) + const ownerId = decodeBytes(id) + const ownerKey = decodeBytes(id) + const authOwnerSig = decodeBytes(id) + return {ownerId, ownerKey, authOwnerSig} +} + +// -- UserLinkData (Agent/Protocol.hs:1891-1894) +// If first byte is 0xFF, read Large; otherwise it's a ByteString (1-byte length) + +export function decodeUserLinkData(d: Decoder): Uint8Array { + const firstByte = d.anyByte() + if (firstByte === 0xFF) return decodeLarge(d) + return d.take(firstByte) +} + +// -- UserContactData (Agent/Protocol.hs:1881-1889) + +export interface UserContactData { + direct: boolean + owners: OwnerAuth[] + relays: ConnShortLinkBinary[] + userData: Uint8Array +} + +export function decodeUserContactData(d: Decoder): UserContactData { + const direct = decodeBool(d) + const ownerCount = d.anyByte() + const owners: OwnerAuth[] = [] + for (let i = 0; i < ownerCount; i++) owners.push(decodeOwnerAuth(d)) + const relayCount = d.anyByte() + const relays: ConnShortLinkBinary[] = [] + for (let i = 0; i < relayCount; i++) relays.push(decodeConnShortLink(d)) + const userData = decodeUserLinkData(d) + return {direct, owners, relays, userData} +} + +// -- ConnLinkData (Agent/Protocol.hs:1838-1855) +// Contact: 'C' + versionRange + UserContactData + +export interface ConnLinkDataContact { + mode: "contact" + agentVRange: {min: number; max: number} + userContactData: UserContactData +} + +export function decodeConnLinkData(d: Decoder): ConnLinkDataContact { + const modeChar = d.anyByte() + if (modeChar !== 0x43) throw new Error("expected Contact mode 'C' (0x43), got 0x" + modeChar.toString(16)) + const min = decodeWord16(d) + const max = decodeWord16(d) + const userContactData = decodeUserContactData(d) + return {mode: "contact", agentVRange: {min, max}, userContactData} +} + +// -- Profile extraction + +export function parseProfile(userData: Uint8Array): unknown { + if (userData.length > 0 && userData[0] === 0x58) { + throw new Error("zstd-compressed profile not yet supported") + } + return JSON.parse(new TextDecoder().decode(userData)) +} + +// -- Short link URI parsing (below) -- + export interface ShortLinkServer { hosts: string[] port: string keyHash: Uint8Array } -export type ConnShortLink = +export type ConnShortLinkURI = | {mode: "invitation", scheme: ShortLinkScheme, server: ShortLinkServer, linkId: Uint8Array, linkKey: Uint8Array} | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ShortLinkServer, linkKey: Uint8Array} -// -- Contact type char mapping (Agent/Protocol.hs:1651-1666) - const ctTypeFromChar: Record = { a: "contact", c: "channel", @@ -28,10 +171,8 @@ const ctTypeFromChar: Record = { r: "relay", } -// -- Short link parser (Agent/Protocol.hs:1596-1629) -// Mirrors strP for AConnShortLink - -export function connShortLinkStrP(uri: string): ConnShortLink { +// Mirrors strP for AConnShortLink (Agent/Protocol.hs:1596-1629) +export function connShortLinkStrP(uri: string): ConnShortLinkURI { let scheme: ShortLinkScheme let firstHost: string | null = null let rest: string @@ -50,7 +191,6 @@ export function connShortLinkStrP(uri: string): ConnShortLink { throw new Error("bad short link scheme") } - // rest: /typeChar#fragment?query if (rest[0] !== "/") throw new Error("bad short link: expected /") const typeChar = rest[1] const hashIdx = rest.indexOf("#") @@ -62,7 +202,6 @@ export function connShortLinkStrP(uri: string): ConnShortLink { const queryStr = qIdx >= 0 ? afterHash.slice(qIdx + 1) : "" const params = new URLSearchParams(queryStr) - // Build server const hParam = params.get("h") const additionalHosts = hParam ? hParam.split(",") : [] const allHosts = firstHost ? [firstHost, ...additionalHosts] : additionalHosts diff --git a/tests/SMPWebTests.hs b/tests/SMPWebTests.hs index b00828f1f..3763eb2d3 100644 --- a/tests/SMPWebTests.hs +++ b/tests/SMPWebTests.hs @@ -12,22 +12,31 @@ -- Run: cabal test --test-option=--match="/SMP Web Client/" module SMPWebTests (smpWebTests) where +import Control.Monad.Except (ExceptT, runExceptT) import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as BC import Data.List.NonEmpty (NonEmpty (..)) import Data.Word (Word16) +import qualified Simplex.Messaging.Agent as A import qualified Simplex.Messaging.Agent.Protocol as AP +import Simplex.Messaging.Agent.Protocol (CreatedConnLink (..), UserLinkData (..), UserContactData (..), UserConnLinkData (..)) +import Simplex.Messaging.Client (pattern NRMInteractive) +import Simplex.Messaging.Version (mkVersionRange) +import Simplex.Messaging.Version.Internal (Version (..)) import qualified Simplex.Messaging.Crypto as C +import qualified Simplex.Messaging.Crypto.Ratchet as CR import Simplex.Messaging.Crypto.ShortLink (contactShortLinkKdf, invShortLinkKdf) import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String (strEncode) -import Simplex.Messaging.Protocol (EntityId (..), SMPServer, pattern SMPServer) +import Simplex.Messaging.Protocol (EntityId (..), SMPServer, SubscriptionMode (..), pattern SMPServer) import Simplex.Messaging.Server.Env.STM (AStoreType (..)) import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..)) import Simplex.Messaging.Server.Web (attachStaticAndWS) import Simplex.Messaging.Transport (TLS) import Simplex.Messaging.Transport.Client (TransportHost (..)) +import SMPAgentClient (agentCfg, initAgentServers, testDB) import SMPClient (cfgWebOn, testKeyHash, testPort, withSmpServerConfig) +import AgentTests.FunctionalAPITests (withAgent) import Test.Hspec hiding (it) import Util import XFTPWebTests (callNode_, jsOut, jsUint8) @@ -38,9 +47,14 @@ smpWebDir = "smp-web" callNode :: String -> IO B.ByteString callNode = callNode_ smpWebDir +impEnc :: String +impEnc = "import { Decoder, decodeLarge } from '@simplex-chat/xftp-web/dist/protocol/encoding.js';" + +impProto_ :: String +impProto_ = "import { encodeTransmission, encodeBatch, decodeTransmission, encodeLGET, decodeLNK, decodeResponse } from './dist/protocol.js';" + impProto :: String -impProto = "import { encodeTransmission, encodeBatch, decodeTransmission, encodeLGET, decodeLNK, decodeResponse } from './dist/protocol.js';" - <> "import { Decoder } from '@simplex-chat/xftp-web/dist/protocol/encoding.js';" +impProto = impEnc <> impProto_ impTransport :: String impTransport = "import { decodeSMPServerHandshake, encodeSMPClientHandshake } from './dist/transport.js';" @@ -50,8 +64,11 @@ impWS :: String impWS = "import { connectSMP, sendBlock, receiveBlock } from './dist/transport/websockets.js';" <> "import { blockPad, blockUnpad } from '@simplex-chat/xftp-web/dist/protocol/transmission.js';" +impAgentProto_ :: String +impAgentProto_ = "import { connShortLinkStrP, decodeConnLinkData, decodeProtocolServer, decodeConnShortLink, decodeOwnerAuth, decodeUserLinkData, parseProfile } from './dist/agent/protocol.js';" + impAgentProto :: String -impAgentProto = "import { connShortLinkStrP } from './dist/agent/protocol.js';" +impAgentProto = impEnc <> impAgentProto_ impCryptoShortLink :: String impCryptoShortLink = "import { contactShortLinkKdf, invShortLinkKdf, decryptLinkData } from './dist/crypto/shortLink.js';" @@ -63,6 +80,9 @@ impSodium = "import sodium from '@simplex-chat/xftp-web/node_modules/libsodium-w jsStr :: B.ByteString -> String jsStr bs = "'" <> BC.unpack bs <> "'" +runRight :: (Show e, HasCallStack) => ExceptT e IO a -> IO a +runRight action = runExceptT action >>= either (error . ("Unexpected error: " <>) . show) pure + smpWebTests :: SpecWith () smpWebTests = describe "SMP Web Client" $ do describe "protocol" $ do @@ -205,7 +225,49 @@ smpWebTests = describe "SMP Web Client" $ do tsResult `shouldBe` (fixedPlain <> B.singleton 0 <> userPlain) describe "agent/protocol" $ do - describe "ConnShortLink" $ do + describe "ProtocolServer binary" $ do + it "decodes Haskell-encoded server" $ do + let srv = SMPServer ("smp.example.com" :| ["smp2.example.com"]) "5223" (C.KeyHash $ B.pack [1..32]) + encoded = smpEncode srv + tsResult <- callNode $ impAgentProto + <> "const s = decodeProtocolServer(new Decoder(" <> jsUint8 encoded <> "));" + <> "const enc = new TextEncoder();" + <> jsOut ("new Uint8Array([s.hosts.length, ...s.keyHash, ...enc.encode(new TextDecoder().decode(s.port))])") + tsResult `shouldBe` B.pack ([2] ++ [1..32]) <> "5223" + + describe "ConnShortLink binary" $ do + it "decodes Haskell-encoded contact link" $ do + let srv = SMPServer ("relay.example.com" :| []) "" (C.KeyHash $ B.pack [1..32]) + linkKey = AP.LinkKey $ B.pack [50..81] + link = AP.CSLContact AP.SLSServer AP.CCTGroup srv linkKey + encoded = smpEncode link + tsResult <- callNode $ impAgentProto + <> "const l = decodeConnShortLink(new Decoder(" <> jsUint8 encoded <> "));" + <> jsOut ("new Uint8Array([l.mode === 'contact' ? 1 : 0, l.connType === 'group' ? 1 : 0, ...l.linkKey])") + tsResult `shouldBe` B.pack ([1, 1] ++ [50..81]) + + describe "ConnLinkData" $ do + it "decodes Haskell-encoded ContactLinkData with profile" $ do + let profileJson = "{\"displayName\":\"alice\",\"fullName\":\"Alice A\"}" + userData = AP.UserLinkData profileJson + ucd = AP.UserContactData {AP.direct = True, AP.owners = [], AP.relays = [], AP.userData = userData} + cld = AP.ContactLinkData (mkVersionRange (Version 1) (Version 3)) ucd :: AP.ConnLinkData 'AP.CMContact + encoded = smpEncode cld + tsResult <- callNode $ impAgentProto + <> "const r = decodeConnLinkData(new Decoder(" <> jsUint8 encoded <> "));" + <> "const p = parseProfile(r.userContactData.userData);" + <> "const enc = new TextEncoder();" + <> jsOut ("new Uint8Array([" + <> "r.agentVRange.min >> 8, r.agentVRange.min & 0xff," + <> "r.agentVRange.max >> 8, r.agentVRange.max & 0xff," + <> "r.userContactData.direct ? 1 : 0," + <> "r.userContactData.owners.length," + <> "r.userContactData.relays.length," + <> "...enc.encode(p.displayName)" + <> "])") + tsResult `shouldBe` B.pack [0, 1, 0, 3, 1, 0, 0] <> "alice" + + describe "ConnShortLink URI" $ do it "parses simplex: contact link" $ do let srv = SMPServer ("smp1.example.com" :| []) "" (C.KeyHash $ B.pack [1..32]) linkKey = AP.LinkKey $ B.pack [100..131] @@ -289,3 +351,44 @@ smpWebTests = describe "SMP Web Client" $ do <> "conn.ws.close(); setTimeout(() => process.exit(0), 100);" <> "} catch(e) { process.stderr.write('ERROR: ' + e.message + '\\n'); process.exit(1); }" tsResult `shouldBe` "PONG" + + describe "end-to-end" $ do + it "TypeScript fetches short link data via WebSocket" $ do + let msType = ASType SQSMemory SMSJournal + attachStaticAndWS "tests/fixtures" $ \attachHTTP -> + withSmpServerConfig (cfgWebOn msType testPort) (Just attachHTTP) $ \_serverThread -> + withAgent 1 agentCfg initAgentServers testDB $ \a -> do + let testData = "hello from short link" + userData = UserLinkData testData + userCtData = UserContactData {direct = True, owners = [], relays = [], userData = userData} + newLinkData = UserContactLinkData userCtData + (_connId, (CCLink _connReq (Just shortLink), Nothing)) <- + runRight $ A.createConnection a NRMInteractive 1 True True AP.SCMContact (Just newLinkData) Nothing CR.IKPQOn SMSubscribe + let linkUri = strEncode shortLink + tsResult <- callNode $ impSodium <> impWS <> impAgentProto <> impProto_ <> impCryptoShortLink + <> "try {" + -- 1. Parse short link URI + <> "const link = connShortLinkStrP(" <> jsStr linkUri <> ");" + -- 2. Derive keys + <> "const {linkId, sbKey} = contactShortLinkKdf(link.linkKey);" + -- 3. Connect via WSS + <> "const conn = await connectSMP('wss://localhost:" <> testPort <> "', " <> jsUint8 (C.unKeyHash testKeyHash) <> ", {rejectUnauthorized: false, ALPNProtocols: ['http/1.1']});" + -- 4. Send LGET + <> "const lget = encodeTransmission(new Uint8Array([0x31]), linkId, encodeLGET());" + <> "sendBlock(conn.ws, blockPad(encodeBatch(lget), 16384));" + -- 5. Receive LNK response + <> "const resp = await receiveBlock(conn.ws);" + <> "const rd = new Decoder(blockUnpad(resp));" + <> "rd.anyByte();" -- batch count + <> "const inner = decodeLarge(rd);" + <> "const t = decodeTransmission(new Decoder(inner));" + <> "const r = decodeResponse(new Decoder(t.command));" + <> "if (r.type !== 'LNK') throw new Error('expected LNK, got ' + r.type);" + -- 6. Decrypt + <> "const dec = decryptLinkData(sbKey, r.response.encFixedData, r.response.encUserData);" + -- 7. Parse ConnLinkData to get UserLinkData + <> "const cld = decodeConnLinkData(new Decoder(dec.userData));" + <> jsOut ("cld.userContactData.userData") + <> "conn.ws.close(); setTimeout(() => process.exit(0), 100);" + <> "} catch(e) { process.stderr.write('ERROR: ' + e.message + '\\n'); process.exit(1); }" + tsResult `shouldBe` testData