fetch link data via websocket

2026-03-30 14:16:00 +00:00 · 2026-03-27 21:48:31 +00:00
parent 8c1cfca208
commit 575d95c4d1
3 changed files with 279 additions and 21 deletions
--- a/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md
+++ b/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md
@@ -248,19 +248,35 @@ Each step produces working, tested code. Steps 1-11 work without block encryptio

 **Tests**: Haskell `encodeSignLinkData` + `sbEncrypt` with known key/nonce → TypeScript decrypts → plaintext matches.

-### Step 9: FixedLinkData / ConnLinkData Parse
+### Step 9: ConnLinkData Parse

 **File**: `agent/protocol.ts`
-**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnLinkData`, `UserContactData` Encoding instances
+**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `ConnLinkData`, `UserContactData`, `OwnerAuth`, `ConnShortLink`, `ProtocolServer` Encoding instances

-**Implementation**:
- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), `decodeLarge` for linkConnReq, optional `decodeBytes` for linkEntityId (if bytes remaining)
+**Implementation** (proper decoding, not skipping):
 - `decodeConnLinkData(d)`: `anyByte` for connectionMode ('C'=Contact), `decodeWord16` × 2 for agentVRange, then `decodeUserContactData`
- `decodeUserContactData(d)`: `decodeBool` for direct, `decodeList(decodeOwnerAuth, d)` for owners, `decodeList(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData
- `decodeUserLinkData(d)`: peek first byte — if 0xFF, skip it and `decodeLarge(d)`; otherwise `decodeBytes(d)`
+- `decodeUserContactData(d)`: `decodeBool` for direct, `smpListP(decodeOwnerAuth, d)` for owners, `smpListP(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData
+- `decodeOwnerAuth(d)`: `decodeBytes` for outer wrapper, then parse inner: `(ownerId, ownerKey, authOwnerSig)` all as ByteStrings
+- `decodeConnShortLink(d)`: `anyByte` for mode, then Contact: `(ctTypeChar, srv, linkKey)` or Invitation: `(srv, linkId, linkKey)`
+- `decodeProtocolServer(d)`: `decodeBytes` for scheme+keyHash, `decodeBytes` for host, `decodeBytes` for port — need to verify exact encoding
+- `decodeUserLinkData(d)`: first byte 0xFF → `decodeLarge`; otherwise it's the 1-byte length of a ByteString
 - `parseProfile(userData)`: check first byte for 'X' (0x58, zstd compressed) — if so, decompress; otherwise `JSON.parse` directly

-**Tests**: Haskell encodes full `FixedLinkData` and `ContactLinkData` with known values → TypeScript decodes → all fields match.
+**Tests**: Haskell encodes `ContactLinkData` with known values → TypeScript decodes → all fields match.
+
+**FixedLinkData**: deferred to step 15. `linkConnReq` (ConnectionRequestUri) is NOT length-prefixed in the tuple encoding — it requires full parsing. FixedLinkData is also needed to validate mutable data signature using rootKey.
+
+### Step 15: FixedLinkData Parse
+
+**File**: `agent/protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnectionRequestUri`, `ConnReqUriData` Encoding instances
+
+**Implementation**:
+- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), then parse `ConnectionRequestUri` (mode byte + `ConnReqUriData`), optional `decodeBytes` for linkEntityId
+- `decodeConnectionRequestUri(d)`: full parsing of `ConnReqUriData` including SMP queue URIs
+- Needed for: connecting to the contact, and validating mutable data signature with rootKey
+
+**Tests**: Haskell encodes full `FixedLinkData` → TypeScript decodes → rootKey and linkConnReq fields match.

 ### Step 10: WebSocket Transport

--- a/smp-web/src/agent/protocol.ts
+++ b/smp-web/src/agent/protocol.ts
@@ -2,6 +2,9 @@
 // Mirrors: Simplex.Messaging.Agent.Protocol

 import {base64urlDecode} from "@simplex-chat/xftp-web/dist/protocol/description.js"
+import {
+  Decoder, decodeBytes, decodeLarge, decodeWord16, decodeBool,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"

 // -- Short link types (Agent/Protocol.hs:1462-1470)

@@ -9,18 +12,158 @@ export type ShortLinkScheme = "simplex" | "https"

 export type ContactConnType = "contact" | "channel" | "group" | "relay"

+export interface ProtocolServer {
+  hosts: Uint8Array[] // NonEmpty, each is the strEncoded host bytes
+  port: Uint8Array
+  keyHash: Uint8Array
+}
+
+export type ConnShortLink =
+  | {mode: "invitation", scheme: ShortLinkScheme, server: ProtocolServer, linkId: Uint8Array, linkKey: Uint8Array}
+  | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ProtocolServer, linkKey: Uint8Array}
+
+// -- ProtocolServer binary encoding (Protocol.hs:1264-1269)
+// smpEncode (host, port, keyHash)
+// host: NonEmpty TransportHost = smpEncodeList (1-byte count + each as ByteString)
+// port: String = ByteString (1-byte len + bytes)
+// keyHash: KeyHash = ByteString (1-byte len + bytes)
+
+export function decodeProtocolServer(d: Decoder): ProtocolServer {
+  const hostCount = d.anyByte()
+  if (hostCount === 0) throw new Error("empty server host list")
+  const hosts: Uint8Array[] = []
+  for (let i = 0; i < hostCount; i++) hosts.push(decodeBytes(d))
+  const port = decodeBytes(d)
+  const keyHash = decodeBytes(d)
+  return {hosts, port, keyHash}
+}
+
+// -- ConnShortLink binary encoding (Agent/Protocol.hs:1631-1649)
+// Contact: smpEncode (CMContact, ctTypeChar, srv, linkKey)
+// Invitation: smpEncode (CMInvitation, srv, linkId, linkKey)
+
+export interface ConnShortLinkBinary {
+  mode: "contact" | "invitation"
+  connType?: ContactConnType
+  server: ProtocolServer
+  linkId?: Uint8Array
+  linkKey: Uint8Array
+}
+
+const ctTypeFromByte: Record<number, ContactConnType> = {
+  0x41: "contact",  // 'A'
+  0x43: "channel",  // 'C'
+  0x47: "group",    // 'G'
+  0x52: "relay",    // 'R'
+}
+
+export function decodeConnShortLink(d: Decoder): ConnShortLinkBinary {
+  const mode = d.anyByte()
+  if (mode === 0x49) {
+    // Invitation: (srv, linkId, linkKey)
+    const server = decodeProtocolServer(d)
+    const linkId = decodeBytes(d)
+    const linkKey = decodeBytes(d)
+    return {mode: "invitation", server, linkId, linkKey}
+  } else if (mode === 0x43) {
+    // Contact: (ctTypeChar, srv, linkKey)
+    const ctByte = d.anyByte()
+    const connType = ctTypeFromByte[ctByte]
+    if (!connType) throw new Error("unknown contact type: 0x" + ctByte.toString(16))
+    const server = decodeProtocolServer(d)
+    const linkKey = decodeBytes(d)
+    return {mode: "contact", connType, server, linkKey}
+  }
+  throw new Error("unknown ConnShortLink mode: 0x" + mode.toString(16))
+}
+
+// -- OwnerAuth (Agent/Protocol.hs:1793-1800)
+// Outer ByteString wrapping inner: (ownerId, ownerKey, authOwnerSig)
+
+export interface OwnerAuth {
+  ownerId: Uint8Array
+  ownerKey: Uint8Array
+  authOwnerSig: Uint8Array
+}
+
+export function decodeOwnerAuth(d: Decoder): OwnerAuth {
+  const inner = decodeBytes(d)
+  const id = new Decoder(inner)
+  const ownerId = decodeBytes(id)
+  const ownerKey = decodeBytes(id)
+  const authOwnerSig = decodeBytes(id)
+  return {ownerId, ownerKey, authOwnerSig}
+}
+
+// -- UserLinkData (Agent/Protocol.hs:1891-1894)
+// If first byte is 0xFF, read Large; otherwise it's a ByteString (1-byte length)
+
+export function decodeUserLinkData(d: Decoder): Uint8Array {
+  const firstByte = d.anyByte()
+  if (firstByte === 0xFF) return decodeLarge(d)
+  return d.take(firstByte)
+}
+
+// -- UserContactData (Agent/Protocol.hs:1881-1889)
+
+export interface UserContactData {
+  direct: boolean
+  owners: OwnerAuth[]
+  relays: ConnShortLinkBinary[]
+  userData: Uint8Array
+}
+
+export function decodeUserContactData(d: Decoder): UserContactData {
+  const direct = decodeBool(d)
+  const ownerCount = d.anyByte()
+  const owners: OwnerAuth[] = []
+  for (let i = 0; i < ownerCount; i++) owners.push(decodeOwnerAuth(d))
+  const relayCount = d.anyByte()
+  const relays: ConnShortLinkBinary[] = []
+  for (let i = 0; i < relayCount; i++) relays.push(decodeConnShortLink(d))
+  const userData = decodeUserLinkData(d)
+  return {direct, owners, relays, userData}
+}
+
+// -- ConnLinkData (Agent/Protocol.hs:1838-1855)
+// Contact: 'C' + versionRange + UserContactData
+
+export interface ConnLinkDataContact {
+  mode: "contact"
+  agentVRange: {min: number; max: number}
+  userContactData: UserContactData
+}
+
+export function decodeConnLinkData(d: Decoder): ConnLinkDataContact {
+  const modeChar = d.anyByte()
+  if (modeChar !== 0x43) throw new Error("expected Contact mode 'C' (0x43), got 0x" + modeChar.toString(16))
+  const min = decodeWord16(d)
+  const max = decodeWord16(d)
+  const userContactData = decodeUserContactData(d)
+  return {mode: "contact", agentVRange: {min, max}, userContactData}
+}
+
+// -- Profile extraction
+
+export function parseProfile(userData: Uint8Array): unknown {
+  if (userData.length > 0 && userData[0] === 0x58) {
+    throw new Error("zstd-compressed profile not yet supported")
+  }
+  return JSON.parse(new TextDecoder().decode(userData))
+}
+
+// -- Short link URI parsing (below) --
+
 export interface ShortLinkServer {
  hosts: string[]
  port: string
  keyHash: Uint8Array
 }

-export type ConnShortLink =
+export type ConnShortLinkURI =
  | {mode: "invitation", scheme: ShortLinkScheme, server: ShortLinkServer, linkId: Uint8Array, linkKey: Uint8Array}
  | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ShortLinkServer, linkKey: Uint8Array}

-// -- Contact type char mapping (Agent/Protocol.hs:1651-1666)
-
 const ctTypeFromChar: Record<string, ContactConnType> = {
  a: "contact",
  c: "channel",
@@ -28,10 +171,8 @@ const ctTypeFromChar: Record<string, ContactConnType> = {
  r: "relay",
 }

-// -- Short link parser (Agent/Protocol.hs:1596-1629)
-// Mirrors strP for AConnShortLink
-
-export function connShortLinkStrP(uri: string): ConnShortLink {
+// Mirrors strP for AConnShortLink (Agent/Protocol.hs:1596-1629)
+export function connShortLinkStrP(uri: string): ConnShortLinkURI {
  let scheme: ShortLinkScheme
  let firstHost: string | null = null
  let rest: string
@@ -50,7 +191,6 @@ export function connShortLinkStrP(uri: string): ConnShortLink {
    throw new Error("bad short link scheme")
  }

-  // rest: /typeChar#fragment?query
  if (rest[0] !== "/") throw new Error("bad short link: expected /")
  const typeChar = rest[1]
  const hashIdx = rest.indexOf("#")
@@ -62,7 +202,6 @@ export function connShortLinkStrP(uri: string): ConnShortLink {
  const queryStr = qIdx >= 0 ? afterHash.slice(qIdx + 1) : ""
  const params = new URLSearchParams(queryStr)

-  // Build server
  const hParam = params.get("h")
  const additionalHosts = hParam ? hParam.split(",") : []
  const allHosts = firstHost ? [firstHost, ...additionalHosts] : additionalHosts
--- a/tests/SMPWebTests.hs
+++ b/tests/SMPWebTests.hs
@@ -12,22 +12,31 @@
 -- Run: cabal test --test-option=--match="/SMP Web Client/"
 module SMPWebTests (smpWebTests) where

+import Control.Monad.Except (ExceptT, runExceptT)
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as BC
 import Data.List.NonEmpty (NonEmpty (..))
 import Data.Word (Word16)
+import qualified Simplex.Messaging.Agent as A
 import qualified Simplex.Messaging.Agent.Protocol as AP
+import Simplex.Messaging.Agent.Protocol (CreatedConnLink (..), UserLinkData (..), UserContactData (..), UserConnLinkData (..))
+import Simplex.Messaging.Client (pattern NRMInteractive)
+import Simplex.Messaging.Version (mkVersionRange)
+import Simplex.Messaging.Version.Internal (Version (..))
 import qualified Simplex.Messaging.Crypto as C
+import qualified Simplex.Messaging.Crypto.Ratchet as CR
 import Simplex.Messaging.Crypto.ShortLink (contactShortLinkKdf, invShortLinkKdf)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String (strEncode)
-import Simplex.Messaging.Protocol (EntityId (..), SMPServer, pattern SMPServer)
+import Simplex.Messaging.Protocol (EntityId (..), SMPServer, SubscriptionMode (..), pattern SMPServer)
 import Simplex.Messaging.Server.Env.STM (AStoreType (..))
 import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..))
 import Simplex.Messaging.Server.Web (attachStaticAndWS)
 import Simplex.Messaging.Transport (TLS)
 import Simplex.Messaging.Transport.Client (TransportHost (..))
+import SMPAgentClient (agentCfg, initAgentServers, testDB)
 import SMPClient (cfgWebOn, testKeyHash, testPort, withSmpServerConfig)
+import AgentTests.FunctionalAPITests (withAgent)
 import Test.Hspec hiding (it)
 import Util
 import XFTPWebTests (callNode_, jsOut, jsUint8)
@@ -38,9 +47,14 @@ smpWebDir = "smp-web"
 callNode :: String -> IO B.ByteString
 callNode = callNode_ smpWebDir

+impEnc :: String
+impEnc = "import { Decoder, decodeLarge } from '@simplex-chat/xftp-web/dist/protocol/encoding.js';"
+
+impProto_ :: String
+impProto_ = "import { encodeTransmission, encodeBatch, decodeTransmission, encodeLGET, decodeLNK, decodeResponse } from './dist/protocol.js';"
+
 impProto :: String
-impProto = "import { encodeTransmission, encodeBatch, decodeTransmission, encodeLGET, decodeLNK, decodeResponse } from './dist/protocol.js';"
-  <> "import { Decoder } from '@simplex-chat/xftp-web/dist/protocol/encoding.js';"
+impProto = impEnc <> impProto_

 impTransport :: String
 impTransport = "import { decodeSMPServerHandshake, encodeSMPClientHandshake } from './dist/transport.js';"
@@ -50,8 +64,11 @@ impWS :: String
 impWS = "import { connectSMP, sendBlock, receiveBlock } from './dist/transport/websockets.js';"
  <> "import { blockPad, blockUnpad } from '@simplex-chat/xftp-web/dist/protocol/transmission.js';"

+impAgentProto_ :: String
+impAgentProto_ = "import { connShortLinkStrP, decodeConnLinkData, decodeProtocolServer, decodeConnShortLink, decodeOwnerAuth, decodeUserLinkData, parseProfile } from './dist/agent/protocol.js';"
+
 impAgentProto :: String
-impAgentProto = "import { connShortLinkStrP } from './dist/agent/protocol.js';"
+impAgentProto = impEnc <> impAgentProto_

 impCryptoShortLink :: String
 impCryptoShortLink = "import { contactShortLinkKdf, invShortLinkKdf, decryptLinkData } from './dist/crypto/shortLink.js';"
@@ -63,6 +80,9 @@ impSodium = "import sodium from '@simplex-chat/xftp-web/node_modules/libsodium-w
 jsStr :: B.ByteString -> String
 jsStr bs = "'" <> BC.unpack bs <> "'"

+runRight :: (Show e, HasCallStack) => ExceptT e IO a -> IO a
+runRight action = runExceptT action >>= either (error . ("Unexpected error: " <>) . show) pure
+
 smpWebTests :: SpecWith ()
 smpWebTests = describe "SMP Web Client" $ do
  describe "protocol" $ do
@@ -205,7 +225,49 @@ smpWebTests = describe "SMP Web Client" $ do
            tsResult `shouldBe` (fixedPlain <> B.singleton 0 <> userPlain)

  describe "agent/protocol" $ do
-    describe "ConnShortLink" $ do
+    describe "ProtocolServer binary" $ do
+      it "decodes Haskell-encoded server" $ do
+        let srv = SMPServer ("smp.example.com" :| ["smp2.example.com"]) "5223" (C.KeyHash $ B.pack [1..32])
+            encoded = smpEncode srv
+        tsResult <- callNode $ impAgentProto
+          <> "const s = decodeProtocolServer(new Decoder(" <> jsUint8 encoded <> "));"
+          <> "const enc = new TextEncoder();"
+          <> jsOut ("new Uint8Array([s.hosts.length, ...s.keyHash, ...enc.encode(new TextDecoder().decode(s.port))])")
+        tsResult `shouldBe` B.pack ([2] ++ [1..32]) <> "5223"
+
+    describe "ConnShortLink binary" $ do
+      it "decodes Haskell-encoded contact link" $ do
+        let srv = SMPServer ("relay.example.com" :| []) "" (C.KeyHash $ B.pack [1..32])
+            linkKey = AP.LinkKey $ B.pack [50..81]
+            link = AP.CSLContact AP.SLSServer AP.CCTGroup srv linkKey
+            encoded = smpEncode link
+        tsResult <- callNode $ impAgentProto
+          <> "const l = decodeConnShortLink(new Decoder(" <> jsUint8 encoded <> "));"
+          <> jsOut ("new Uint8Array([l.mode === 'contact' ? 1 : 0, l.connType === 'group' ? 1 : 0, ...l.linkKey])")
+        tsResult `shouldBe` B.pack ([1, 1] ++ [50..81])
+
+    describe "ConnLinkData" $ do
+      it "decodes Haskell-encoded ContactLinkData with profile" $ do
+        let profileJson = "{\"displayName\":\"alice\",\"fullName\":\"Alice A\"}"
+            userData = AP.UserLinkData profileJson
+            ucd = AP.UserContactData {AP.direct = True, AP.owners = [], AP.relays = [], AP.userData = userData}
+            cld = AP.ContactLinkData (mkVersionRange (Version 1) (Version 3)) ucd :: AP.ConnLinkData 'AP.CMContact
+            encoded = smpEncode cld
+        tsResult <- callNode $ impAgentProto
+          <> "const r = decodeConnLinkData(new Decoder(" <> jsUint8 encoded <> "));"
+          <> "const p = parseProfile(r.userContactData.userData);"
+          <> "const enc = new TextEncoder();"
+          <> jsOut ("new Uint8Array(["
+          <> "r.agentVRange.min >> 8, r.agentVRange.min & 0xff,"
+          <> "r.agentVRange.max >> 8, r.agentVRange.max & 0xff,"
+          <> "r.userContactData.direct ? 1 : 0,"
+          <> "r.userContactData.owners.length,"
+          <> "r.userContactData.relays.length,"
+          <> "...enc.encode(p.displayName)"
+          <> "])")
+        tsResult `shouldBe` B.pack [0, 1, 0, 3, 1, 0, 0] <> "alice"
+
+    describe "ConnShortLink URI" $ do
      it "parses simplex: contact link" $ do
        let srv = SMPServer ("smp1.example.com" :| []) "" (C.KeyHash $ B.pack [1..32])
            linkKey = AP.LinkKey $ B.pack [100..131]
@@ -289,3 +351,44 @@ smpWebTests = describe "SMP Web Client" $ do
              <> "conn.ws.close(); setTimeout(() => process.exit(0), 100);"
              <> "} catch(e) { process.stderr.write('ERROR: ' + e.message + '\\n'); process.exit(1); }"
            tsResult `shouldBe` "PONG"
+
+  describe "end-to-end" $ do
+    it "TypeScript fetches short link data via WebSocket" $ do
+      let msType = ASType SQSMemory SMSJournal
+      attachStaticAndWS "tests/fixtures" $ \attachHTTP ->
+        withSmpServerConfig (cfgWebOn msType testPort) (Just attachHTTP) $ \_serverThread ->
+          withAgent 1 agentCfg initAgentServers testDB $ \a -> do
+            let testData = "hello from short link"
+                userData = UserLinkData testData
+                userCtData = UserContactData {direct = True, owners = [], relays = [], userData = userData}
+                newLinkData = UserContactLinkData userCtData
+            (_connId, (CCLink _connReq (Just shortLink), Nothing)) <-
+              runRight $ A.createConnection a NRMInteractive 1 True True AP.SCMContact (Just newLinkData) Nothing CR.IKPQOn SMSubscribe
+            let linkUri = strEncode shortLink
+            tsResult <- callNode $ impSodium <> impWS <> impAgentProto <> impProto_ <> impCryptoShortLink
+              <> "try {"
+              -- 1. Parse short link URI
+              <> "const link = connShortLinkStrP(" <> jsStr linkUri <> ");"
+              -- 2. Derive keys
+              <> "const {linkId, sbKey} = contactShortLinkKdf(link.linkKey);"
+              -- 3. Connect via WSS
+              <> "const conn = await connectSMP('wss://localhost:" <> testPort <> "', " <> jsUint8 (C.unKeyHash testKeyHash) <> ", {rejectUnauthorized: false, ALPNProtocols: ['http/1.1']});"
+              -- 4. Send LGET
+              <> "const lget = encodeTransmission(new Uint8Array([0x31]), linkId, encodeLGET());"
+              <> "sendBlock(conn.ws, blockPad(encodeBatch(lget), 16384));"
+              -- 5. Receive LNK response
+              <> "const resp = await receiveBlock(conn.ws);"
+              <> "const rd = new Decoder(blockUnpad(resp));"
+              <> "rd.anyByte();"  -- batch count
+              <> "const inner = decodeLarge(rd);"
+              <> "const t = decodeTransmission(new Decoder(inner));"
+              <> "const r = decodeResponse(new Decoder(t.command));"
+              <> "if (r.type !== 'LNK') throw new Error('expected LNK, got ' + r.type);"
+              -- 6. Decrypt
+              <> "const dec = decryptLinkData(sbKey, r.response.encFixedData, r.response.encUserData);"
+              -- 7. Parse ConnLinkData to get UserLinkData
+              <> "const cld = decodeConnLinkData(new Decoder(dec.userData));"
+              <> jsOut ("cld.userContactData.userData")
+              <> "conn.ws.close(); setTimeout(() => process.exit(0), 100);"
+              <> "} catch(e) { process.stderr.write('ERROR: ' + e.message + '\\n'); process.exit(1); }"
+            tsResult `shouldBe` testData