diff --git a/apps/xftp-server/static/xftp-web-bundle/index.js b/apps/xftp-server/static/xftp-web-bundle/index.js index 49b8d9d7f..eca2b10bf 100644 --- a/apps/xftp-server/static/xftp-web-bundle/index.js +++ b/apps/xftp-server/static/xftp-web-bundle/index.js @@ -625,12 +625,15 @@ function parseReplicaField(entry, lines, nextIdx, group) { return nextIdx; } function validateFileDescription(fd) { + if (fd.chunks.length === 0) return "no chunks"; for (let i = 0; i < fd.chunks.length; i++) { if (fd.chunks[i].chunkNo !== i + 1) return "chunk numbers are not sequential"; } let total = 0; for (const c of fd.chunks) total += c.chunkSize; if (total !== fd.size) return "chunks total size is different than file size"; + if (fd.chunks.some((c) => !serverChunkSizes.includes(c.chunkSize))) return "incorrect chunk size"; + if (fd.redirect !== null && fd.size > chunkSize3) return "redirect file description is too large"; return null; } @@ -10279,6 +10282,7 @@ const chunkSize0 = kb(64); const chunkSize1 = kb(256); const chunkSize2 = mb(1); const chunkSize3 = mb(4); +const serverChunkSizes = [chunkSize0, chunkSize1, chunkSize2, chunkSize3]; const fileSizeLen = 8; const authTagSize = 16; function size34(sz) { @@ -10961,16 +10965,17 @@ async function uploadXFTPChunk(agent, server, spKey, fId, chunkData) { const { response } = await sendXFTPCommand(agent, server, spKey, fId, encodeFPUT(), chunkData); if (response.type !== "FROk") throw new Error("unexpected response: " + response.type); } -async function downloadXFTPChunkRaw(agent, server, rpKey, fId) { +async function downloadXFTPChunkRaw(agent, server, rpKey, fId, chunkSize) { const { publicKey, privateKey } = generateX25519KeyPair(); const cmd = encodeFGET(encodePubKeyX25519(publicKey)); const { response, body } = await sendXFTPCommand(agent, server, rpKey, fId, cmd); if (response.type !== "FRFile") throw new Error("unexpected response: " + response.type); + if (body.length !== chunkSize + authTagSize) throw new Error("chunk size mismatch"); const dhSecret = dh(response.rcvDhKey, privateKey); return { dhSecret, nonce: response.nonce, body }; } -async function downloadXFTPChunk(agent, server, rpKey, fId, digest) { - const { dhSecret, nonce, body } = await downloadXFTPChunkRaw(agent, server, rpKey, fId); +async function downloadXFTPChunk(agent, server, rpKey, fId, chunkSize, digest) { + const { dhSecret, nonce, body } = await downloadXFTPChunkRaw(agent, server, rpKey, fId, chunkSize); return decryptReceivedChunk(dhSecret, nonce, body, digest ?? null); } @@ -11161,18 +11166,18 @@ async function downloadFileRaw(agent, fd, onRawChunk, options) { let downloaded = 0; const byServer = /* @__PURE__ */ new Map(); for (const chunk of resolvedFd.chunks) { - const srv = chunk.replicas[0]?.server ?? ""; - if (!byServer.has(srv)) byServer.set(srv, []); - byServer.get(srv).push(chunk); + const server = parseXFTPServer(chunk.replicas[0]?.server ?? ""); + const key = formatXFTPServer(server); + if (!byServer.has(key)) byServer.set(key, { server, chunks: [] }); + byServer.get(key).chunks.push(chunk); } - await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => { - const server = parseXFTPServer(srv); + await Promise.all([...byServer.values()].map(async ({ server, chunks }) => { for (const chunk of chunks) { const replica = chunk.replicas[0]; if (!replica) throw new Error("downloadFileRaw: chunk has no replicas"); const seed = decodePrivKeyEd25519(replica.replicaKey); const kp = ed25519KeyPairFromSeed(seed); - const raw = await downloadXFTPChunkRaw(agent, server, kp.privateKey, replica.replicaId); + const raw = await downloadXFTPChunkRaw(agent, server, kp.privateKey, replica.replicaId, chunk.chunkSize); await onRawChunk({ chunkNo: chunk.chunkNo, dhSecret: raw.dhSecret, @@ -11194,7 +11199,7 @@ async function resolveRedirect(agent, fd) { const server = parseXFTPServer(replica.server); const seed = decodePrivKeyEd25519(replica.replicaKey); const kp = ed25519KeyPairFromSeed(seed); - const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest); + const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.chunkSize, chunk.digest); plaintextChunks[chunk.chunkNo - 1] = data; } const totalSize = plaintextChunks.reduce((s, c) => s + c.length, 0); @@ -11301,13 +11306,17 @@ function initUpload(app) { stage.hidden = false; } function showError(msg) { - errorMsg.innerHTML = msg; + errorMsg.textContent = msg; + showStage(errorStage); + } + function showErrorHTML(html) { + errorMsg.innerHTML = html; showStage(errorStage); } async function startUpload(file) { aborted = false; if (file.size > MAX_SIZE) { - showError(t("fileTooLarge", "File too large (%size%). Maximum is 100 MB. The SimpleX app supports files up to 1 GB.").replace("%size%", formatSize$1(file.size))); + showErrorHTML(t("fileTooLarge", "File too large (%size%). Maximum is 100 MB. The SimpleX app supports files up to 1 GB.").replace("%size%", formatSize$1(file.size))); return; } if (file.size === 0) { @@ -11449,7 +11458,7 @@ function initDownload(app, hash) { stage.hidden = false; } function showError(msg) { - errorMsg.innerHTML = msg; + errorMsg.textContent = msg; showStage(errorStage); } if (wrongServer) { @@ -11584,7 +11593,8 @@ window.__xftp_initApp = async () => { main().catch((err) => { const app = getAppElement(); if (app) { - app.innerHTML = `

${t("initError", "Failed to initialize: %error%").replace("%error%", err.message)}

`; + app.innerHTML = '

'; + app.querySelector("p").textContent = t("initError", "Failed to initialize: %error%").replace("%error%", err.message); } console.error(err); }); diff --git a/tests/XFTPWebTests.hs b/tests/XFTPWebTests.hs index 6148b3318..4d1275f26 100644 --- a/tests/XFTPWebTests.hs +++ b/tests/XFTPWebTests.hs @@ -20,6 +20,7 @@ import qualified Crypto.PubKey.Curve25519 as X25519 import qualified Crypto.PubKey.Ed25519 as Ed25519 import qualified Data.ByteArray as BA import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as LB import Data.Int (Int64) import Data.List (intercalate) @@ -194,7 +195,7 @@ xftpWebTests = do pendingWith "TS project not compiled" xftpWebSourceHygieneTests :: Spec -xftpWebSourceHygieneTests = describe "source hygiene" $ +xftpWebSourceHygieneTests = describe "source hygiene" $ do it "does not commit XFTP web debug logs that expose secrets or plaintext" $ do let files = [ "xftp-web/src/client.ts", @@ -208,6 +209,17 @@ xftpWebSourceHygieneTests = describe "source hygiene" $ forM_ files $ \path -> do contents <- B.readFile path mapM_ (\marker -> contents `shouldNotSatisfy` B.isInfixOf marker) markers + it "renders XFTP web error messages as text" $ do + let files = + [ "xftp-web/web/download.ts", + "xftp-web/web/upload.ts", + "xftp-web/web/main.ts", + "apps/xftp-server/static/xftp-web-bundle/index.js" + ] + messageAsHTML line = "innerHTML" `B.isInfixOf` line && any (`B.isInfixOf` line) ["msg", "message"] + forM_ files $ \path -> do + contents <- B.readFile path + (path, filter messageAsHTML $ BC.lines contents) `shouldBe` (path, []) -- ── protocol/encoding ────────────────────────────────────────────── @@ -2234,6 +2246,53 @@ tsDescriptionTests = describe "protocol/description" $ do <> jsOut "new TextEncoder().encode(r || 'ok')" tsResult `shouldBe` "chunks total size is different than file size" + it "incorrect chunk size" $ do + fixture <- B.readFile "tests/fixtures/file_description.yaml" + tsResult <- + callNode $ + impDesc + <> "const yaml = new TextDecoder().decode(" + <> jsUint8 fixture + <> ");" + <> "const fd = Desc.decodeFileDescription(yaml);" + <> "fd.chunks[3].chunkSize = 2097152;" + <> "fd.size += 1048576;" + <> "const r = Desc.validateFileDescription(fd);" + <> jsOut "new TextEncoder().encode(r || 'ok')" + tsResult `shouldBe` "incorrect chunk size" + + it "redirect description size limit" $ do + fixture <- B.readFile "tests/fixtures/file_description.yaml" + tsResult <- + callNode $ + impDesc + <> "const yaml = new TextDecoder().decode(" + <> jsUint8 fixture + <> ");" + <> "const fd = Desc.decodeFileDescription(yaml);" + <> "fd.redirect = {size: 1024, digest: new Uint8Array(64)};" + <> "const tooLarge = Desc.validateFileDescription(fd);" + <> "fd.chunks = fd.chunks.slice(0, 1);" + <> "fd.size = fd.chunks[0].chunkSize;" + <> "const atLimit = Desc.validateFileDescription(fd);" + <> jsOut "new TextEncoder().encode([tooLarge || 'ok', atLimit || 'ok'].join(','))" + tsResult `shouldBe` "redirect file description is too large,ok" + + it "no chunks" $ do + fixture <- B.readFile "tests/fixtures/file_description.yaml" + tsResult <- + callNode $ + impDesc + <> "const yaml = new TextDecoder().decode(" + <> jsUint8 fixture + <> ");" + <> "const fd = Desc.decodeFileDescription(yaml);" + <> "fd.chunks = [];" + <> "fd.size = 0;" + <> "const r = Desc.validateFileDescription(fd);" + <> jsOut "new TextEncoder().encode(r || 'ok')" + tsResult `shouldBe` "no chunks" + -- ── protocol/chunks ─────────────────────────────────────────────── tsChunkTests :: Spec @@ -2863,6 +2922,12 @@ tsIntegrationTests = describe "integration" $ do agentDeleteTest testXFTPServerConfigEd25519SNI "tests/fixtures/ed25519/ca.crt" it "agent redirect: upload with redirect, download" $ agentRedirectTest testXFTPServerConfigEd25519SNI "tests/fixtures/ed25519/ca.crt" + it "agent download: accept only chunks of the size in the description" $ + agentChunkSizeTest "r.server" "null" testXFTPServerConfigEd25519SNI "tests/fixtures/ed25519/ca.crt" + it "agent download: download chunks of one server in sequence for all its addresses" $ + agentChunkSizeTest "r.server + ',' + chunkNo" "null" testXFTPServerConfigEd25519SNI "tests/fixtures/ed25519/ca.crt" + it "agent redirect: accept only chunks of the size in the description" $ + agentChunkSizeTest "r.server" "{size: 1024, digest: new Uint8Array(64)}" testXFTPServerConfigEd25519SNI "tests/fixtures/ed25519/ca.crt" it "cross-language: TS upload, Haskell download" $ tsUploadHaskellDownloadTest testXFTPServerConfigSNI "tests/fixtures/ca.crt" it "cross-language: TS upload with redirect, Haskell download" $ @@ -2967,11 +3032,11 @@ fullRoundTripTest cfg caFile = do \const rcvKeys = [K.encodePubKeyEd25519(rcvKp1.publicKey)];\ \const {senderId, recipientIds} = await createXFTPChunk(agent, server, sndKp.privateKey, file, rcvKeys, null);\ \await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData);\ - \const dl1 = await downloadXFTPChunk(agent, server, rcvKp1.privateKey, recipientIds[0], digest);\ + \const dl1 = await downloadXFTPChunk(agent, server, rcvKp1.privateKey, recipientIds[0], chunkData.length, digest);\ \const match1 = dl1.length === chunkData.length && dl1.every((b, i) => b === chunkData[i]);\ \const newIds = await addXFTPRecipients(agent, server, sndKp.privateKey, senderId,\ \ [K.encodePubKeyEd25519(rcvKp2.publicKey)]);\ - \const dl2 = await downloadXFTPChunk(agent, server, rcvKp2.privateKey, newIds[0], digest);\ + \const dl2 = await downloadXFTPChunk(agent, server, rcvKp2.privateKey, newIds[0], chunkData.length, digest);\ \const match2 = dl2.length === chunkData.length && dl2.every((b, i) => b === chunkData[i]);\ \await deleteXFTPChunk(agent, server, sndKp.privateKey, senderId);\ \closeXFTPAgent(agent);" @@ -3110,6 +3175,59 @@ agentRedirectTest cfg caFile = do <> jsOut "new Uint8Array([hasRedirect, nameMatch, sizeMatch, dataMatch])" result `shouldBe` B.pack [1, 1, 1, 1] +agentChunkSizeTest :: HasTestEnv => String -> String -> XFTPServerConfig STMFileStore -> FilePath -> Expectation +agentChunkSizeTest replicaServer redirect cfg caFile = do + withXFTPServerCfg cfg $ \_ -> do + Fingerprint fp <- loadFileFingerprint caFile + let fpStr = map (toEnum . fromIntegral) $ B.unpack $ strEncode fp + addr = "xftp://" <> fpStr <> "@localhost:" <> xftpTestPort + result <- + callNode $ + "import sodium from 'libsodium-wrappers-sumo';\ + \import crypto from 'node:crypto';\ + \import * as Addr from './dist/protocol/address.js';\ + \import * as Agent from './dist/agent.js';\ + \await sodium.ready;\ + \const server = Addr.parseXFTPServer('" + <> addr + <> "');\ + \const uploadAgent = Agent.newXFTPAgent();\ + \const encrypted = Agent.encryptFileForUpload(new Uint8Array(crypto.randomBytes(200000)), 'chunk-size.bin');\ + \const {rcvDescription} = await Agent.uploadFile(uploadAgent, [server], encrypted);\ + \Agent.closeXFTPAgent(uploadAgent);\ + \const [chunk] = rcvDescription.chunks;\ + \const chunks = [1, 2, 3, 4].map(chunkNo => ({\ + \ ...chunk, chunkNo, chunkSize: 65536, replicas: chunk.replicas.map(r => ({...r, server: " + <> replicaServer + <> "}))\ + \}));\ + \const fd = {...rcvDescription, size: 262144, chunkSize: 65536, chunks, redirect: " + <> redirect + <> "};\ + \const agent = Agent.newXFTPAgent();\ + \const connect = agent._connectFn;\ + \let bodies = 0;\ + \agent._connectFn = async (srv) => {\ + \ const client = await connect(srv);\ + \ const post = client.transport.post;\ + \ client.transport.post = async (body, headers) => {\ + \ const resp = await post(body, headers);\ + \ if (resp.length > 16384) bodies++;\ + \ return resp;\ + \ };\ + \ return client;\ + \};\ + \let stored = 0;\ + \let outcome = 'downloaded';\ + \try {\ + \ await Agent.downloadFileRaw(agent, fd, async () => { stored++; });\ + \} catch (e) {\ + \ outcome = e.message;\ + \}\ + \Agent.closeXFTPAgent(agent);" + <> jsOut "new TextEncoder().encode([chunk.chunkSize, outcome, stored, bodies].join(','))" + result `shouldBe` "262144,chunk size mismatch,0,1" + tsUploadHaskellDownloadTest :: HasTestEnv => XFTPServerConfig STMFileStore -> FilePath -> Expectation tsUploadHaskellDownloadTest cfg caFile = do createDirectoryIfMissing False recipientFiles diff --git a/xftp-web/src/agent.ts b/xftp-web/src/agent.ts index 4a3b2fcff..7caa8d02b 100644 --- a/xftp-web/src/agent.ts +++ b/xftp-web/src/agent.ts @@ -284,20 +284,20 @@ export async function downloadFileRaw( const resolvedFd = fd // Group chunks by server, sequential within each server, parallel across servers let downloaded = 0 - const byServer = new Map() + const byServer = new Map() for (const chunk of resolvedFd.chunks) { - const srv = chunk.replicas[0]?.server ?? "" - if (!byServer.has(srv)) byServer.set(srv, []) - byServer.get(srv)!.push(chunk) + const server = parseXFTPServer(chunk.replicas[0]?.server ?? "") + const key = formatXFTPServer(server) + if (!byServer.has(key)) byServer.set(key, {server, chunks: []}) + byServer.get(key)!.chunks.push(chunk) } - await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => { - const server = parseXFTPServer(srv) + await Promise.all([...byServer.values()].map(async ({server, chunks}) => { for (const chunk of chunks) { const replica = chunk.replicas[0] if (!replica) throw new Error("downloadFileRaw: chunk has no replicas") const seed = decodePrivKeyEd25519(replica.replicaKey) const kp = ed25519KeyPairFromSeed(seed) - const raw = await downloadXFTPChunkRaw(agent, server, kp.privateKey, replica.replicaId) + const raw = await downloadXFTPChunkRaw(agent, server, kp.privateKey, replica.replicaId, chunk.chunkSize) await onRawChunk({ chunkNo: chunk.chunkNo, dhSecret: raw.dhSecret, @@ -341,7 +341,7 @@ async function resolveRedirect( const server = parseXFTPServer(replica.server) const seed = decodePrivKeyEd25519(replica.replicaKey) const kp = ed25519KeyPairFromSeed(seed) - const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest) + const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.chunkSize, chunk.digest) plaintextChunks[chunk.chunkNo - 1] = data } const totalSize = plaintextChunks.reduce((s, c) => s + c.length, 0) diff --git a/xftp-web/src/client.ts b/xftp-web/src/client.ts index 0c6cd4d06..03296e029 100644 --- a/xftp-web/src/client.ts +++ b/xftp-web/src/client.ts @@ -24,6 +24,7 @@ import type {XFTPServer} from "./protocol/address.js" import {formatXFTPServer} from "./protocol/address.js" import {concatBytes} from "./protocol/encoding.js" import {blockUnpad} from "./protocol/transmission.js" +import {authTagSize} from "./protocol/chunks.js" // -- Error types @@ -401,20 +402,21 @@ export interface RawChunkResponse { } export async function downloadXFTPChunkRaw( - agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array + agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array, chunkSize: number ): Promise { const {publicKey, privateKey} = generateX25519KeyPair() const cmd = encodeFGET(encodePubKeyX25519(publicKey)) const {response, body} = await sendXFTPCommand(agent, server, rpKey, fId, cmd) if (response.type !== "FRFile") throw new Error("unexpected response: " + response.type) + if (body.length !== chunkSize + authTagSize) throw new Error("chunk size mismatch") const dhSecret = dh(response.rcvDhKey, privateKey) return {dhSecret, nonce: response.nonce, body} } export async function downloadXFTPChunk( - agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array, digest?: Uint8Array + agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array, chunkSize: number, digest?: Uint8Array ): Promise { - const {dhSecret, nonce, body} = await downloadXFTPChunkRaw(agent, server, rpKey, fId) + const {dhSecret, nonce, body} = await downloadXFTPChunkRaw(agent, server, rpKey, fId, chunkSize) return decryptReceivedChunk(dhSecret, nonce, body, digest ?? null) } diff --git a/xftp-web/src/protocol/chunks.ts b/xftp-web/src/protocol/chunks.ts index fdbac31d7..62f94c287 100644 --- a/xftp-web/src/protocol/chunks.ts +++ b/xftp-web/src/protocol/chunks.ts @@ -3,11 +3,14 @@ // Computes chunk sizes for file uploads, chunk specifications with offsets, // and per-chunk SHA-256 digests. -import {kb, mb} from "./description.js" import {sha256} from "../crypto/digest.js" // -- Chunk size constants (Simplex.FileTransfer.Chunks) +export const kb = (n: number): number => n * 1024 +export const mb = (n: number): number => n * 1048576 +export const gb = (n: number): number => n * 1073741824 + export const chunkSize0 = kb(64) // 65536 export const chunkSize1 = kb(256) // 262144 export const chunkSize2 = mb(1) // 1048576 diff --git a/xftp-web/src/protocol/description.ts b/xftp-web/src/protocol/description.ts index c58151f47..9f00082d5 100644 --- a/xftp-web/src/protocol/description.ts +++ b/xftp-web/src/protocol/description.ts @@ -3,6 +3,8 @@ // Handles YAML-encoded file descriptions matching Haskell Data.Yaml output format. // Base64url encoding matches Haskell Data.ByteString.Base64.URL.encode (with padding). +import {chunkSize3, serverChunkSizes} from "./chunks.js" + // -- Base64url (RFC 4648 section 5) with '=' padding const B64URL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" @@ -63,10 +65,6 @@ export function base64urlDecode(s: string): Uint8Array { // -- FileSize encoding/decoding -export const kb = (n: number): number => n * 1024 -export const mb = (n: number): number => n * 1048576 -export const gb = (n: number): number => n * 1073741824 - export function encodeFileSize(bytes: number): string { const ks = Math.floor(bytes / 1024) if (bytes % 1024 !== 0) return String(bytes) @@ -351,12 +349,15 @@ function parseReplicaField( // -- Validation export function validateFileDescription(fd: FileDescription): string | null { + if (fd.chunks.length === 0) return "no chunks" for (let i = 0; i < fd.chunks.length; i++) { if (fd.chunks[i].chunkNo !== i + 1) return "chunk numbers are not sequential" } let total = 0 for (const c of fd.chunks) total += c.chunkSize if (total !== fd.size) return "chunks total size is different than file size" + if (fd.chunks.some(c => !serverChunkSizes.includes(c.chunkSize))) return "incorrect chunk size" + if (fd.redirect !== null && fd.size > chunkSize3) return "redirect file description is too large" return null } diff --git a/xftp-web/web/download.ts b/xftp-web/web/download.ts index da7d8a38f..acbf6d3a6 100644 --- a/xftp-web/web/download.ts +++ b/xftp-web/web/download.ts @@ -64,7 +64,7 @@ export function initDownload(app: HTMLElement, hash: string) { } function showError(msg: string) { - errorMsg.innerHTML = msg + errorMsg.textContent = msg showStage(errorStage) } diff --git a/xftp-web/web/main.ts b/xftp-web/web/main.ts index 246b45d02..57fd2cd14 100644 --- a/xftp-web/web/main.ts +++ b/xftp-web/web/main.ts @@ -48,7 +48,8 @@ function initApp() { main().catch(err => { const app = getAppElement() if (app) { - app.innerHTML = `

${t('initError', 'Failed to initialize: %error%').replace('%error%', err.message)}

` + app.innerHTML = '

' + app.querySelector('p')!.textContent = t('initError', 'Failed to initialize: %error%').replace('%error%', err.message) } console.error(err) }) diff --git a/xftp-web/web/upload.ts b/xftp-web/web/upload.ts index 78906bb35..632271028 100644 --- a/xftp-web/web/upload.ts +++ b/xftp-web/web/upload.ts @@ -98,7 +98,12 @@ export function initUpload(app: HTMLElement) { } function showError(msg: string) { - errorMsg.innerHTML = msg + errorMsg.textContent = msg + showStage(errorStage) + } + + function showErrorHTML(html: string) { + errorMsg.innerHTML = html showStage(errorStage) } @@ -107,7 +112,7 @@ export function initUpload(app: HTMLElement) { aborted = false if (file.size > MAX_SIZE) { - showError(t('fileTooLarge', 'File too large (%size%). Maximum is 100 MB. The SimpleX app supports files up to 1 GB.').replace('%size%', formatSize(file.size))) + showErrorHTML(t('fileTooLarge', 'File too large (%size%). Maximum is 100 MB. The SimpleX app supports files up to 1 GB.').replace('%size%', formatSize(file.size))) return } if (file.size === 0) {