From 8519a745bd489e2713d5fef482dac749fa4cce26 Mon Sep 17 00:00:00 2001 From: shum Date: Thu, 19 Feb 2026 07:49:07 +0000 Subject: [PATCH] streaming sendFile pipeline, receiveFile, prepareEncryption helper --- xftp-web/README.md | 59 +++++++++++---- xftp-web/src/agent.ts | 136 +++++++++++++++++++++++++++++++--- xftp-web/src/crypto/file.ts | 25 +++++++ xftp-web/web/crypto.worker.ts | 13 +--- 4 files changed, 198 insertions(+), 35 deletions(-) diff --git a/xftp-web/README.md b/xftp-web/README.md index b7154ee71..6e47f09d9 100644 --- a/xftp-web/README.md +++ b/xftp-web/README.md @@ -14,40 +14,73 @@ npm install xftp-web import { newXFTPAgent, closeXFTPAgent, parseXFTPServer, - encryptFileForUpload, uploadFile, downloadFile, deleteFile, - decodeDescriptionURI, encodeDescriptionURI, + sendFile, receiveFile, deleteFile, XFTPRetriableError, XFTPPermanentError, isRetriable, } from "xftp-web" // Create agent (manages connections) const agent = newXFTPAgent() -// Upload — chunks are distributed randomly across servers const servers = [ parseXFTPServer("xftp://server1..."), parseXFTPServer("xftp://server2..."), parseXFTPServer("xftp://server3..."), ] -const encrypted = await encryptFileForUpload(fileBytes, "photo.jpg") -const {rcvDescriptions, sndDescription, uri} = await uploadFile(agent, servers, encrypted, { - onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`), -}) -// Download (from URI or FileDescription) -const fd = decodeDescriptionURI(uri) -const {header, content} = await downloadFile(agent, fd) +// Upload (from Uint8Array) +const {rcvDescriptions, sndDescription, uri} = await sendFile( + agent, servers, fileBytes, "photo.jpg", + {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)} +) -// Delete (using sender description) +// Upload (streaming — constant memory, no full-file buffer) +const file = inputEl.files[0] +const result = await sendFile( + agent, servers, file.stream(), file.size, file.name, + {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)} +) + +// Download +const {header, content} = await receiveFile(agent, uri) + +// Delete await deleteFile(agent, sndDescription) // Cleanup closeXFTPAgent(agent) ``` +### Advanced usage + +For streaming encryption (avoids buffering the full encrypted file) or worker-based uploads: + +```typescript +import { + encryptFileForUpload, uploadFile, downloadFile, + decodeDescriptionURI, +} from "xftp-web" + +// Streaming encryption — encrypted slices emitted via callback +const metadata = await encryptFileForUpload(fileBytes, "photo.jpg", { + onSlice: (data) => { /* write to OPFS, IndexedDB, etc. */ }, + onProgress: (done, total) => {}, +}) +// metadata has {digest, key, nonce, chunkSizes} but no encData + +// Upload with custom chunk reader (e.g. reading from OPFS) +const result = await uploadFile(agent, servers, metadata, { + readChunk: (offset, size) => readFromStorage(offset, size), +}) + +// Download with FileDescription object +const fd = decodeDescriptionURI(uri) +const {header, content} = await downloadFile(agent, fd) +``` + ### Upload options ```typescript -await uploadFile(agent, servers, encrypted, { +await sendFile(agent, servers, fileBytes, "photo.jpg", { onProgress: (uploaded, total) => {}, // progress callback auth: basicAuthBytes, // BasicAuth for auth-required servers numRecipients: 3, // multiple independent download credentials (default: 1) @@ -58,7 +91,7 @@ await uploadFile(agent, servers, encrypted, { ```typescript try { - await uploadFile(agent, servers, encrypted) + await sendFile(agent, servers, fileBytes, "photo.jpg") } catch (e) { if (e instanceof XFTPRetriableError) { // Network/timeout/session errors — safe to retry diff --git a/xftp-web/src/agent.ts b/xftp-web/src/agent.ts index df0295c0d..139c67767 100644 --- a/xftp-web/src/agent.ts +++ b/xftp-web/src/agent.ts @@ -4,10 +4,13 @@ // file descriptions, and DEFLATE-compressed URI encoding. import pako from "pako" -import {encryptFileAsync, encodeFileHeader} from "./crypto/file.js" +import {encryptFileAsync, prepareEncryption} from "./crypto/file.js" +import {sbInit, sbEncryptChunk, sbAuth} from "./crypto/secretbox.js" +import {concatBytes, encodeInt64} from "./protocol/encoding.js" +export {prepareEncryption, type EncryptionParams} from "./crypto/file.js" import {generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed} from "./crypto/keys.js" import {sha512Streaming, sha512Init, sha512Update, sha512Final} from "./crypto/digest.js" -import {prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize} from "./protocol/chunks.js" +import {prepareChunkSpecs, getChunkDigest} from "./protocol/chunks.js" import { encodeFileDescription, decodeFileDescription, validateFileDescription, base64urlEncode, base64urlDecode, @@ -98,15 +101,7 @@ export async function encryptFileForUpload( options?: EncryptForUploadOptions ): Promise { const {onProgress, onSlice} = options ?? {} - const key = new Uint8Array(32) - const nonce = new Uint8Array(24) - crypto.getRandomValues(key) - crypto.getRandomValues(nonce) - const fileHdr = encodeFileHeader({fileName, fileExtra: null}) - const fileSize = BigInt(fileHdr.length + source.length) - const payloadSize = Number(fileSize) + fileSizeLen + authTagSize - const chunkSizes = prepareChunkSizes(payloadSize) - const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) + const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName) if (onSlice) { const hashState = sha512Init() await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => { @@ -228,6 +223,116 @@ export async function uploadFile( return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri} } +export interface SendFileOptions { + onProgress?: (uploaded: number, total: number) => void + auth?: Uint8Array + numRecipients?: number +} + +export async function sendFile( + agent: XFTPClientAgent, servers: XFTPServer[], + source: Uint8Array, fileName: string, + options?: SendFileOptions +): Promise +export async function sendFile( + agent: XFTPClientAgent, servers: XFTPServer[], + source: AsyncIterable, sourceSize: number, + fileName: string, options?: SendFileOptions +): Promise +export async function sendFile( + agent: XFTPClientAgent, servers: XFTPServer[], + source: Uint8Array | AsyncIterable, + fileNameOrSize: string | number, + fileNameOrOptions?: string | SendFileOptions, + maybeOptions?: SendFileOptions +): Promise { + let sourceSize: number, fileName: string, options: SendFileOptions | undefined + if (source instanceof Uint8Array) { + sourceSize = source.length + fileName = fileNameOrSize as string + options = fileNameOrOptions as SendFileOptions | undefined + } else { + sourceSize = fileNameOrSize as number + fileName = fileNameOrOptions as string + options = maybeOptions + } + if (servers.length === 0) throw new Error("sendFile: servers list is empty") + const {onProgress, auth, numRecipients = 1} = options ?? {} + const params = prepareEncryption(sourceSize, fileName) + const specs = prepareChunkSpecs(params.chunkSizes) + const total = params.chunkSizes.reduce((a, b) => a + b, 0) + + const encState = sbInit(params.key, params.nonce) + const hashState = sha512Init() + + const sentChunks: SentChunk[] = new Array(specs.length) + let specIdx = 0, chunkOff = 0, uploaded = 0 + let chunkBuf = new Uint8Array(specs[0].chunkSize) + + async function flushChunk() { + const server = servers[Math.floor(Math.random() * servers.length)] + sentChunks[specIdx] = await uploadSingleChunk( + agent, server, specIdx + 1, chunkBuf, specs[specIdx].chunkSize, numRecipients, auth ?? null + ) + uploaded += specs[specIdx].chunkSize + onProgress?.(uploaded, total) + specIdx++ + if (specIdx < specs.length) { + chunkBuf = new Uint8Array(specs[specIdx].chunkSize) + chunkOff = 0 + } + } + + async function feedEncrypted(data: Uint8Array) { + sha512Update(hashState, data) + let off = 0 + while (off < data.length) { + const space = specs[specIdx].chunkSize - chunkOff + const n = Math.min(space, data.length - off) + chunkBuf.set(data.subarray(off, off + n), chunkOff) + chunkOff += n + off += n + if (chunkOff === specs[specIdx].chunkSize) await flushChunk() + } + } + + await feedEncrypted(sbEncryptChunk(encState, concatBytes(encodeInt64(params.fileSize), params.fileHdr))) + + const SLICE = 65536 + if (source instanceof Uint8Array) { + for (let off = 0; off < source.length; off += SLICE) { + await feedEncrypted(sbEncryptChunk(encState, source.subarray(off, Math.min(off + SLICE, source.length)))) + } + } else { + for await (const chunk of source) { + for (let off = 0; off < chunk.length; off += SLICE) { + await feedEncrypted(sbEncryptChunk(encState, chunk.subarray(off, Math.min(off + SLICE, chunk.length)))) + } + } + } + + const padLen = Number(params.encSize - 16n - params.fileSize - 8n) + const padding = new Uint8Array(padLen) + padding.fill(0x23) + await feedEncrypted(sbEncryptChunk(encState, padding)) + await feedEncrypted(sbAuth(encState)) + + const digest = sha512Final(hashState) + const encrypted: EncryptedFileMetadata = {digest, key: params.key, nonce: params.nonce, chunkSizes: params.chunkSizes} + const rcvDescriptions = Array.from({length: numRecipients}, (_, ri) => + buildDescription("recipient", ri, encrypted, sentChunks) + ) + const sndDescription = buildDescription("sender", 0, encrypted, sentChunks) + let uri = encodeDescriptionURI(rcvDescriptions[0]) + let finalRcvDescriptions = rcvDescriptions + if (uri.length > DEFAULT_REDIRECT_THRESHOLD && sentChunks.length > 1) { + const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth) + finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)] + uri = encodeDescriptionURI(redirected) + } + return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri} +} + function buildDescription( party: "recipient" | "sender", recipientIndex: number, @@ -389,6 +494,15 @@ export async function downloadFile( return processDownloadedFile(resolvedFd, chunks) } +export async function receiveFile( + agent: XFTPClientAgent, + uri: string, + options?: {onProgress?: (downloaded: number, total: number) => void} +): Promise { + const fd = decodeDescriptionURI(uri) + return downloadFile(agent, fd, options?.onProgress) +} + async function resolveRedirect( agent: XFTPClientAgent, fd: FileDescription diff --git a/xftp-web/src/crypto/file.ts b/xftp-web/src/crypto/file.ts index cdc7184c7..3ae215cea 100644 --- a/xftp-web/src/crypto/file.ts +++ b/xftp-web/src/crypto/file.ts @@ -3,6 +3,7 @@ import {Decoder, concatBytes, encodeInt64, encodeString, decodeString, encodeMaybe, decodeMaybe} from "../protocol/encoding.js" import {sbInit, sbEncryptChunk, sbDecryptTailTag, sbAuth} from "./secretbox.js" +import {prepareChunkSizes, fileSizeLen, authTagSize} from "../protocol/chunks.js" const AUTH_TAG_SIZE = 16n @@ -127,6 +128,30 @@ export async function encryptFileAsync( if (out) return out } +// -- Encryption preparation (key gen + chunk sizing) + +export interface EncryptionParams { + fileHdr: Uint8Array + key: Uint8Array + nonce: Uint8Array + fileSize: bigint + encSize: bigint + chunkSizes: number[] +} + +export function prepareEncryption(sourceSize: number, fileName: string): EncryptionParams { + const key = new Uint8Array(32) + const nonce = new Uint8Array(24) + crypto.getRandomValues(key) + crypto.getRandomValues(nonce) + const fileHdr = encodeFileHeader({fileName, fileExtra: null}) + const fileSize = BigInt(fileHdr.length + sourceSize) + const payloadSize = Number(fileSize) + fileSizeLen + authTagSize + const chunkSizes = prepareChunkSizes(payloadSize) + const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) + return {fileHdr, key, nonce, fileSize, encSize, chunkSizes} +} + // -- Decryption (FileTransfer.Crypto:decryptChunks) // Decrypt one or more XFTP chunks into a FileHeader and file content. diff --git a/xftp-web/web/crypto.worker.ts b/xftp-web/web/crypto.worker.ts index b9bc7e705..39c2ba580 100644 --- a/xftp-web/web/crypto.worker.ts +++ b/xftp-web/web/crypto.worker.ts @@ -1,7 +1,6 @@ import sodium from 'libsodium-wrappers-sumo' -import {encryptFile, encodeFileHeader, decryptChunks} from '../src/crypto/file.js' +import {encryptFile, prepareEncryption, decryptChunks} from '../src/crypto/file.js' import {sha512Streaming} from '../src/crypto/digest.js' -import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js' import {decryptReceivedChunk} from '../src/download.js' // ── OPFS session management ───────────────────────────────────── @@ -40,15 +39,7 @@ async function sweepStale() { async function handleEncrypt(id: number, data: ArrayBuffer, fileName: string) { const source = new Uint8Array(data) - const key = new Uint8Array(32) - const nonce = new Uint8Array(24) - crypto.getRandomValues(key) - crypto.getRandomValues(nonce) - const fileHdr = encodeFileHeader({fileName, fileExtra: null}) - const fileSize = BigInt(fileHdr.length + source.length) - const payloadSize = Number(fileSize) + fileSizeLen + authTagSize - const chunkSizes = prepareChunkSizes(payloadSize) - const encSize = BigInt(chunkSizes.reduce((a: number, b: number) => a + b, 0)) + const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName) const encData = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) self.postMessage({id, type: 'progress', done: 50, total: 100})