diff --git a/xftp-web/src/agent.ts b/xftp-web/src/agent.ts index 8704065a1..32b08fbb3 100644 --- a/xftp-web/src/agent.ts +++ b/xftp-web/src/agent.ts @@ -6,7 +6,7 @@ import pako from "pako" import {encryptFileAsync, encodeFileHeader} from "./crypto/file.js" import {generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed} from "./crypto/keys.js" -import {sha512Streaming} from "./crypto/digest.js" +import {sha512Streaming, sha512Init, sha512Update, sha512Final} from "./crypto/digest.js" import {prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize} from "./protocol/chunks.js" import { encodeFileDescription, decodeFileDescription, validateFileDescription, @@ -79,11 +79,25 @@ export function decodeDescriptionURI(fragment: string): FileDescription { // -- Upload +export interface EncryptForUploadOptions { + onProgress?: (done: number, total: number) => void + onSlice?: (data: Uint8Array) => void | Promise +} + +export async function encryptFileForUpload( + source: Uint8Array, fileName: string, + options: EncryptForUploadOptions & {onSlice: NonNullable} +): Promise +export async function encryptFileForUpload( + source: Uint8Array, fileName: string, + options?: EncryptForUploadOptions +): Promise export async function encryptFileForUpload( source: Uint8Array, fileName: string, - onProgress?: (done: number, total: number) => void -): Promise { + options?: EncryptForUploadOptions +): Promise { + const {onProgress, onSlice} = options ?? {} const key = new Uint8Array(32) const nonce = new Uint8Array(24) crypto.getRandomValues(key) @@ -93,10 +107,20 @@ export async function encryptFileForUpload( const payloadSize = Number(fileSize) + fileSizeLen + authTagSize const chunkSizes = prepareChunkSizes(payloadSize) const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) - const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress) - const digest = sha512Streaming([encData]) - console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`) - return {encData, digest, key, nonce, chunkSizes} + if (onSlice) { + const hashState = sha512Init() + await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => { + sha512Update(hashState, data) + return onSlice(data) + }) + const digest = sha512Final(hashState) + return {digest, key, nonce, chunkSizes} + } else { + const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress) + const digest = sha512Streaming([encData]) + console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`) + return {encData, digest, key, nonce, chunkSizes} + } } const DEFAULT_REDIRECT_THRESHOLD = 400 diff --git a/xftp-web/src/crypto/digest.ts b/xftp-web/src/crypto/digest.ts index bc376415d..78702afbb 100644 --- a/xftp-web/src/crypto/digest.ts +++ b/xftp-web/src/crypto/digest.ts @@ -12,6 +12,19 @@ export function sha512(data: Uint8Array): Uint8Array { return sodium.crypto_hash_sha512(data) } +// Incremental SHA-512 — for computing digest during streaming encryption. +export function sha512Init(): StateAddress { + return sodium.crypto_hash_sha512_init() as unknown as StateAddress +} + +export function sha512Update(state: StateAddress, data: Uint8Array): void { + sodium.crypto_hash_sha512_update(state, data) +} + +export function sha512Final(state: StateAddress): Uint8Array { + return sodium.crypto_hash_sha512_final(state) +} + // Streaming SHA-512 over multiple chunks -- avoids copying large data into WASM memory at once. // Internally segments chunks larger than 4MB to limit peak WASM memory usage. export function sha512Streaming(chunks: Iterable): Uint8Array { diff --git a/xftp-web/src/crypto/file.ts b/xftp-web/src/crypto/file.ts index a6468a3f7..cdc7184c7 100644 --- a/xftp-web/src/crypto/file.ts +++ b/xftp-web/src/crypto/file.ts @@ -71,8 +71,22 @@ export function encryptFile( // Async variant: encrypts source in 64KB slices, yielding between each to avoid blocking the main thread. // Produces identical output to encryptFile. +// When onSlice is provided, encrypted data is streamed to the callback instead of buffered. const ENCRYPT_SLICE = 65536 +export async function encryptFileAsync( + source: Uint8Array, fileHdr: Uint8Array, + key: Uint8Array, nonce: Uint8Array, + fileSize: bigint, encSize: bigint, + onProgress?: (done: number, total: number) => void +): Promise +export async function encryptFileAsync( + source: Uint8Array, fileHdr: Uint8Array, + key: Uint8Array, nonce: Uint8Array, + fileSize: bigint, encSize: bigint, + onProgress: ((done: number, total: number) => void) | undefined, + onSlice: (data: Uint8Array) => void | Promise +): Promise export async function encryptFileAsync( source: Uint8Array, fileHdr: Uint8Array, @@ -80,35 +94,37 @@ export async function encryptFileAsync( nonce: Uint8Array, fileSize: bigint, encSize: bigint, - onProgress?: (done: number, total: number) => void -): Promise { + onProgress?: (done: number, total: number) => void, + onSlice?: (data: Uint8Array) => void | Promise +): Promise { const state = sbInit(key, nonce) const lenStr = encodeInt64(fileSize) const padLen = Number(encSize - AUTH_TAG_SIZE - fileSize - 8n) if (padLen < 0) throw new Error("encryptFile: encSize too small") - const totalOut = Number(encSize) - const out = new Uint8Array(totalOut) + const out = onSlice ? null : new Uint8Array(Number(encSize)) let outOff = 0 - // Header (small, no yield needed) - const hdr = sbEncryptChunk(state, concatBytes(lenStr, fileHdr)) - out.set(hdr, outOff); outOff += hdr.length - // Source in 64KB slices, yielding between each + + async function emit(data: Uint8Array) { + if (onSlice) { + await onSlice(data) + } else { + out!.set(data, outOff) + outOff += data.length + } + } + + await emit(sbEncryptChunk(state, concatBytes(lenStr, fileHdr))) for (let off = 0; off < source.length; off += ENCRYPT_SLICE) { const end = Math.min(off + ENCRYPT_SLICE, source.length) - const enc = sbEncryptChunk(state, source.subarray(off, end)) - out.set(enc, outOff); outOff += enc.length + await emit(sbEncryptChunk(state, source.subarray(off, end))) onProgress?.(end, source.length) await new Promise(r => setTimeout(r, 0)) } - // Padding (small, no yield needed) const padding = new Uint8Array(padLen) padding.fill(0x23) - const encPad = sbEncryptChunk(state, padding) - out.set(encPad, outOff); outOff += encPad.length - // Auth tag - const tag = sbAuth(state) - out.set(tag, outOff) - return out + await emit(sbEncryptChunk(state, padding)) + await emit(sbAuth(state)) + if (out) return out } // -- Decryption (FileTransfer.Crypto:decryptChunks) diff --git a/xftp-web/test/crypto.node.test.ts b/xftp-web/test/crypto.node.test.ts index 088f0ce0e..78f6567ff 100644 --- a/xftp-web/test/crypto.node.test.ts +++ b/xftp-web/test/crypto.node.test.ts @@ -2,6 +2,8 @@ import {test, expect} from 'vitest' import sodium from 'libsodium-wrappers-sumo' import {encryptFile, encryptFileAsync, encodeFileHeader} from '../src/crypto/file.js' import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js' +import {sha512Streaming} from '../src/crypto/digest.js' +import {encryptFileForUpload} from '../src/agent.js' await sodium.ready @@ -71,3 +73,30 @@ test('encryptFileAsync matches sync for empty source', async () => { const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) }) + +test('encryptFileAsync streaming produces identical output to buffered', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(256 * 1024) + const slices: Uint8Array[] = [] + await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, undefined, (data) => { + slices.push(data) + }) + const streamed = Buffer.concat(slices) + const buffered = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(streamed).toEqual(Buffer.from(buffered)) +}) + +test('encryptFileForUpload streaming digest matches slice data', async () => { + const source = new Uint8Array(256 * 1024) + fillRandom(source) + const slices: Uint8Array[] = [] + const result = await encryptFileForUpload(source, 'test.bin', { + onSlice: (data) => { slices.push(data) } + }) + const combined = Buffer.concat(slices) + const actualDigest = sha512Streaming([combined]) + expect(Buffer.from(result.digest)).toEqual(Buffer.from(actualDigest)) + expect(result.key.length).toBe(32) + expect(result.nonce.length).toBe(24) + expect(result.chunkSizes.length).toBeGreaterThan(0) + expect('encData' in result).toBe(false) +})