streaming encryption API: onSlice callback for memory-efficient large file encryption

xftp-web/src/agent.ts

@@ -6,7 +6,7 @@
 import pako from "pako"
 import {encryptFileAsync, encodeFileHeader} from "./crypto/file.js"
 import {generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed} from "./crypto/keys.js"
-import {sha512Streaming} from "./crypto/digest.js"
+import {sha512Streaming, sha512Init, sha512Update, sha512Final} from "./crypto/digest.js"
 import {prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize} from "./protocol/chunks.js"
 import {
   encodeFileDescription, decodeFileDescription, validateFileDescription,
@@ -79,11 +79,25 @@ export function decodeDescriptionURI(fragment: string): FileDescription {
 
 // -- Upload
 
+export interface EncryptForUploadOptions {
+  onProgress?: (done: number, total: number) => void
+  onSlice?: (data: Uint8Array) => void | Promise<void>
+}
+
+export async function encryptFileForUpload(
+  source: Uint8Array, fileName: string,
+  options: EncryptForUploadOptions & {onSlice: NonNullable<EncryptForUploadOptions['onSlice']>}
+): Promise<EncryptedFileMetadata>
+export async function encryptFileForUpload(
+  source: Uint8Array, fileName: string,
+  options?: EncryptForUploadOptions
+): Promise<EncryptedFileInfo>
 export async function encryptFileForUpload(
   source: Uint8Array,
   fileName: string,
-  onProgress?: (done: number, total: number) => void
-): Promise<EncryptedFileInfo> {
+  options?: EncryptForUploadOptions
+): Promise<EncryptedFileInfo | EncryptedFileMetadata> {
+  const {onProgress, onSlice} = options ?? {}
   const key = new Uint8Array(32)
   const nonce = new Uint8Array(24)
   crypto.getRandomValues(key)
@@ -93,10 +107,20 @@ export async function encryptFileForUpload(
   const payloadSize = Number(fileSize) + fileSizeLen + authTagSize
   const chunkSizes = prepareChunkSizes(payloadSize)
   const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0))
-  const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress)
-  const digest = sha512Streaming([encData])
-  console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`)
-  return {encData, digest, key, nonce, chunkSizes}
+  if (onSlice) {
+    const hashState = sha512Init()
+    await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => {
+      sha512Update(hashState, data)
+      return onSlice(data)
+    })
+    const digest = sha512Final(hashState)
+    return {digest, key, nonce, chunkSizes}
+  } else {
+    const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress)
+    const digest = sha512Streaming([encData])
+    console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`)
+    return {encData, digest, key, nonce, chunkSizes}
+  }
 }
 
 const DEFAULT_REDIRECT_THRESHOLD = 400

xftp-web/src/crypto/digest.ts

@@ -12,6 +12,19 @@ export function sha512(data: Uint8Array): Uint8Array {
   return sodium.crypto_hash_sha512(data)
 }
 
+// Incremental SHA-512 — for computing digest during streaming encryption.
+export function sha512Init(): StateAddress {
+  return sodium.crypto_hash_sha512_init() as unknown as StateAddress
+}
+
+export function sha512Update(state: StateAddress, data: Uint8Array): void {
+  sodium.crypto_hash_sha512_update(state, data)
+}
+
+export function sha512Final(state: StateAddress): Uint8Array {
+  return sodium.crypto_hash_sha512_final(state)
+}
+
 // Streaming SHA-512 over multiple chunks -- avoids copying large data into WASM memory at once.
 // Internally segments chunks larger than 4MB to limit peak WASM memory usage.
 export function sha512Streaming(chunks: Iterable<Uint8Array>): Uint8Array {

xftp-web/src/crypto/file.ts

@@ -71,8 +71,22 @@ export function encryptFile(
 
 // Async variant: encrypts source in 64KB slices, yielding between each to avoid blocking the main thread.
 // Produces identical output to encryptFile.
+// When onSlice is provided, encrypted data is streamed to the callback instead of buffered.
 const ENCRYPT_SLICE = 65536
 
+export async function encryptFileAsync(
+  source: Uint8Array, fileHdr: Uint8Array,
+  key: Uint8Array, nonce: Uint8Array,
+  fileSize: bigint, encSize: bigint,
+  onProgress?: (done: number, total: number) => void
+): Promise<Uint8Array>
+export async function encryptFileAsync(
+  source: Uint8Array, fileHdr: Uint8Array,
+  key: Uint8Array, nonce: Uint8Array,
+  fileSize: bigint, encSize: bigint,
+  onProgress: ((done: number, total: number) => void) | undefined,
+  onSlice: (data: Uint8Array) => void | Promise<void>
+): Promise<void>
 export async function encryptFileAsync(
   source: Uint8Array,
   fileHdr: Uint8Array,
@@ -80,35 +94,37 @@ export async function encryptFileAsync(
   nonce: Uint8Array,
   fileSize: bigint,
   encSize: bigint,
-  onProgress?: (done: number, total: number) => void
-): Promise<Uint8Array> {
+  onProgress?: (done: number, total: number) => void,
+  onSlice?: (data: Uint8Array) => void | Promise<void>
+): Promise<Uint8Array | void> {
   const state = sbInit(key, nonce)
   const lenStr = encodeInt64(fileSize)
   const padLen = Number(encSize - AUTH_TAG_SIZE - fileSize - 8n)
   if (padLen < 0) throw new Error("encryptFile: encSize too small")
-  const totalOut = Number(encSize)
-  const out = new Uint8Array(totalOut)
+  const out = onSlice ? null : new Uint8Array(Number(encSize))
   let outOff = 0
-  // Header (small, no yield needed)
-  const hdr = sbEncryptChunk(state, concatBytes(lenStr, fileHdr))
-  out.set(hdr, outOff); outOff += hdr.length
-  // Source in 64KB slices, yielding between each
+
+  async function emit(data: Uint8Array) {
+    if (onSlice) {
+      await onSlice(data)
+    } else {
+      out!.set(data, outOff)
+      outOff += data.length
+    }
+  }
+
+  await emit(sbEncryptChunk(state, concatBytes(lenStr, fileHdr)))
   for (let off = 0; off < source.length; off += ENCRYPT_SLICE) {
     const end = Math.min(off + ENCRYPT_SLICE, source.length)
-    const enc = sbEncryptChunk(state, source.subarray(off, end))
-    out.set(enc, outOff); outOff += enc.length
+    await emit(sbEncryptChunk(state, source.subarray(off, end)))
     onProgress?.(end, source.length)
     await new Promise<void>(r => setTimeout(r, 0))
   }
-  // Padding (small, no yield needed)
   const padding = new Uint8Array(padLen)
   padding.fill(0x23)
-  const encPad = sbEncryptChunk(state, padding)
-  out.set(encPad, outOff); outOff += encPad.length
-  // Auth tag
-  const tag = sbAuth(state)
-  out.set(tag, outOff)
-  return out
+  await emit(sbEncryptChunk(state, padding))
+  await emit(sbAuth(state))
+  if (out) return out
 }
 
 // -- Decryption (FileTransfer.Crypto:decryptChunks)

xftp-web/test/crypto.node.test.ts

@@ -2,6 +2,8 @@ import {test, expect} from 'vitest'
 import sodium from 'libsodium-wrappers-sumo'
 import {encryptFile, encryptFileAsync, encodeFileHeader} from '../src/crypto/file.js'
 import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js'
+import {sha512Streaming} from '../src/crypto/digest.js'
+import {encryptFileForUpload} from '../src/agent.js'
 
 await sodium.ready
 
@@ -71,3 +73,30 @@ test('encryptFileAsync matches sync for empty source', async () => {
   const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize)
   expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult))
 })
+
+test('encryptFileAsync streaming produces identical output to buffered', async () => {
+  const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(256 * 1024)
+  const slices: Uint8Array[] = []
+  await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, undefined, (data) => {
+    slices.push(data)
+  })
+  const streamed = Buffer.concat(slices)
+  const buffered = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize)
+  expect(streamed).toEqual(Buffer.from(buffered))
+})
+
+test('encryptFileForUpload streaming digest matches slice data', async () => {
+  const source = new Uint8Array(256 * 1024)
+  fillRandom(source)
+  const slices: Uint8Array[] = []
+  const result = await encryptFileForUpload(source, 'test.bin', {
+    onSlice: (data) => { slices.push(data) }
+  })
+  const combined = Buffer.concat(slices)
+  const actualDigest = sha512Streaming([combined])
+  expect(Buffer.from(result.digest)).toEqual(Buffer.from(actualDigest))
+  expect(result.key.length).toBe(32)
+  expect(result.nonce.length).toBe(24)
+  expect(result.chunkSizes.length).toBeGreaterThan(0)
+  expect('encData' in result).toBe(false)
+})