dandri/simplexmq
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplexmq.git synced 2026-06-07 08:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

streaming sendFile pipeline, receiveFile, prepareEncryption helper

Browse Source
This commit is contained in:
shum
2026-02-19 07:49:07 +00:00
parent eb759ee3ea
commit 8519a745bd
4 changed files with 198 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xftp-web/README.md
+46 -13
View File
@@ -14,40 +14,73 @@ npm install xftp-web
import {
newXFTPAgent, closeXFTPAgent,
parseXFTPServer,
encryptFileForUpload, uploadFile, downloadFile, deleteFile,
decodeDescriptionURI, encodeDescriptionURI,
sendFile, receiveFile, deleteFile,
XFTPRetriableError, XFTPPermanentError, isRetriable,
} from "xftp-web"
// Create agent (manages connections)
const agent = newXFTPAgent()
// Upload — chunks are distributed randomly across servers
const servers = [
parseXFTPServer("xftp://server1..."),
parseXFTPServer("xftp://server2..."),
parseXFTPServer("xftp://server3..."),
]
const encrypted = await encryptFileForUpload(fileBytes, "photo.jpg")
const {rcvDescriptions, sndDescription, uri} = await uploadFile(agent, servers, encrypted, {
onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`),
})
// Download (from URI or FileDescription)
const fd = decodeDescriptionURI(uri)
const {header, content} = await downloadFile(agent, fd)
// Upload (from Uint8Array)
const {rcvDescriptions, sndDescription, uri} = await sendFile(
agent, servers, fileBytes, "photo.jpg",
{onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)}
)
// Delete (using sender description)
// Upload (streaming — constant memory, no full-file buffer)
const file = inputEl.files[0]
const result = await sendFile(
agent, servers, file.stream(), file.size, file.name,
{onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)}
)
// Download
const {header, content} = await receiveFile(agent, uri)
// Delete
await deleteFile(agent, sndDescription)
// Cleanup
closeXFTPAgent(agent)
```
### Advanced usage
For streaming encryption (avoids buffering the full encrypted file) or worker-based uploads:
```typescript
import {
encryptFileForUpload, uploadFile, downloadFile,
decodeDescriptionURI,
} from "xftp-web"
// Streaming encryption — encrypted slices emitted via callback
const metadata = await encryptFileForUpload(fileBytes, "photo.jpg", {
onSlice: (data) => { /* write to OPFS, IndexedDB, etc. */ },
onProgress: (done, total) => {},
})
// metadata has {digest, key, nonce, chunkSizes} but no encData
// Upload with custom chunk reader (e.g. reading from OPFS)
const result = await uploadFile(agent, servers, metadata, {
readChunk: (offset, size) => readFromStorage(offset, size),
})
// Download with FileDescription object
const fd = decodeDescriptionURI(uri)
const {header, content} = await downloadFile(agent, fd)
```
### Upload options
```typescript
await uploadFile(agent, servers, encrypted, {
await sendFile(agent, servers, fileBytes, "photo.jpg", {
onProgress: (uploaded, total) => {}, // progress callback
auth: basicAuthBytes, // BasicAuth for auth-required servers
numRecipients: 3, // multiple independent download credentials (default: 1)
@@ -58,7 +91,7 @@ await uploadFile(agent, servers, encrypted, {
```typescript
try {
await uploadFile(agent, servers, encrypted)
await sendFile(agent, servers, fileBytes, "photo.jpg")
} catch (e) {
if (e instanceof XFTPRetriableError) {
// Network/timeout/session errors — safe to retry
xftp-web/src/agent.ts
+125 -11
View File
@@ -4,10 +4,13 @@
// file descriptions, and DEFLATE-compressed URI encoding.
import pako from "pako"
import {encryptFileAsync, encodeFileHeader} from "./crypto/file.js"
import {encryptFileAsync, prepareEncryption} from "./crypto/file.js"
import {sbInit, sbEncryptChunk, sbAuth} from "./crypto/secretbox.js"
import {concatBytes, encodeInt64} from "./protocol/encoding.js"
export {prepareEncryption, type EncryptionParams} from "./crypto/file.js"
import {generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed} from "./crypto/keys.js"
import {sha512Streaming, sha512Init, sha512Update, sha512Final} from "./crypto/digest.js"
import {prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize} from "./protocol/chunks.js"
import {prepareChunkSpecs, getChunkDigest} from "./protocol/chunks.js"
import {
encodeFileDescription, decodeFileDescription, validateFileDescription,
base64urlEncode, base64urlDecode,
@@ -98,15 +101,7 @@ export async function encryptFileForUpload(
options?: EncryptForUploadOptions
): Promise<EncryptedFileInfo | EncryptedFileMetadata> {
const {onProgress, onSlice} = options ?? {}
const key = new Uint8Array(32)
const nonce = new Uint8Array(24)
crypto.getRandomValues(key)
crypto.getRandomValues(nonce)
const fileHdr = encodeFileHeader({fileName, fileExtra: null})
const fileSize = BigInt(fileHdr.length + source.length)
const payloadSize = Number(fileSize) + fileSizeLen + authTagSize
const chunkSizes = prepareChunkSizes(payloadSize)
const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0))
const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName)
if (onSlice) {
const hashState = sha512Init()
await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => {
@@ -228,6 +223,116 @@ export async function uploadFile(
return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri}
}
export interface SendFileOptions {
onProgress?: (uploaded: number, total: number) => void
auth?: Uint8Array
numRecipients?: number
}
export async function sendFile(
agent: XFTPClientAgent, servers: XFTPServer[],
source: Uint8Array, fileName: string,
options?: SendFileOptions
): Promise<UploadResult>
export async function sendFile(
agent: XFTPClientAgent, servers: XFTPServer[],
source: AsyncIterable<Uint8Array>, sourceSize: number,
fileName: string, options?: SendFileOptions
): Promise<UploadResult>
export async function sendFile(
agent: XFTPClientAgent, servers: XFTPServer[],
source: Uint8Array | AsyncIterable<Uint8Array>,
fileNameOrSize: string | number,
fileNameOrOptions?: string | SendFileOptions,
maybeOptions?: SendFileOptions
): Promise<UploadResult> {
let sourceSize: number, fileName: string, options: SendFileOptions | undefined
if (source instanceof Uint8Array) {
sourceSize = source.length
fileName = fileNameOrSize as string
options = fileNameOrOptions as SendFileOptions | undefined
} else {
sourceSize = fileNameOrSize as number
fileName = fileNameOrOptions as string
options = maybeOptions
}
if (servers.length === 0) throw new Error("sendFile: servers list is empty")
const {onProgress, auth, numRecipients = 1} = options ?? {}
const params = prepareEncryption(sourceSize, fileName)
const specs = prepareChunkSpecs(params.chunkSizes)
const total = params.chunkSizes.reduce((a, b) => a + b, 0)
const encState = sbInit(params.key, params.nonce)
const hashState = sha512Init()
const sentChunks: SentChunk[] = new Array(specs.length)
let specIdx = 0, chunkOff = 0, uploaded = 0
let chunkBuf = new Uint8Array(specs[0].chunkSize)
async function flushChunk() {
const server = servers[Math.floor(Math.random() * servers.length)]
sentChunks[specIdx] = await uploadSingleChunk(
agent, server, specIdx + 1, chunkBuf, specs[specIdx].chunkSize, numRecipients, auth ?? null
)
uploaded += specs[specIdx].chunkSize
onProgress?.(uploaded, total)
specIdx++
if (specIdx < specs.length) {
chunkBuf = new Uint8Array(specs[specIdx].chunkSize)
chunkOff = 0
}
}
async function feedEncrypted(data: Uint8Array) {
sha512Update(hashState, data)
let off = 0
while (off < data.length) {
const space = specs[specIdx].chunkSize - chunkOff
const n = Math.min(space, data.length - off)
chunkBuf.set(data.subarray(off, off + n), chunkOff)
chunkOff += n
off += n
if (chunkOff === specs[specIdx].chunkSize) await flushChunk()
}
}
await feedEncrypted(sbEncryptChunk(encState, concatBytes(encodeInt64(params.fileSize), params.fileHdr)))
const SLICE = 65536
if (source instanceof Uint8Array) {
for (let off = 0; off < source.length; off += SLICE) {
await feedEncrypted(sbEncryptChunk(encState, source.subarray(off, Math.min(off + SLICE, source.length))))
}
} else {
for await (const chunk of source) {
for (let off = 0; off < chunk.length; off += SLICE) {
await feedEncrypted(sbEncryptChunk(encState, chunk.subarray(off, Math.min(off + SLICE, chunk.length))))
}
}
}
const padLen = Number(params.encSize - 16n - params.fileSize - 8n)
const padding = new Uint8Array(padLen)
padding.fill(0x23)
await feedEncrypted(sbEncryptChunk(encState, padding))
await feedEncrypted(sbAuth(encState))
const digest = sha512Final(hashState)
const encrypted: EncryptedFileMetadata = {digest, key: params.key, nonce: params.nonce, chunkSizes: params.chunkSizes}
const rcvDescriptions = Array.from({length: numRecipients}, (_, ri) =>
buildDescription("recipient", ri, encrypted, sentChunks)
)
const sndDescription = buildDescription("sender", 0, encrypted, sentChunks)
let uri = encodeDescriptionURI(rcvDescriptions[0])
let finalRcvDescriptions = rcvDescriptions
if (uri.length > DEFAULT_REDIRECT_THRESHOLD && sentChunks.length > 1) {
const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth)
finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)]
uri = encodeDescriptionURI(redirected)
}
return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri}
}
function buildDescription(
party: "recipient" | "sender",
recipientIndex: number,
@@ -389,6 +494,15 @@ export async function downloadFile(
return processDownloadedFile(resolvedFd, chunks)
}
export async function receiveFile(
agent: XFTPClientAgent,
uri: string,
options?: {onProgress?: (downloaded: number, total: number) => void}
): Promise<DownloadResult> {
const fd = decodeDescriptionURI(uri)
return downloadFile(agent, fd, options?.onProgress)
}
async function resolveRedirect(
agent: XFTPClientAgent,
fd: FileDescription
xftp-web/src/crypto/file.ts
+25
View File
@@ -3,6 +3,7 @@
import {Decoder, concatBytes, encodeInt64, encodeString, decodeString, encodeMaybe, decodeMaybe} from "../protocol/encoding.js"
import {sbInit, sbEncryptChunk, sbDecryptTailTag, sbAuth} from "./secretbox.js"
import {prepareChunkSizes, fileSizeLen, authTagSize} from "../protocol/chunks.js"
const AUTH_TAG_SIZE = 16n
@@ -127,6 +128,30 @@ export async function encryptFileAsync(
if (out) return out
}
// -- Encryption preparation (key gen + chunk sizing)
export interface EncryptionParams {
fileHdr: Uint8Array
key: Uint8Array
nonce: Uint8Array
fileSize: bigint
encSize: bigint
chunkSizes: number[]
}
export function prepareEncryption(sourceSize: number, fileName: string): EncryptionParams {
const key = new Uint8Array(32)
const nonce = new Uint8Array(24)
crypto.getRandomValues(key)
crypto.getRandomValues(nonce)
const fileHdr = encodeFileHeader({fileName, fileExtra: null})
const fileSize = BigInt(fileHdr.length + sourceSize)
const payloadSize = Number(fileSize) + fileSizeLen + authTagSize
const chunkSizes = prepareChunkSizes(payloadSize)
const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0))
return {fileHdr, key, nonce, fileSize, encSize, chunkSizes}
}
// -- Decryption (FileTransfer.Crypto:decryptChunks)
// Decrypt one or more XFTP chunks into a FileHeader and file content.
xftp-web/web/crypto.worker.ts
+2 -11
View File
@@ -1,7 +1,6 @@
import sodium from 'libsodium-wrappers-sumo'
import {encryptFile, encodeFileHeader, decryptChunks} from '../src/crypto/file.js'
import {encryptFile, prepareEncryption, decryptChunks} from '../src/crypto/file.js'
import {sha512Streaming} from '../src/crypto/digest.js'
import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js'
import {decryptReceivedChunk} from '../src/download.js'
// ── OPFS session management ─────────────────────────────────────
@@ -40,15 +39,7 @@ async function sweepStale() {
async function handleEncrypt(id: number, data: ArrayBuffer, fileName: string) {
const source = new Uint8Array(data)
const key = new Uint8Array(32)
const nonce = new Uint8Array(24)
crypto.getRandomValues(key)
crypto.getRandomValues(nonce)
const fileHdr = encodeFileHeader({fileName, fileExtra: null})
const fileSize = BigInt(fileHdr.length + source.length)
const payloadSize = Number(fileSize) + fileSizeLen + authTagSize
const chunkSizes = prepareChunkSizes(payloadSize)
const encSize = BigInt(chunkSizes.reduce((a: number, b: number) => a + b, 0))
const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName)
const encData = encryptFile(source, fileHdr, key, nonce, fileSize, encSize)
self.postMessage({id, type: 'progress', done: 50, total: 100})